10f3154e6SShannon Nelson // SPDX-License-Identifier: GPL-2.0 20f3154e6SShannon Nelson /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 30f3154e6SShannon Nelson 40f3154e6SShannon Nelson #include <linux/ip.h> 50f3154e6SShannon Nelson #include <linux/ipv6.h> 60f3154e6SShannon Nelson #include <linux/if_vlan.h> 70f3154e6SShannon Nelson #include <net/ip6_checksum.h> 80f3154e6SShannon Nelson 90f3154e6SShannon Nelson #include "ionic.h" 100f3154e6SShannon Nelson #include "ionic_lif.h" 110f3154e6SShannon Nelson #include "ionic_txrx.h" 120f3154e6SShannon Nelson 13b14e4e95SShannon Nelson 14b14e4e95SShannon Nelson static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); 15b14e4e95SShannon Nelson 160f3154e6SShannon Nelson static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 170f3154e6SShannon Nelson ionic_desc_cb cb_func, void *cb_arg) 180f3154e6SShannon Nelson { 19f1d2e894SShannon Nelson DEBUG_STATS_TXQ_POST(q, ring_dbell); 200f3154e6SShannon Nelson 210f3154e6SShannon Nelson ionic_q_post(q, ring_dbell, cb_func, cb_arg); 220f3154e6SShannon Nelson } 230f3154e6SShannon Nelson 240f3154e6SShannon Nelson static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 250f3154e6SShannon Nelson ionic_desc_cb cb_func, void *cb_arg) 260f3154e6SShannon Nelson { 270f3154e6SShannon Nelson ionic_q_post(q, ring_dbell, cb_func, cb_arg); 280f3154e6SShannon Nelson 2934dec947SShannon Nelson DEBUG_STATS_RX_BUFF_CNT(q); 300f3154e6SShannon Nelson } 310f3154e6SShannon Nelson 320f3154e6SShannon Nelson static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 330f3154e6SShannon Nelson { 340f3154e6SShannon Nelson return netdev_get_tx_queue(q->lif->netdev, q->index); 350f3154e6SShannon Nelson } 360f3154e6SShannon Nelson 374b0a7539SShannon Nelson static void ionic_rx_buf_reset(struct ionic_buf_info *buf_info) 384b0a7539SShannon Nelson { 394b0a7539SShannon Nelson buf_info->page = NULL; 404b0a7539SShannon Nelson buf_info->page_offset = 0; 414b0a7539SShannon Nelson buf_info->dma_addr = 0; 424b0a7539SShannon Nelson } 434b0a7539SShannon Nelson 442b5720f2SShannon Nelson static int ionic_rx_page_alloc(struct ionic_queue *q, 454b0a7539SShannon Nelson struct ionic_buf_info *buf_info) 462b5720f2SShannon Nelson { 4789e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev; 482b5720f2SShannon Nelson struct ionic_rx_stats *stats; 492b5720f2SShannon Nelson struct device *dev; 502b5720f2SShannon Nelson 51f37bc346SShannon Nelson dev = q->dev; 522b5720f2SShannon Nelson stats = q_to_rx_stats(q); 532b5720f2SShannon Nelson 544b0a7539SShannon Nelson if (unlikely(!buf_info)) { 554b0a7539SShannon Nelson net_err_ratelimited("%s: %s invalid buf_info in alloc\n", 562b5720f2SShannon Nelson netdev->name, q->name); 572b5720f2SShannon Nelson return -EINVAL; 582b5720f2SShannon Nelson } 592b5720f2SShannon Nelson 604b0a7539SShannon Nelson buf_info->page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 614b0a7539SShannon Nelson if (unlikely(!buf_info->page)) { 622b5720f2SShannon Nelson net_err_ratelimited("%s: %s page alloc failed\n", 632b5720f2SShannon Nelson netdev->name, q->name); 642b5720f2SShannon Nelson stats->alloc_err++; 652b5720f2SShannon Nelson return -ENOMEM; 662b5720f2SShannon Nelson } 674b0a7539SShannon Nelson buf_info->page_offset = 0; 682b5720f2SShannon Nelson 694b0a7539SShannon Nelson buf_info->dma_addr = dma_map_page(dev, buf_info->page, buf_info->page_offset, 704b0a7539SShannon Nelson IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 714b0a7539SShannon Nelson if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { 724b0a7539SShannon Nelson __free_pages(buf_info->page, 0); 734b0a7539SShannon Nelson ionic_rx_buf_reset(buf_info); 742b5720f2SShannon Nelson net_err_ratelimited("%s: %s dma map failed\n", 752b5720f2SShannon Nelson netdev->name, q->name); 762b5720f2SShannon Nelson stats->dma_map_err++; 772b5720f2SShannon Nelson return -EIO; 782b5720f2SShannon Nelson } 792b5720f2SShannon Nelson 802b5720f2SShannon Nelson return 0; 812b5720f2SShannon Nelson } 822b5720f2SShannon Nelson 832b5720f2SShannon Nelson static void ionic_rx_page_free(struct ionic_queue *q, 844b0a7539SShannon Nelson struct ionic_buf_info *buf_info) 852b5720f2SShannon Nelson { 864b0a7539SShannon Nelson struct net_device *netdev = q->lif->netdev; 87f37bc346SShannon Nelson struct device *dev = q->dev; 882b5720f2SShannon Nelson 894b0a7539SShannon Nelson if (unlikely(!buf_info)) { 904b0a7539SShannon Nelson net_err_ratelimited("%s: %s invalid buf_info in free\n", 912b5720f2SShannon Nelson netdev->name, q->name); 922b5720f2SShannon Nelson return; 932b5720f2SShannon Nelson } 942b5720f2SShannon Nelson 954b0a7539SShannon Nelson if (!buf_info->page) 962b5720f2SShannon Nelson return; 974b0a7539SShannon Nelson 984b0a7539SShannon Nelson dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 994b0a7539SShannon Nelson __free_pages(buf_info->page, 0); 1004b0a7539SShannon Nelson ionic_rx_buf_reset(buf_info); 1012b5720f2SShannon Nelson } 1022b5720f2SShannon Nelson 1034b0a7539SShannon Nelson static bool ionic_rx_buf_recycle(struct ionic_queue *q, 1044b0a7539SShannon Nelson struct ionic_buf_info *buf_info, u32 used) 1054b0a7539SShannon Nelson { 1064b0a7539SShannon Nelson u32 size; 1072b5720f2SShannon Nelson 1084b0a7539SShannon Nelson /* don't re-use pages allocated in low-mem condition */ 1094b0a7539SShannon Nelson if (page_is_pfmemalloc(buf_info->page)) 1104b0a7539SShannon Nelson return false; 1114b0a7539SShannon Nelson 1124b0a7539SShannon Nelson /* don't re-use buffers from non-local numa nodes */ 1134b0a7539SShannon Nelson if (page_to_nid(buf_info->page) != numa_mem_id()) 1144b0a7539SShannon Nelson return false; 1154b0a7539SShannon Nelson 1164b0a7539SShannon Nelson size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); 1174b0a7539SShannon Nelson buf_info->page_offset += size; 1184b0a7539SShannon Nelson if (buf_info->page_offset >= IONIC_PAGE_SIZE) 1194b0a7539SShannon Nelson return false; 1204b0a7539SShannon Nelson 1214b0a7539SShannon Nelson get_page(buf_info->page); 1224b0a7539SShannon Nelson 1234b0a7539SShannon Nelson return true; 1242b5720f2SShannon Nelson } 1252b5720f2SShannon Nelson 12608f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 12708f2e4b2SShannon Nelson struct ionic_desc_info *desc_info, 128a25edab9SShannon Nelson struct ionic_rxq_comp *comp) 1290f3154e6SShannon Nelson { 13089e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev; 1314b0a7539SShannon Nelson struct ionic_buf_info *buf_info; 13289e572e7SShannon Nelson struct ionic_rx_stats *stats; 133f37bc346SShannon Nelson struct device *dev = q->dev; 13408f2e4b2SShannon Nelson struct sk_buff *skb; 13508f2e4b2SShannon Nelson unsigned int i; 13608f2e4b2SShannon Nelson u16 frag_len; 13708f2e4b2SShannon Nelson u16 len; 1380f3154e6SShannon Nelson 13989e572e7SShannon Nelson stats = q_to_rx_stats(q); 14089e572e7SShannon Nelson 1414b0a7539SShannon Nelson buf_info = &desc_info->bufs[0]; 14208f2e4b2SShannon Nelson len = le16_to_cpu(comp->len); 14308f2e4b2SShannon Nelson 1444b0a7539SShannon Nelson prefetch(buf_info->page); 14508f2e4b2SShannon Nelson 14689e572e7SShannon Nelson skb = napi_get_frags(&q_to_qcq(q)->napi); 14789e572e7SShannon Nelson if (unlikely(!skb)) { 14889e572e7SShannon Nelson net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 14989e572e7SShannon Nelson netdev->name, q->name); 15089e572e7SShannon Nelson stats->alloc_err++; 15108f2e4b2SShannon Nelson return NULL; 15289e572e7SShannon Nelson } 15308f2e4b2SShannon Nelson 15408f2e4b2SShannon Nelson i = comp->num_sg_elems + 1; 15508f2e4b2SShannon Nelson do { 1564b0a7539SShannon Nelson if (unlikely(!buf_info->page)) { 15708f2e4b2SShannon Nelson dev_kfree_skb(skb); 15808f2e4b2SShannon Nelson return NULL; 1590f3154e6SShannon Nelson } 1600f3154e6SShannon Nelson 1614b0a7539SShannon Nelson frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 16208f2e4b2SShannon Nelson len -= frag_len; 16308f2e4b2SShannon Nelson 1644b0a7539SShannon Nelson dma_sync_single_for_cpu(dev, 1654b0a7539SShannon Nelson buf_info->dma_addr + buf_info->page_offset, 1664b0a7539SShannon Nelson frag_len, DMA_FROM_DEVICE); 1674b0a7539SShannon Nelson 16808f2e4b2SShannon Nelson skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 1694b0a7539SShannon Nelson buf_info->page, buf_info->page_offset, frag_len, 1704b0a7539SShannon Nelson IONIC_PAGE_SIZE); 1714b0a7539SShannon Nelson 1724b0a7539SShannon Nelson if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 1734b0a7539SShannon Nelson dma_unmap_page(dev, buf_info->dma_addr, 1744b0a7539SShannon Nelson IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 1754b0a7539SShannon Nelson ionic_rx_buf_reset(buf_info); 1764b0a7539SShannon Nelson } 1774b0a7539SShannon Nelson 1784b0a7539SShannon Nelson buf_info++; 1794b0a7539SShannon Nelson 18008f2e4b2SShannon Nelson i--; 18108f2e4b2SShannon Nelson } while (i > 0); 18208f2e4b2SShannon Nelson 18308f2e4b2SShannon Nelson return skb; 1840f3154e6SShannon Nelson } 1850f3154e6SShannon Nelson 18608f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 18708f2e4b2SShannon Nelson struct ionic_desc_info *desc_info, 188a25edab9SShannon Nelson struct ionic_rxq_comp *comp) 18908f2e4b2SShannon Nelson { 19089e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev; 1914b0a7539SShannon Nelson struct ionic_buf_info *buf_info; 19289e572e7SShannon Nelson struct ionic_rx_stats *stats; 193f37bc346SShannon Nelson struct device *dev = q->dev; 19408f2e4b2SShannon Nelson struct sk_buff *skb; 19508f2e4b2SShannon Nelson u16 len; 1960f3154e6SShannon Nelson 19789e572e7SShannon Nelson stats = q_to_rx_stats(q); 19889e572e7SShannon Nelson 1994b0a7539SShannon Nelson buf_info = &desc_info->bufs[0]; 20008f2e4b2SShannon Nelson len = le16_to_cpu(comp->len); 2010f3154e6SShannon Nelson 20289e572e7SShannon Nelson skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 20389e572e7SShannon Nelson if (unlikely(!skb)) { 20489e572e7SShannon Nelson net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 20589e572e7SShannon Nelson netdev->name, q->name); 20689e572e7SShannon Nelson stats->alloc_err++; 20708f2e4b2SShannon Nelson return NULL; 20889e572e7SShannon Nelson } 2090f3154e6SShannon Nelson 2104b0a7539SShannon Nelson if (unlikely(!buf_info->page)) { 21108f2e4b2SShannon Nelson dev_kfree_skb(skb); 21208f2e4b2SShannon Nelson return NULL; 21308f2e4b2SShannon Nelson } 21408f2e4b2SShannon Nelson 2154b0a7539SShannon Nelson dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, 21608f2e4b2SShannon Nelson len, DMA_FROM_DEVICE); 2174b0a7539SShannon Nelson skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); 2184b0a7539SShannon Nelson dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, 21908f2e4b2SShannon Nelson len, DMA_FROM_DEVICE); 22008f2e4b2SShannon Nelson 22108f2e4b2SShannon Nelson skb_put(skb, len); 22208f2e4b2SShannon Nelson skb->protocol = eth_type_trans(skb, q->lif->netdev); 22308f2e4b2SShannon Nelson 22408f2e4b2SShannon Nelson return skb; 2250f3154e6SShannon Nelson } 2260f3154e6SShannon Nelson 2275b3f3f2aSShannon Nelson static void ionic_rx_clean(struct ionic_queue *q, 2285b3f3f2aSShannon Nelson struct ionic_desc_info *desc_info, 2295b3f3f2aSShannon Nelson struct ionic_cq_info *cq_info, 2305b3f3f2aSShannon Nelson void *cb_arg) 2310f3154e6SShannon Nelson { 232a25edab9SShannon Nelson struct ionic_rxq_comp *comp = cq_info->rxcq; 23389e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev; 2340f3154e6SShannon Nelson struct ionic_qcq *qcq = q_to_qcq(q); 2350f3154e6SShannon Nelson struct ionic_rx_stats *stats; 23608f2e4b2SShannon Nelson struct sk_buff *skb; 2370f3154e6SShannon Nelson 2380f3154e6SShannon Nelson stats = q_to_rx_stats(q); 2390f3154e6SShannon Nelson 24024cfa8c7SShannon Nelson if (comp->status) { 24124cfa8c7SShannon Nelson stats->dropped++; 2420f3154e6SShannon Nelson return; 24324cfa8c7SShannon Nelson } 2440f3154e6SShannon Nelson 2450f3154e6SShannon Nelson stats->pkts++; 2460f3154e6SShannon Nelson stats->bytes += le16_to_cpu(comp->len); 2470f3154e6SShannon Nelson 24808f2e4b2SShannon Nelson if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 249a25edab9SShannon Nelson skb = ionic_rx_copybreak(q, desc_info, comp); 25008f2e4b2SShannon Nelson else 251a25edab9SShannon Nelson skb = ionic_rx_frags(q, desc_info, comp); 2520f3154e6SShannon Nelson 25324cfa8c7SShannon Nelson if (unlikely(!skb)) { 25424cfa8c7SShannon Nelson stats->dropped++; 25508f2e4b2SShannon Nelson return; 25624cfa8c7SShannon Nelson } 2570f3154e6SShannon Nelson 2580f3154e6SShannon Nelson skb_record_rx_queue(skb, q->index); 2590f3154e6SShannon Nelson 26008f2e4b2SShannon Nelson if (likely(netdev->features & NETIF_F_RXHASH)) { 2610f3154e6SShannon Nelson switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 2620f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4: 2630f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6: 2640f3154e6SShannon Nelson skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 2650f3154e6SShannon Nelson PKT_HASH_TYPE_L3); 2660f3154e6SShannon Nelson break; 2670f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4_TCP: 2680f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6_TCP: 2690f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4_UDP: 2700f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6_UDP: 2710f3154e6SShannon Nelson skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 2720f3154e6SShannon Nelson PKT_HASH_TYPE_L4); 2730f3154e6SShannon Nelson break; 2740f3154e6SShannon Nelson } 2750f3154e6SShannon Nelson } 2760f3154e6SShannon Nelson 27708f2e4b2SShannon Nelson if (likely(netdev->features & NETIF_F_RXCSUM)) { 2780f3154e6SShannon Nelson if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 2790f3154e6SShannon Nelson skb->ip_summed = CHECKSUM_COMPLETE; 280d701ec32SShannon Nelson skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 2810f3154e6SShannon Nelson stats->csum_complete++; 2820f3154e6SShannon Nelson } 2830f3154e6SShannon Nelson } else { 2840f3154e6SShannon Nelson stats->csum_none++; 2850f3154e6SShannon Nelson } 2860f3154e6SShannon Nelson 28708f2e4b2SShannon Nelson if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 2880f3154e6SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 28908f2e4b2SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 2900f3154e6SShannon Nelson stats->csum_error++; 2910f3154e6SShannon Nelson 292f64e0c56SShannon Nelson if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 293f64e0c56SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 2940f3154e6SShannon Nelson __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 2950f3154e6SShannon Nelson le16_to_cpu(comp->vlan_tci)); 296f64e0c56SShannon Nelson stats->vlan_stripped++; 2970f3154e6SShannon Nelson } 2980f3154e6SShannon Nelson 29908f2e4b2SShannon Nelson if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 3000f3154e6SShannon Nelson napi_gro_receive(&qcq->napi, skb); 30108f2e4b2SShannon Nelson else 30208f2e4b2SShannon Nelson napi_gro_frags(&qcq->napi); 3030f3154e6SShannon Nelson } 3040f3154e6SShannon Nelson 3050f3154e6SShannon Nelson static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 3060f3154e6SShannon Nelson { 307a25edab9SShannon Nelson struct ionic_rxq_comp *comp = cq_info->rxcq; 3080f3154e6SShannon Nelson struct ionic_queue *q = cq->bound_q; 3090f3154e6SShannon Nelson struct ionic_desc_info *desc_info; 3100f3154e6SShannon Nelson 3110f3154e6SShannon Nelson if (!color_match(comp->pkt_type_color, cq->done_color)) 3120f3154e6SShannon Nelson return false; 3130f3154e6SShannon Nelson 3140f3154e6SShannon Nelson /* check for empty queue */ 315f1d2e894SShannon Nelson if (q->tail_idx == q->head_idx) 3160f3154e6SShannon Nelson return false; 3170f3154e6SShannon Nelson 318339dcf7fSShannon Nelson if (q->tail_idx != le16_to_cpu(comp->comp_index)) 3190f3154e6SShannon Nelson return false; 3200f3154e6SShannon Nelson 321339dcf7fSShannon Nelson desc_info = &q->info[q->tail_idx]; 322f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 3230f3154e6SShannon Nelson 3240f3154e6SShannon Nelson /* clean the related q entry, only one per qc completion */ 3250f3154e6SShannon Nelson ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 3260f3154e6SShannon Nelson 3270f3154e6SShannon Nelson desc_info->cb = NULL; 3280f3154e6SShannon Nelson desc_info->cb_arg = NULL; 3290f3154e6SShannon Nelson 3300f3154e6SShannon Nelson return true; 3310f3154e6SShannon Nelson } 3320f3154e6SShannon Nelson 3330f3154e6SShannon Nelson void ionic_rx_fill(struct ionic_queue *q) 3340f3154e6SShannon Nelson { 3350f3154e6SShannon Nelson struct net_device *netdev = q->lif->netdev; 33608f2e4b2SShannon Nelson struct ionic_desc_info *desc_info; 33708f2e4b2SShannon Nelson struct ionic_rxq_sg_desc *sg_desc; 33808f2e4b2SShannon Nelson struct ionic_rxq_sg_elem *sg_elem; 3394b0a7539SShannon Nelson struct ionic_buf_info *buf_info; 3400f3154e6SShannon Nelson struct ionic_rxq_desc *desc; 341c37d6e3fSShannon Nelson unsigned int remain_len; 3424b0a7539SShannon Nelson unsigned int frag_len; 34308f2e4b2SShannon Nelson unsigned int nfrags; 34408f2e4b2SShannon Nelson unsigned int i, j; 3450f3154e6SShannon Nelson unsigned int len; 3460f3154e6SShannon Nelson 34783469893SShannon Nelson len = netdev->mtu + ETH_HLEN + VLAN_HLEN; 3480f3154e6SShannon Nelson 3490f3154e6SShannon Nelson for (i = ionic_q_space_avail(q); i; i--) { 3504b0a7539SShannon Nelson nfrags = 0; 351c37d6e3fSShannon Nelson remain_len = len; 352f1d2e894SShannon Nelson desc_info = &q->info[q->head_idx]; 35308f2e4b2SShannon Nelson desc = desc_info->desc; 3544b0a7539SShannon Nelson buf_info = &desc_info->bufs[0]; 3550f3154e6SShannon Nelson 3564b0a7539SShannon Nelson if (!buf_info->page) { /* alloc a new buffer? */ 3574b0a7539SShannon Nelson if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 35808f2e4b2SShannon Nelson desc->addr = 0; 35908f2e4b2SShannon Nelson desc->len = 0; 36008f2e4b2SShannon Nelson return; 36108f2e4b2SShannon Nelson } 3624b0a7539SShannon Nelson } 36308f2e4b2SShannon Nelson 3644b0a7539SShannon Nelson /* fill main descriptor - buf[0] */ 3654b0a7539SShannon Nelson desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 3664b0a7539SShannon Nelson frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 3674b0a7539SShannon Nelson desc->len = cpu_to_le16(frag_len); 3684b0a7539SShannon Nelson remain_len -= frag_len; 3694b0a7539SShannon Nelson buf_info++; 3704b0a7539SShannon Nelson nfrags++; 37108f2e4b2SShannon Nelson 3724b0a7539SShannon Nelson /* fill sg descriptors - buf[1..n] */ 3734b0a7539SShannon Nelson sg_desc = desc_info->sg_desc; 374f37bc346SShannon Nelson for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { 37508f2e4b2SShannon Nelson sg_elem = &sg_desc->elems[j]; 3764b0a7539SShannon Nelson if (!buf_info->page) { /* alloc a new sg buffer? */ 3774b0a7539SShannon Nelson if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 37808f2e4b2SShannon Nelson sg_elem->addr = 0; 37908f2e4b2SShannon Nelson sg_elem->len = 0; 38008f2e4b2SShannon Nelson return; 38108f2e4b2SShannon Nelson } 38208f2e4b2SShannon Nelson } 3830f3154e6SShannon Nelson 3844b0a7539SShannon Nelson sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 3854b0a7539SShannon Nelson frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset); 3864b0a7539SShannon Nelson sg_elem->len = cpu_to_le16(frag_len); 3874b0a7539SShannon Nelson remain_len -= frag_len; 3884b0a7539SShannon Nelson buf_info++; 3894b0a7539SShannon Nelson nfrags++; 3904b0a7539SShannon Nelson } 3914b0a7539SShannon Nelson 3924b0a7539SShannon Nelson /* clear end sg element as a sentinel */ 393f37bc346SShannon Nelson if (j < q->max_sg_elems) { 3944b0a7539SShannon Nelson sg_elem = &sg_desc->elems[j]; 3954b0a7539SShannon Nelson memset(sg_elem, 0, sizeof(*sg_elem)); 3964b0a7539SShannon Nelson } 3974b0a7539SShannon Nelson 3984b0a7539SShannon Nelson desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 3994b0a7539SShannon Nelson IONIC_RXQ_DESC_OPCODE_SIMPLE; 4004b0a7539SShannon Nelson desc_info->nbufs = nfrags; 4014b0a7539SShannon Nelson 402155f15adSShannon Nelson ionic_rxq_post(q, false, ionic_rx_clean, NULL); 4030f3154e6SShannon Nelson } 404155f15adSShannon Nelson 405155f15adSShannon Nelson ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 406f1d2e894SShannon Nelson q->dbval | q->head_idx); 4070f3154e6SShannon Nelson } 4080f3154e6SShannon Nelson 4090f3154e6SShannon Nelson void ionic_rx_empty(struct ionic_queue *q) 4100f3154e6SShannon Nelson { 411f1d2e894SShannon Nelson struct ionic_desc_info *desc_info; 4124b0a7539SShannon Nelson struct ionic_buf_info *buf_info; 4130c32a28eSShannon Nelson unsigned int i, j; 4140f3154e6SShannon Nelson 4150c32a28eSShannon Nelson for (i = 0; i < q->num_descs; i++) { 4160c32a28eSShannon Nelson desc_info = &q->info[i]; 4170c32a28eSShannon Nelson for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 4184b0a7539SShannon Nelson buf_info = &desc_info->bufs[j]; 4194b0a7539SShannon Nelson if (buf_info->page) 4204b0a7539SShannon Nelson ionic_rx_page_free(q, buf_info); 4210c32a28eSShannon Nelson } 42208f2e4b2SShannon Nelson 4234b0a7539SShannon Nelson desc_info->nbufs = 0; 4240c32a28eSShannon Nelson desc_info->cb = NULL; 425f1d2e894SShannon Nelson desc_info->cb_arg = NULL; 4260f3154e6SShannon Nelson } 4274b0a7539SShannon Nelson 4284b0a7539SShannon Nelson q->head_idx = 0; 4294b0a7539SShannon Nelson q->tail_idx = 0; 4300f3154e6SShannon Nelson } 4310f3154e6SShannon Nelson 43204a83459SShannon Nelson static void ionic_dim_update(struct ionic_qcq *qcq) 43304a83459SShannon Nelson { 43404a83459SShannon Nelson struct dim_sample dim_sample; 43504a83459SShannon Nelson struct ionic_lif *lif; 43604a83459SShannon Nelson unsigned int qi; 43704a83459SShannon Nelson 43804a83459SShannon Nelson if (!qcq->intr.dim_coal_hw) 43904a83459SShannon Nelson return; 44004a83459SShannon Nelson 44104a83459SShannon Nelson lif = qcq->q.lif; 44204a83459SShannon Nelson qi = qcq->cq.bound_q->index; 44304a83459SShannon Nelson 44404a83459SShannon Nelson ionic_intr_coal_init(lif->ionic->idev.intr_ctrl, 44504a83459SShannon Nelson lif->rxqcqs[qi]->intr.index, 44604a83459SShannon Nelson qcq->intr.dim_coal_hw); 44704a83459SShannon Nelson 44804a83459SShannon Nelson dim_update_sample(qcq->cq.bound_intr->rearm_count, 44904a83459SShannon Nelson lif->txqstats[qi].pkts, 45004a83459SShannon Nelson lif->txqstats[qi].bytes, 45104a83459SShannon Nelson &dim_sample); 45204a83459SShannon Nelson 45304a83459SShannon Nelson net_dim(&qcq->dim, dim_sample); 45404a83459SShannon Nelson } 45504a83459SShannon Nelson 456fe8c30b5SShannon Nelson int ionic_tx_napi(struct napi_struct *napi, int budget) 457fe8c30b5SShannon Nelson { 458fe8c30b5SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi); 459fe8c30b5SShannon Nelson struct ionic_cq *cq = napi_to_cq(napi); 460fe8c30b5SShannon Nelson struct ionic_dev *idev; 461fe8c30b5SShannon Nelson struct ionic_lif *lif; 462fe8c30b5SShannon Nelson u32 work_done = 0; 463fe8c30b5SShannon Nelson u32 flags = 0; 464fe8c30b5SShannon Nelson 465fe8c30b5SShannon Nelson lif = cq->bound_q->lif; 466fe8c30b5SShannon Nelson idev = &lif->ionic->idev; 467fe8c30b5SShannon Nelson 468fe8c30b5SShannon Nelson work_done = ionic_cq_service(cq, budget, 469fe8c30b5SShannon Nelson ionic_tx_service, NULL, NULL); 470fe8c30b5SShannon Nelson 471fe8c30b5SShannon Nelson if (work_done < budget && napi_complete_done(napi, work_done)) { 47204a83459SShannon Nelson ionic_dim_update(qcq); 473fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK; 47404a83459SShannon Nelson cq->bound_intr->rearm_count++; 475fe8c30b5SShannon Nelson } 476fe8c30b5SShannon Nelson 477fe8c30b5SShannon Nelson if (work_done || flags) { 478fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE; 479fe8c30b5SShannon Nelson ionic_intr_credits(idev->intr_ctrl, 480fe8c30b5SShannon Nelson cq->bound_intr->index, 481fe8c30b5SShannon Nelson work_done, flags); 482fe8c30b5SShannon Nelson } 483fe8c30b5SShannon Nelson 484fe8c30b5SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, work_done); 485fe8c30b5SShannon Nelson 486fe8c30b5SShannon Nelson return work_done; 487fe8c30b5SShannon Nelson } 488fe8c30b5SShannon Nelson 4890f3154e6SShannon Nelson int ionic_rx_napi(struct napi_struct *napi, int budget) 4900f3154e6SShannon Nelson { 4910f3154e6SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi); 492fe8c30b5SShannon Nelson struct ionic_cq *cq = napi_to_cq(napi); 493fe8c30b5SShannon Nelson struct ionic_dev *idev; 494fe8c30b5SShannon Nelson struct ionic_lif *lif; 495a8205ab6SShannon Nelson u16 rx_fill_threshold; 496fe8c30b5SShannon Nelson u32 work_done = 0; 497fe8c30b5SShannon Nelson u32 flags = 0; 498fe8c30b5SShannon Nelson 499fe8c30b5SShannon Nelson lif = cq->bound_q->lif; 500fe8c30b5SShannon Nelson idev = &lif->ionic->idev; 501fe8c30b5SShannon Nelson 502fe8c30b5SShannon Nelson work_done = ionic_cq_service(cq, budget, 503fe8c30b5SShannon Nelson ionic_rx_service, NULL, NULL); 504fe8c30b5SShannon Nelson 505a8205ab6SShannon Nelson rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD, 506a8205ab6SShannon Nelson cq->num_descs / IONIC_RX_FILL_DIV); 507a8205ab6SShannon Nelson if (work_done && ionic_q_space_avail(cq->bound_q) >= rx_fill_threshold) 508fe8c30b5SShannon Nelson ionic_rx_fill(cq->bound_q); 509fe8c30b5SShannon Nelson 510fe8c30b5SShannon Nelson if (work_done < budget && napi_complete_done(napi, work_done)) { 51104a83459SShannon Nelson ionic_dim_update(qcq); 512fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK; 51304a83459SShannon Nelson cq->bound_intr->rearm_count++; 514fe8c30b5SShannon Nelson } 515fe8c30b5SShannon Nelson 516fe8c30b5SShannon Nelson if (work_done || flags) { 517fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE; 518fe8c30b5SShannon Nelson ionic_intr_credits(idev->intr_ctrl, 519fe8c30b5SShannon Nelson cq->bound_intr->index, 520fe8c30b5SShannon Nelson work_done, flags); 521fe8c30b5SShannon Nelson } 522fe8c30b5SShannon Nelson 523fe8c30b5SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, work_done); 524fe8c30b5SShannon Nelson 525fe8c30b5SShannon Nelson return work_done; 526fe8c30b5SShannon Nelson } 527fe8c30b5SShannon Nelson 528fe8c30b5SShannon Nelson int ionic_txrx_napi(struct napi_struct *napi, int budget) 529fe8c30b5SShannon Nelson { 530fe8c30b5SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi); 5310f3154e6SShannon Nelson struct ionic_cq *rxcq = napi_to_cq(napi); 5320f3154e6SShannon Nelson unsigned int qi = rxcq->bound_q->index; 5330f3154e6SShannon Nelson struct ionic_dev *idev; 5340f3154e6SShannon Nelson struct ionic_lif *lif; 5350f3154e6SShannon Nelson struct ionic_cq *txcq; 536a8205ab6SShannon Nelson u16 rx_fill_threshold; 537b14e4e95SShannon Nelson u32 rx_work_done = 0; 538b14e4e95SShannon Nelson u32 tx_work_done = 0; 5390f3154e6SShannon Nelson u32 flags = 0; 5400f3154e6SShannon Nelson 5410f3154e6SShannon Nelson lif = rxcq->bound_q->lif; 5420f3154e6SShannon Nelson idev = &lif->ionic->idev; 54334dec947SShannon Nelson txcq = &lif->txqcqs[qi]->cq; 5440f3154e6SShannon Nelson 545f37bc346SShannon Nelson tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, 546b14e4e95SShannon Nelson ionic_tx_service, NULL, NULL); 5470f3154e6SShannon Nelson 548b14e4e95SShannon Nelson rx_work_done = ionic_cq_service(rxcq, budget, 549b14e4e95SShannon Nelson ionic_rx_service, NULL, NULL); 550a8205ab6SShannon Nelson 551a8205ab6SShannon Nelson rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD, 552a8205ab6SShannon Nelson rxcq->num_descs / IONIC_RX_FILL_DIV); 553a8205ab6SShannon Nelson if (rx_work_done && ionic_q_space_avail(rxcq->bound_q) >= rx_fill_threshold) 554a8205ab6SShannon Nelson ionic_rx_fill(rxcq->bound_q); 5550f3154e6SShannon Nelson 5569dda5110SShannon Nelson if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 55704a83459SShannon Nelson ionic_dim_update(qcq); 5580f3154e6SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK; 55904a83459SShannon Nelson rxcq->bound_intr->rearm_count++; 5600f3154e6SShannon Nelson } 5610f3154e6SShannon Nelson 5629dda5110SShannon Nelson if (rx_work_done || flags) { 5630f3154e6SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE; 5640f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 565b14e4e95SShannon Nelson tx_work_done + rx_work_done, flags); 5660f3154e6SShannon Nelson } 5670f3154e6SShannon Nelson 568b14e4e95SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, rx_work_done); 569b14e4e95SShannon Nelson DEBUG_STATS_NAPI_POLL(qcq, tx_work_done); 5700f3154e6SShannon Nelson 5719dda5110SShannon Nelson return rx_work_done; 5720f3154e6SShannon Nelson } 5730f3154e6SShannon Nelson 5745b3f3f2aSShannon Nelson static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 5755b3f3f2aSShannon Nelson void *data, size_t len) 5760f3154e6SShannon Nelson { 5770f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 578f37bc346SShannon Nelson struct device *dev = q->dev; 5790f3154e6SShannon Nelson dma_addr_t dma_addr; 5800f3154e6SShannon Nelson 5810f3154e6SShannon Nelson dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 5820f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) { 5830f3154e6SShannon Nelson net_warn_ratelimited("%s: DMA single map failed on %s!\n", 5840f3154e6SShannon Nelson q->lif->netdev->name, q->name); 5850f3154e6SShannon Nelson stats->dma_map_err++; 5860f3154e6SShannon Nelson return 0; 5870f3154e6SShannon Nelson } 5880f3154e6SShannon Nelson return dma_addr; 5890f3154e6SShannon Nelson } 5900f3154e6SShannon Nelson 5915b3f3f2aSShannon Nelson static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 5925b3f3f2aSShannon Nelson const skb_frag_t *frag, 5930f3154e6SShannon Nelson size_t offset, size_t len) 5940f3154e6SShannon Nelson { 5950f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 596f37bc346SShannon Nelson struct device *dev = q->dev; 5970f3154e6SShannon Nelson dma_addr_t dma_addr; 5980f3154e6SShannon Nelson 5990f3154e6SShannon Nelson dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 6000f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) { 6010f3154e6SShannon Nelson net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 6020f3154e6SShannon Nelson q->lif->netdev->name, q->name); 6030f3154e6SShannon Nelson stats->dma_map_err++; 6040f3154e6SShannon Nelson } 6050f3154e6SShannon Nelson return dma_addr; 6060f3154e6SShannon Nelson } 6070f3154e6SShannon Nelson 608*2da479caSShannon Nelson static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 609*2da479caSShannon Nelson struct ionic_desc_info *desc_info) 6105b039241SShannon Nelson { 611*2da479caSShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs; 6125b039241SShannon Nelson struct device *dev = q->dev; 6135b039241SShannon Nelson dma_addr_t dma_addr; 614*2da479caSShannon Nelson unsigned int nfrags; 6155b039241SShannon Nelson skb_frag_t *frag; 6165b039241SShannon Nelson int frag_idx; 6175b039241SShannon Nelson 6185b039241SShannon Nelson dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 6195b039241SShannon Nelson if (dma_mapping_error(dev, dma_addr)) 6205b039241SShannon Nelson return -EIO; 6215b039241SShannon Nelson buf_info->dma_addr = dma_addr; 6225b039241SShannon Nelson buf_info->len = skb_headlen(skb); 6235b039241SShannon Nelson buf_info++; 6245b039241SShannon Nelson 625*2da479caSShannon Nelson frag = skb_shinfo(skb)->frags; 626*2da479caSShannon Nelson nfrags = skb_shinfo(skb)->nr_frags; 627*2da479caSShannon Nelson for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 6285b039241SShannon Nelson dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 6295b039241SShannon Nelson if (dma_mapping_error(dev, dma_addr)) 6305b039241SShannon Nelson goto dma_fail; 6315b039241SShannon Nelson buf_info->dma_addr = dma_addr; 6325b039241SShannon Nelson buf_info->len = skb_frag_size(frag); 633*2da479caSShannon Nelson buf_info++; 6345b039241SShannon Nelson } 6355b039241SShannon Nelson 636*2da479caSShannon Nelson desc_info->nbufs = 1 + nfrags; 637*2da479caSShannon Nelson 6385b039241SShannon Nelson return 0; 6395b039241SShannon Nelson 6405b039241SShannon Nelson dma_fail: 6415b039241SShannon Nelson /* unwind the frag mappings and the head mapping */ 6425b039241SShannon Nelson while (frag_idx > 0) { 6435b039241SShannon Nelson frag_idx--; 6445b039241SShannon Nelson buf_info--; 6455b039241SShannon Nelson dma_unmap_page(dev, buf_info->dma_addr, 6465b039241SShannon Nelson buf_info->len, DMA_TO_DEVICE); 6475b039241SShannon Nelson } 6485b039241SShannon Nelson dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); 6495b039241SShannon Nelson return -EIO; 6505b039241SShannon Nelson } 6515b039241SShannon Nelson 6525b3f3f2aSShannon Nelson static void ionic_tx_clean(struct ionic_queue *q, 6535b3f3f2aSShannon Nelson struct ionic_desc_info *desc_info, 6545b3f3f2aSShannon Nelson struct ionic_cq_info *cq_info, 6555b3f3f2aSShannon Nelson void *cb_arg) 6560f3154e6SShannon Nelson { 6570f3154e6SShannon Nelson struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc; 6585b039241SShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs; 6590f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem = sg_desc->elems; 6600f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 6610f3154e6SShannon Nelson struct ionic_txq_desc *desc = desc_info->desc; 662f37bc346SShannon Nelson struct device *dev = q->dev; 6630f3154e6SShannon Nelson u8 opcode, flags, nsge; 6640f3154e6SShannon Nelson u16 queue_index; 6650f3154e6SShannon Nelson unsigned int i; 6660f3154e6SShannon Nelson u64 addr; 6670f3154e6SShannon Nelson 6680f3154e6SShannon Nelson decode_txq_desc_cmd(le64_to_cpu(desc->cmd), 6690f3154e6SShannon Nelson &opcode, &flags, &nsge, &addr); 6700f3154e6SShannon Nelson 6715b039241SShannon Nelson if (opcode != IONIC_TXQ_DESC_OPCODE_TSO) { 6720f3154e6SShannon Nelson dma_unmap_single(dev, (dma_addr_t)addr, 6730f3154e6SShannon Nelson le16_to_cpu(desc->len), DMA_TO_DEVICE); 6740f3154e6SShannon Nelson for (i = 0; i < nsge; i++, elem++) 6750f3154e6SShannon Nelson dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr), 6760f3154e6SShannon Nelson le16_to_cpu(elem->len), DMA_TO_DEVICE); 6775b039241SShannon Nelson } else { 6785b039241SShannon Nelson if (flags & IONIC_TXQ_DESC_FLAG_TSO_EOT) { 6795b039241SShannon Nelson dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, 6805b039241SShannon Nelson buf_info->len, DMA_TO_DEVICE); 6815b039241SShannon Nelson buf_info++; 6825b039241SShannon Nelson for (i = 1; i < desc_info->nbufs; i++, buf_info++) 6835b039241SShannon Nelson dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, 6845b039241SShannon Nelson buf_info->len, DMA_TO_DEVICE); 6855b039241SShannon Nelson } 6865b039241SShannon Nelson } 6870f3154e6SShannon Nelson 6880f3154e6SShannon Nelson if (cb_arg) { 6890f3154e6SShannon Nelson struct sk_buff *skb = cb_arg; 6900f3154e6SShannon Nelson u32 len = skb->len; 6910f3154e6SShannon Nelson 6920f3154e6SShannon Nelson queue_index = skb_get_queue_mapping(skb); 6930f3154e6SShannon Nelson if (unlikely(__netif_subqueue_stopped(q->lif->netdev, 6940f3154e6SShannon Nelson queue_index))) { 6950f3154e6SShannon Nelson netif_wake_subqueue(q->lif->netdev, queue_index); 6960f3154e6SShannon Nelson q->wake++; 6970f3154e6SShannon Nelson } 6980f3154e6SShannon Nelson dev_kfree_skb_any(skb); 6990f3154e6SShannon Nelson stats->clean++; 7000f3154e6SShannon Nelson netdev_tx_completed_queue(q_to_ndq(q), 1, len); 7010f3154e6SShannon Nelson } 7020f3154e6SShannon Nelson } 7030f3154e6SShannon Nelson 704b14e4e95SShannon Nelson static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 7050f3154e6SShannon Nelson { 706a25edab9SShannon Nelson struct ionic_txq_comp *comp = cq_info->txcq; 7070f3154e6SShannon Nelson struct ionic_queue *q = cq->bound_q; 7080f3154e6SShannon Nelson struct ionic_desc_info *desc_info; 709339dcf7fSShannon Nelson u16 index; 7100f3154e6SShannon Nelson 711b14e4e95SShannon Nelson if (!color_match(comp->color, cq->done_color)) 712b14e4e95SShannon Nelson return false; 7130f3154e6SShannon Nelson 7140f3154e6SShannon Nelson /* clean the related q entries, there could be 7150f3154e6SShannon Nelson * several q entries completed for each cq completion 7160f3154e6SShannon Nelson */ 7170f3154e6SShannon Nelson do { 718f1d2e894SShannon Nelson desc_info = &q->info[q->tail_idx]; 719339dcf7fSShannon Nelson index = q->tail_idx; 720f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 721f1d2e894SShannon Nelson ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 7220f3154e6SShannon Nelson desc_info->cb = NULL; 7230f3154e6SShannon Nelson desc_info->cb_arg = NULL; 724339dcf7fSShannon Nelson } while (index != le16_to_cpu(comp->comp_index)); 7250f3154e6SShannon Nelson 726b14e4e95SShannon Nelson return true; 7270f3154e6SShannon Nelson } 7280f3154e6SShannon Nelson 729b14e4e95SShannon Nelson void ionic_tx_flush(struct ionic_cq *cq) 730b14e4e95SShannon Nelson { 731b14e4e95SShannon Nelson struct ionic_dev *idev = &cq->lif->ionic->idev; 732b14e4e95SShannon Nelson u32 work_done; 733b14e4e95SShannon Nelson 734b14e4e95SShannon Nelson work_done = ionic_cq_service(cq, cq->num_descs, 735b14e4e95SShannon Nelson ionic_tx_service, NULL, NULL); 7360f3154e6SShannon Nelson if (work_done) 7370f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 738b14e4e95SShannon Nelson work_done, IONIC_INTR_CRED_RESET_COALESCE); 7390f3154e6SShannon Nelson } 7400f3154e6SShannon Nelson 741f9c00e2cSShannon Nelson void ionic_tx_empty(struct ionic_queue *q) 742f9c00e2cSShannon Nelson { 743f9c00e2cSShannon Nelson struct ionic_desc_info *desc_info; 744f9c00e2cSShannon Nelson 745f9c00e2cSShannon Nelson /* walk the not completed tx entries, if any */ 746f1d2e894SShannon Nelson while (q->head_idx != q->tail_idx) { 747f1d2e894SShannon Nelson desc_info = &q->info[q->tail_idx]; 748f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 749f9c00e2cSShannon Nelson ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 750f9c00e2cSShannon Nelson desc_info->cb = NULL; 751f9c00e2cSShannon Nelson desc_info->cb_arg = NULL; 752f9c00e2cSShannon Nelson } 753f9c00e2cSShannon Nelson } 754f9c00e2cSShannon Nelson 7550f3154e6SShannon Nelson static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 7560f3154e6SShannon Nelson { 7570f3154e6SShannon Nelson int err; 7580f3154e6SShannon Nelson 7590f3154e6SShannon Nelson err = skb_cow_head(skb, 0); 7600f3154e6SShannon Nelson if (err) 7610f3154e6SShannon Nelson return err; 7620f3154e6SShannon Nelson 7630f3154e6SShannon Nelson if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 7640f3154e6SShannon Nelson inner_ip_hdr(skb)->check = 0; 7650f3154e6SShannon Nelson inner_tcp_hdr(skb)->check = 7660f3154e6SShannon Nelson ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 7670f3154e6SShannon Nelson inner_ip_hdr(skb)->daddr, 7680f3154e6SShannon Nelson 0, IPPROTO_TCP, 0); 7690f3154e6SShannon Nelson } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 7700f3154e6SShannon Nelson inner_tcp_hdr(skb)->check = 7710f3154e6SShannon Nelson ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 7720f3154e6SShannon Nelson &inner_ipv6_hdr(skb)->daddr, 7730f3154e6SShannon Nelson 0, IPPROTO_TCP, 0); 7740f3154e6SShannon Nelson } 7750f3154e6SShannon Nelson 7760f3154e6SShannon Nelson return 0; 7770f3154e6SShannon Nelson } 7780f3154e6SShannon Nelson 7790f3154e6SShannon Nelson static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 7800f3154e6SShannon Nelson { 7810f3154e6SShannon Nelson int err; 7820f3154e6SShannon Nelson 7830f3154e6SShannon Nelson err = skb_cow_head(skb, 0); 7840f3154e6SShannon Nelson if (err) 7850f3154e6SShannon Nelson return err; 7860f3154e6SShannon Nelson 7870f3154e6SShannon Nelson if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 7880f3154e6SShannon Nelson ip_hdr(skb)->check = 0; 7890f3154e6SShannon Nelson tcp_hdr(skb)->check = 7900f3154e6SShannon Nelson ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 7910f3154e6SShannon Nelson ip_hdr(skb)->daddr, 7920f3154e6SShannon Nelson 0, IPPROTO_TCP, 0); 7930f3154e6SShannon Nelson } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 794fa6b8429SHeiner Kallweit tcp_v6_gso_csum_prep(skb); 7950f3154e6SShannon Nelson } 7960f3154e6SShannon Nelson 7970f3154e6SShannon Nelson return 0; 7980f3154e6SShannon Nelson } 7990f3154e6SShannon Nelson 8000f3154e6SShannon Nelson static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 8010f3154e6SShannon Nelson struct sk_buff *skb, 8020f3154e6SShannon Nelson dma_addr_t addr, u8 nsge, u16 len, 8030f3154e6SShannon Nelson unsigned int hdrlen, unsigned int mss, 8040f3154e6SShannon Nelson bool outer_csum, 8050f3154e6SShannon Nelson u16 vlan_tci, bool has_vlan, 8060f3154e6SShannon Nelson bool start, bool done) 8070f3154e6SShannon Nelson { 8080f3154e6SShannon Nelson u8 flags = 0; 8090f3154e6SShannon Nelson u64 cmd; 8100f3154e6SShannon Nelson 8110f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 8120f3154e6SShannon Nelson flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 8130f3154e6SShannon Nelson flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 8140f3154e6SShannon Nelson flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 8150f3154e6SShannon Nelson 8160f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 8170f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd); 8180f3154e6SShannon Nelson desc->len = cpu_to_le16(len); 8190f3154e6SShannon Nelson desc->vlan_tci = cpu_to_le16(vlan_tci); 8200f3154e6SShannon Nelson desc->hdr_len = cpu_to_le16(hdrlen); 8210f3154e6SShannon Nelson desc->mss = cpu_to_le16(mss); 8220f3154e6SShannon Nelson 823*2da479caSShannon Nelson if (start) { 8240f3154e6SShannon Nelson skb_tx_timestamp(skb); 8250f3154e6SShannon Nelson netdev_tx_sent_queue(q_to_ndq(q), skb->len); 826*2da479caSShannon Nelson ionic_txq_post(q, false, ionic_tx_clean, skb); 8270f3154e6SShannon Nelson } else { 828*2da479caSShannon Nelson ionic_txq_post(q, done, NULL, NULL); 8290f3154e6SShannon Nelson } 8300f3154e6SShannon Nelson } 8310f3154e6SShannon Nelson 8320f3154e6SShannon Nelson static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 8330f3154e6SShannon Nelson { 8340f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 835*2da479caSShannon Nelson struct ionic_desc_info *desc_info; 836*2da479caSShannon Nelson struct ionic_buf_info *buf_info; 8370f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem; 8380f3154e6SShannon Nelson struct ionic_txq_desc *desc; 8395b039241SShannon Nelson unsigned int chunk_len; 8405b039241SShannon Nelson unsigned int frag_rem; 8415b039241SShannon Nelson unsigned int tso_rem; 8425b039241SShannon Nelson unsigned int seg_rem; 8430f3154e6SShannon Nelson dma_addr_t desc_addr; 8445b039241SShannon Nelson dma_addr_t frag_addr; 8450f3154e6SShannon Nelson unsigned int hdrlen; 8460f3154e6SShannon Nelson unsigned int len; 8470f3154e6SShannon Nelson unsigned int mss; 8480f3154e6SShannon Nelson bool start, done; 8490f3154e6SShannon Nelson bool outer_csum; 8500f3154e6SShannon Nelson bool has_vlan; 8510f3154e6SShannon Nelson u16 desc_len; 8520f3154e6SShannon Nelson u8 desc_nsge; 8530f3154e6SShannon Nelson u16 vlan_tci; 8540f3154e6SShannon Nelson bool encap; 8550f3154e6SShannon Nelson int err; 8560f3154e6SShannon Nelson 857*2da479caSShannon Nelson desc_info = &q->info[q->head_idx]; 858*2da479caSShannon Nelson buf_info = desc_info->bufs; 859*2da479caSShannon Nelson 860*2da479caSShannon Nelson if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 8615b039241SShannon Nelson return -EIO; 8625b039241SShannon Nelson 8635b039241SShannon Nelson len = skb->len; 8640f3154e6SShannon Nelson mss = skb_shinfo(skb)->gso_size; 8650f3154e6SShannon Nelson outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) || 8660f3154e6SShannon Nelson (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); 8670f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb); 8680f3154e6SShannon Nelson vlan_tci = skb_vlan_tag_get(skb); 8690f3154e6SShannon Nelson encap = skb->encapsulation; 8700f3154e6SShannon Nelson 8710f3154e6SShannon Nelson /* Preload inner-most TCP csum field with IP pseudo hdr 8720f3154e6SShannon Nelson * calculated with IP length set to zero. HW will later 8730f3154e6SShannon Nelson * add in length to each TCP segment resulting from the TSO. 8740f3154e6SShannon Nelson */ 8750f3154e6SShannon Nelson 8760f3154e6SShannon Nelson if (encap) 8770f3154e6SShannon Nelson err = ionic_tx_tcp_inner_pseudo_csum(skb); 8780f3154e6SShannon Nelson else 8790f3154e6SShannon Nelson err = ionic_tx_tcp_pseudo_csum(skb); 8800f3154e6SShannon Nelson if (err) 8810f3154e6SShannon Nelson return err; 8820f3154e6SShannon Nelson 8830f3154e6SShannon Nelson if (encap) 8840f3154e6SShannon Nelson hdrlen = skb_inner_transport_header(skb) - skb->data + 8850f3154e6SShannon Nelson inner_tcp_hdrlen(skb); 8860f3154e6SShannon Nelson else 8870f3154e6SShannon Nelson hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 8880f3154e6SShannon Nelson 8895b039241SShannon Nelson tso_rem = len; 8905b039241SShannon Nelson seg_rem = min(tso_rem, hdrlen + mss); 8910f3154e6SShannon Nelson 8925b039241SShannon Nelson frag_addr = 0; 8935b039241SShannon Nelson frag_rem = 0; 8945b039241SShannon Nelson 8950f3154e6SShannon Nelson start = true; 8960f3154e6SShannon Nelson 8975b039241SShannon Nelson while (tso_rem > 0) { 8985b039241SShannon Nelson desc = NULL; 8995b039241SShannon Nelson elem = NULL; 9005b039241SShannon Nelson desc_addr = 0; 9015b039241SShannon Nelson desc_len = 0; 9020f3154e6SShannon Nelson desc_nsge = 0; 903*2da479caSShannon Nelson /* use fragments until we have enough to post a single descriptor */ 9045b039241SShannon Nelson while (seg_rem > 0) { 905*2da479caSShannon Nelson /* if the fragment is exhausted then move to the next one */ 9065b039241SShannon Nelson if (frag_rem == 0) { 9075b039241SShannon Nelson /* grab the next fragment */ 908*2da479caSShannon Nelson frag_addr = buf_info->dma_addr; 909*2da479caSShannon Nelson frag_rem = buf_info->len; 910*2da479caSShannon Nelson buf_info++; 9110f3154e6SShannon Nelson } 9125b039241SShannon Nelson chunk_len = min(frag_rem, seg_rem); 9135b039241SShannon Nelson if (!desc) { 9145b039241SShannon Nelson /* fill main descriptor */ 915*2da479caSShannon Nelson desc = desc_info->txq_desc; 916*2da479caSShannon Nelson elem = desc_info->txq_sg_desc->elems; 9175b039241SShannon Nelson desc_addr = frag_addr; 9185b039241SShannon Nelson desc_len = chunk_len; 9195b039241SShannon Nelson } else { 9205b039241SShannon Nelson /* fill sg descriptor */ 9215b039241SShannon Nelson elem->addr = cpu_to_le64(frag_addr); 9225b039241SShannon Nelson elem->len = cpu_to_le16(chunk_len); 9230f3154e6SShannon Nelson elem++; 9240f3154e6SShannon Nelson desc_nsge++; 9250f3154e6SShannon Nelson } 9265b039241SShannon Nelson frag_addr += chunk_len; 9275b039241SShannon Nelson frag_rem -= chunk_len; 9285b039241SShannon Nelson tso_rem -= chunk_len; 9295b039241SShannon Nelson seg_rem -= chunk_len; 9300f3154e6SShannon Nelson } 9315b039241SShannon Nelson seg_rem = min(tso_rem, mss); 9325b039241SShannon Nelson done = (tso_rem == 0); 9335b039241SShannon Nelson /* post descriptor */ 9345b039241SShannon Nelson ionic_tx_tso_post(q, desc, skb, 9355b039241SShannon Nelson desc_addr, desc_nsge, desc_len, 9365b039241SShannon Nelson hdrlen, mss, outer_csum, vlan_tci, has_vlan, 9375b039241SShannon Nelson start, done); 9385b039241SShannon Nelson start = false; 939*2da479caSShannon Nelson /* Buffer information is stored with the first tso descriptor */ 940*2da479caSShannon Nelson desc_info = &q->info[q->head_idx]; 941*2da479caSShannon Nelson desc_info->nbufs = 0; 9420f3154e6SShannon Nelson } 9430f3154e6SShannon Nelson 9445b039241SShannon Nelson stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 9455b039241SShannon Nelson stats->bytes += len; 9460f3154e6SShannon Nelson stats->tso++; 9475b039241SShannon Nelson stats->tso_bytes = len; 9480f3154e6SShannon Nelson 9490f3154e6SShannon Nelson return 0; 9500f3154e6SShannon Nelson } 9510f3154e6SShannon Nelson 952*2da479caSShannon Nelson static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 953*2da479caSShannon Nelson struct ionic_desc_info *desc_info) 9540f3154e6SShannon Nelson { 955*2da479caSShannon Nelson struct ionic_txq_desc *desc = desc_info->txq_desc; 956*2da479caSShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs; 9570f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 9580f3154e6SShannon Nelson bool has_vlan; 9590f3154e6SShannon Nelson u8 flags = 0; 9600f3154e6SShannon Nelson bool encap; 9610f3154e6SShannon Nelson u64 cmd; 9620f3154e6SShannon Nelson 9630f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb); 9640f3154e6SShannon Nelson encap = skb->encapsulation; 9650f3154e6SShannon Nelson 9660f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 9670f3154e6SShannon Nelson flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 9680f3154e6SShannon Nelson 9690f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 970*2da479caSShannon Nelson flags, skb_shinfo(skb)->nr_frags, 971*2da479caSShannon Nelson buf_info->dma_addr); 9720f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd); 973*2da479caSShannon Nelson desc->len = cpu_to_le16(buf_info->len); 974f64e0c56SShannon Nelson if (has_vlan) { 975f64e0c56SShannon Nelson desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 976f64e0c56SShannon Nelson stats->vlan_inserted++; 977*2da479caSShannon Nelson } else { 978*2da479caSShannon Nelson desc->vlan_tci = 0; 979f64e0c56SShannon Nelson } 980*2da479caSShannon Nelson desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 981*2da479caSShannon Nelson desc->csum_offset = cpu_to_le16(skb->csum_offset); 9820f3154e6SShannon Nelson 983fa821170SXin Long if (skb_csum_is_sctp(skb)) 9840f3154e6SShannon Nelson stats->crc32_csum++; 9850f3154e6SShannon Nelson else 9860f3154e6SShannon Nelson stats->csum++; 9870f3154e6SShannon Nelson 9880f3154e6SShannon Nelson return 0; 9890f3154e6SShannon Nelson } 9900f3154e6SShannon Nelson 991*2da479caSShannon Nelson static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 992*2da479caSShannon Nelson struct ionic_desc_info *desc_info) 9930f3154e6SShannon Nelson { 994*2da479caSShannon Nelson struct ionic_txq_desc *desc = desc_info->txq_desc; 995*2da479caSShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs; 9960f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 9970f3154e6SShannon Nelson bool has_vlan; 9980f3154e6SShannon Nelson u8 flags = 0; 9990f3154e6SShannon Nelson bool encap; 10000f3154e6SShannon Nelson u64 cmd; 10010f3154e6SShannon Nelson 10020f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb); 10030f3154e6SShannon Nelson encap = skb->encapsulation; 10040f3154e6SShannon Nelson 10050f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 10060f3154e6SShannon Nelson flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 10070f3154e6SShannon Nelson 10080f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1009*2da479caSShannon Nelson flags, skb_shinfo(skb)->nr_frags, 1010*2da479caSShannon Nelson buf_info->dma_addr); 10110f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd); 1012*2da479caSShannon Nelson desc->len = cpu_to_le16(buf_info->len); 1013f64e0c56SShannon Nelson if (has_vlan) { 10140f3154e6SShannon Nelson desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1015f64e0c56SShannon Nelson stats->vlan_inserted++; 1016*2da479caSShannon Nelson } else { 1017*2da479caSShannon Nelson desc->vlan_tci = 0; 1018f64e0c56SShannon Nelson } 1019*2da479caSShannon Nelson desc->csum_start = 0; 1020*2da479caSShannon Nelson desc->csum_offset = 0; 10210f3154e6SShannon Nelson 1022f64e0c56SShannon Nelson stats->csum_none++; 10230f3154e6SShannon Nelson 10240f3154e6SShannon Nelson return 0; 10250f3154e6SShannon Nelson } 10260f3154e6SShannon Nelson 1027*2da479caSShannon Nelson static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1028*2da479caSShannon Nelson struct ionic_desc_info *desc_info) 10290f3154e6SShannon Nelson { 1030*2da479caSShannon Nelson struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; 1031*2da479caSShannon Nelson struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 10320f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem = sg_desc->elems; 10330f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 1034*2da479caSShannon Nelson unsigned int i; 10350f3154e6SShannon Nelson 1036*2da479caSShannon Nelson for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1037*2da479caSShannon Nelson elem->addr = cpu_to_le64(buf_info->dma_addr); 1038*2da479caSShannon Nelson elem->len = cpu_to_le16(buf_info->len); 10390f3154e6SShannon Nelson } 10400f3154e6SShannon Nelson 1041*2da479caSShannon Nelson stats->frags += skb_shinfo(skb)->nr_frags; 1042*2da479caSShannon Nelson 10430f3154e6SShannon Nelson return 0; 10440f3154e6SShannon Nelson } 10450f3154e6SShannon Nelson 10460f3154e6SShannon Nelson static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 10470f3154e6SShannon Nelson { 1048*2da479caSShannon Nelson struct ionic_desc_info *desc_info = &q->info[q->head_idx]; 10490f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 10500f3154e6SShannon Nelson int err; 10510f3154e6SShannon Nelson 1052*2da479caSShannon Nelson if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1053*2da479caSShannon Nelson return -EIO; 1054*2da479caSShannon Nelson 10550f3154e6SShannon Nelson /* set up the initial descriptor */ 10560f3154e6SShannon Nelson if (skb->ip_summed == CHECKSUM_PARTIAL) 1057*2da479caSShannon Nelson err = ionic_tx_calc_csum(q, skb, desc_info); 10580f3154e6SShannon Nelson else 1059*2da479caSShannon Nelson err = ionic_tx_calc_no_csum(q, skb, desc_info); 10600f3154e6SShannon Nelson if (err) 10610f3154e6SShannon Nelson return err; 10620f3154e6SShannon Nelson 10630f3154e6SShannon Nelson /* add frags */ 1064*2da479caSShannon Nelson err = ionic_tx_skb_frags(q, skb, desc_info); 10650f3154e6SShannon Nelson if (err) 10660f3154e6SShannon Nelson return err; 10670f3154e6SShannon Nelson 10680f3154e6SShannon Nelson skb_tx_timestamp(skb); 10690f3154e6SShannon Nelson stats->pkts++; 10700f3154e6SShannon Nelson stats->bytes += skb->len; 10710f3154e6SShannon Nelson 10720f3154e6SShannon Nelson netdev_tx_sent_queue(q_to_ndq(q), skb->len); 10730f3154e6SShannon Nelson ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 10740f3154e6SShannon Nelson 10750f3154e6SShannon Nelson return 0; 10760f3154e6SShannon Nelson } 10770f3154e6SShannon Nelson 10780f3154e6SShannon Nelson static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 10790f3154e6SShannon Nelson { 10800f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q); 10810f3154e6SShannon Nelson int err; 10820f3154e6SShannon Nelson 10830f3154e6SShannon Nelson /* If TSO, need roundup(skb->len/mss) descs */ 10840f3154e6SShannon Nelson if (skb_is_gso(skb)) 10850f3154e6SShannon Nelson return (skb->len / skb_shinfo(skb)->gso_size) + 1; 10860f3154e6SShannon Nelson 10870f3154e6SShannon Nelson /* If non-TSO, just need 1 desc and nr_frags sg elems */ 1088f37bc346SShannon Nelson if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems) 10890f3154e6SShannon Nelson return 1; 10900f3154e6SShannon Nelson 10910f3154e6SShannon Nelson /* Too many frags, so linearize */ 10920f3154e6SShannon Nelson err = skb_linearize(skb); 10930f3154e6SShannon Nelson if (err) 10940f3154e6SShannon Nelson return err; 10950f3154e6SShannon Nelson 10960f3154e6SShannon Nelson stats->linearize++; 10970f3154e6SShannon Nelson 10980f3154e6SShannon Nelson /* Need 1 desc and zero sg elems */ 10990f3154e6SShannon Nelson return 1; 11000f3154e6SShannon Nelson } 11010f3154e6SShannon Nelson 11020f3154e6SShannon Nelson static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 11030f3154e6SShannon Nelson { 11040f3154e6SShannon Nelson int stopped = 0; 11050f3154e6SShannon Nelson 11060f3154e6SShannon Nelson if (unlikely(!ionic_q_has_space(q, ndescs))) { 11070f3154e6SShannon Nelson netif_stop_subqueue(q->lif->netdev, q->index); 11080f3154e6SShannon Nelson q->stop++; 11090f3154e6SShannon Nelson stopped = 1; 11100f3154e6SShannon Nelson 11110f3154e6SShannon Nelson /* Might race with ionic_tx_clean, check again */ 11120f3154e6SShannon Nelson smp_rmb(); 11130f3154e6SShannon Nelson if (ionic_q_has_space(q, ndescs)) { 11140f3154e6SShannon Nelson netif_wake_subqueue(q->lif->netdev, q->index); 11150f3154e6SShannon Nelson stopped = 0; 11160f3154e6SShannon Nelson } 11170f3154e6SShannon Nelson } 11180f3154e6SShannon Nelson 11190f3154e6SShannon Nelson return stopped; 11200f3154e6SShannon Nelson } 11210f3154e6SShannon Nelson 11220f3154e6SShannon Nelson netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 11230f3154e6SShannon Nelson { 11240f3154e6SShannon Nelson u16 queue_index = skb_get_queue_mapping(skb); 11250f3154e6SShannon Nelson struct ionic_lif *lif = netdev_priv(netdev); 11260f3154e6SShannon Nelson struct ionic_queue *q; 11270f3154e6SShannon Nelson int ndescs; 11280f3154e6SShannon Nelson int err; 11290f3154e6SShannon Nelson 1130c6d3d73aSShannon Nelson if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 11310f3154e6SShannon Nelson dev_kfree_skb(skb); 11320f3154e6SShannon Nelson return NETDEV_TX_OK; 11330f3154e6SShannon Nelson } 11340f3154e6SShannon Nelson 113534dec947SShannon Nelson if (unlikely(queue_index >= lif->nxqs)) 11360f3154e6SShannon Nelson queue_index = 0; 113734dec947SShannon Nelson q = &lif->txqcqs[queue_index]->q; 11380f3154e6SShannon Nelson 11390f3154e6SShannon Nelson ndescs = ionic_tx_descs_needed(q, skb); 11400f3154e6SShannon Nelson if (ndescs < 0) 11410f3154e6SShannon Nelson goto err_out_drop; 11420f3154e6SShannon Nelson 11430f3154e6SShannon Nelson if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 11440f3154e6SShannon Nelson return NETDEV_TX_BUSY; 11450f3154e6SShannon Nelson 11460f3154e6SShannon Nelson if (skb_is_gso(skb)) 11470f3154e6SShannon Nelson err = ionic_tx_tso(q, skb); 11480f3154e6SShannon Nelson else 11490f3154e6SShannon Nelson err = ionic_tx(q, skb); 11500f3154e6SShannon Nelson 11510f3154e6SShannon Nelson if (err) 11520f3154e6SShannon Nelson goto err_out_drop; 11530f3154e6SShannon Nelson 11540f3154e6SShannon Nelson /* Stop the queue if there aren't descriptors for the next packet. 11550f3154e6SShannon Nelson * Since our SG lists per descriptor take care of most of the possible 11560f3154e6SShannon Nelson * fragmentation, we don't need to have many descriptors available. 11570f3154e6SShannon Nelson */ 11580f3154e6SShannon Nelson ionic_maybe_stop_tx(q, 4); 11590f3154e6SShannon Nelson 11600f3154e6SShannon Nelson return NETDEV_TX_OK; 11610f3154e6SShannon Nelson 11620f3154e6SShannon Nelson err_out_drop: 11630f3154e6SShannon Nelson q->stop++; 11640f3154e6SShannon Nelson q->drop++; 11650f3154e6SShannon Nelson dev_kfree_skb(skb); 11660f3154e6SShannon Nelson return NETDEV_TX_OK; 11670f3154e6SShannon Nelson } 1168