10f3154e6SShannon Nelson // SPDX-License-Identifier: GPL-2.0
20f3154e6SShannon Nelson /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
30f3154e6SShannon Nelson
40f3154e6SShannon Nelson #include <linux/ip.h>
50f3154e6SShannon Nelson #include <linux/ipv6.h>
60f3154e6SShannon Nelson #include <linux/if_vlan.h>
70f3154e6SShannon Nelson #include <net/ip6_checksum.h>
80f3154e6SShannon Nelson
90f3154e6SShannon Nelson #include "ionic.h"
100f3154e6SShannon Nelson #include "ionic_lif.h"
110f3154e6SShannon Nelson #include "ionic_txrx.h"
120f3154e6SShannon Nelson
ionic_txq_post(struct ionic_queue * q,bool ring_dbell,ionic_desc_cb cb_func,void * cb_arg)130f3154e6SShannon Nelson static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
140f3154e6SShannon Nelson ionic_desc_cb cb_func, void *cb_arg)
150f3154e6SShannon Nelson {
160f3154e6SShannon Nelson ionic_q_post(q, ring_dbell, cb_func, cb_arg);
170f3154e6SShannon Nelson }
180f3154e6SShannon Nelson
ionic_rxq_post(struct ionic_queue * q,bool ring_dbell,ionic_desc_cb cb_func,void * cb_arg)190f3154e6SShannon Nelson static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
200f3154e6SShannon Nelson ionic_desc_cb cb_func, void *cb_arg)
210f3154e6SShannon Nelson {
220f3154e6SShannon Nelson ionic_q_post(q, ring_dbell, cb_func, cb_arg);
230f3154e6SShannon Nelson }
240f3154e6SShannon Nelson
ionic_txq_poke_doorbell(struct ionic_queue * q)25b69585bfSAllen Hubbe bool ionic_txq_poke_doorbell(struct ionic_queue *q)
26b69585bfSAllen Hubbe {
27b69585bfSAllen Hubbe unsigned long now, then, dif;
28b69585bfSAllen Hubbe struct netdev_queue *netdev_txq;
29b69585bfSAllen Hubbe struct net_device *netdev;
30b69585bfSAllen Hubbe
31b69585bfSAllen Hubbe netdev = q->lif->netdev;
32b69585bfSAllen Hubbe netdev_txq = netdev_get_tx_queue(netdev, q->index);
33b69585bfSAllen Hubbe
34b69585bfSAllen Hubbe HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id());
35b69585bfSAllen Hubbe
36b69585bfSAllen Hubbe if (q->tail_idx == q->head_idx) {
37b69585bfSAllen Hubbe HARD_TX_UNLOCK(netdev, netdev_txq);
38b69585bfSAllen Hubbe return false;
39b69585bfSAllen Hubbe }
40b69585bfSAllen Hubbe
41b69585bfSAllen Hubbe now = READ_ONCE(jiffies);
42b69585bfSAllen Hubbe then = q->dbell_jiffies;
43b69585bfSAllen Hubbe dif = now - then;
44b69585bfSAllen Hubbe
45b69585bfSAllen Hubbe if (dif > q->dbell_deadline) {
46b69585bfSAllen Hubbe ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
47b69585bfSAllen Hubbe q->dbval | q->head_idx);
48b69585bfSAllen Hubbe
49b69585bfSAllen Hubbe q->dbell_jiffies = now;
50b69585bfSAllen Hubbe }
51b69585bfSAllen Hubbe
52b69585bfSAllen Hubbe HARD_TX_UNLOCK(netdev, netdev_txq);
53b69585bfSAllen Hubbe
54b69585bfSAllen Hubbe return true;
55b69585bfSAllen Hubbe }
56b69585bfSAllen Hubbe
ionic_rxq_poke_doorbell(struct ionic_queue * q)57b69585bfSAllen Hubbe bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
58b69585bfSAllen Hubbe {
59b69585bfSAllen Hubbe unsigned long now, then, dif;
60b69585bfSAllen Hubbe
61b69585bfSAllen Hubbe /* no lock, called from rx napi or txrx napi, nothing else can fill */
62b69585bfSAllen Hubbe
63b69585bfSAllen Hubbe if (q->tail_idx == q->head_idx)
64b69585bfSAllen Hubbe return false;
65b69585bfSAllen Hubbe
66b69585bfSAllen Hubbe now = READ_ONCE(jiffies);
67b69585bfSAllen Hubbe then = q->dbell_jiffies;
68b69585bfSAllen Hubbe dif = now - then;
69b69585bfSAllen Hubbe
70b69585bfSAllen Hubbe if (dif > q->dbell_deadline) {
71b69585bfSAllen Hubbe ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
72b69585bfSAllen Hubbe q->dbval | q->head_idx);
73b69585bfSAllen Hubbe
74b69585bfSAllen Hubbe q->dbell_jiffies = now;
75b69585bfSAllen Hubbe
76b69585bfSAllen Hubbe dif = 2 * q->dbell_deadline;
77b69585bfSAllen Hubbe if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)
78b69585bfSAllen Hubbe dif = IONIC_RX_MAX_DOORBELL_DEADLINE;
79b69585bfSAllen Hubbe
80b69585bfSAllen Hubbe q->dbell_deadline = dif;
81b69585bfSAllen Hubbe }
82b69585bfSAllen Hubbe
83b69585bfSAllen Hubbe return true;
84b69585bfSAllen Hubbe }
85b69585bfSAllen Hubbe
q_to_ndq(struct ionic_queue * q)860f3154e6SShannon Nelson static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
870f3154e6SShannon Nelson {
880f3154e6SShannon Nelson return netdev_get_tx_queue(q->lif->netdev, q->index);
890f3154e6SShannon Nelson }
900f3154e6SShannon Nelson
ionic_rx_page_alloc(struct ionic_queue * q,struct ionic_buf_info * buf_info)912b5720f2SShannon Nelson static int ionic_rx_page_alloc(struct ionic_queue *q,
924b0a7539SShannon Nelson struct ionic_buf_info *buf_info)
932b5720f2SShannon Nelson {
9489e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev;
952b5720f2SShannon Nelson struct ionic_rx_stats *stats;
962b5720f2SShannon Nelson struct device *dev;
97e75ccac1SShannon Nelson struct page *page;
982b5720f2SShannon Nelson
99f37bc346SShannon Nelson dev = q->dev;
1002b5720f2SShannon Nelson stats = q_to_rx_stats(q);
1012b5720f2SShannon Nelson
1024b0a7539SShannon Nelson if (unlikely(!buf_info)) {
1034b0a7539SShannon Nelson net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
1042b5720f2SShannon Nelson netdev->name, q->name);
1052b5720f2SShannon Nelson return -EINVAL;
1062b5720f2SShannon Nelson }
1072b5720f2SShannon Nelson
108e75ccac1SShannon Nelson page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
109e75ccac1SShannon Nelson if (unlikely(!page)) {
1102b5720f2SShannon Nelson net_err_ratelimited("%s: %s page alloc failed\n",
1112b5720f2SShannon Nelson netdev->name, q->name);
1122b5720f2SShannon Nelson stats->alloc_err++;
1132b5720f2SShannon Nelson return -ENOMEM;
1142b5720f2SShannon Nelson }
1152b5720f2SShannon Nelson
116e75ccac1SShannon Nelson buf_info->dma_addr = dma_map_page(dev, page, 0,
1174b0a7539SShannon Nelson IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
1184b0a7539SShannon Nelson if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
119e75ccac1SShannon Nelson __free_pages(page, 0);
1202b5720f2SShannon Nelson net_err_ratelimited("%s: %s dma map failed\n",
1212b5720f2SShannon Nelson netdev->name, q->name);
1222b5720f2SShannon Nelson stats->dma_map_err++;
1232b5720f2SShannon Nelson return -EIO;
1242b5720f2SShannon Nelson }
1252b5720f2SShannon Nelson
126e75ccac1SShannon Nelson buf_info->page = page;
127e75ccac1SShannon Nelson buf_info->page_offset = 0;
128e75ccac1SShannon Nelson
1292b5720f2SShannon Nelson return 0;
1302b5720f2SShannon Nelson }
1312b5720f2SShannon Nelson
ionic_rx_page_free(struct ionic_queue * q,struct ionic_buf_info * buf_info)1322b5720f2SShannon Nelson static void ionic_rx_page_free(struct ionic_queue *q,
1334b0a7539SShannon Nelson struct ionic_buf_info *buf_info)
1342b5720f2SShannon Nelson {
1354b0a7539SShannon Nelson struct net_device *netdev = q->lif->netdev;
136f37bc346SShannon Nelson struct device *dev = q->dev;
1372b5720f2SShannon Nelson
1384b0a7539SShannon Nelson if (unlikely(!buf_info)) {
1394b0a7539SShannon Nelson net_err_ratelimited("%s: %s invalid buf_info in free\n",
1402b5720f2SShannon Nelson netdev->name, q->name);
1412b5720f2SShannon Nelson return;
1422b5720f2SShannon Nelson }
1432b5720f2SShannon Nelson
1444b0a7539SShannon Nelson if (!buf_info->page)
1452b5720f2SShannon Nelson return;
1464b0a7539SShannon Nelson
1474b0a7539SShannon Nelson dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
1484b0a7539SShannon Nelson __free_pages(buf_info->page, 0);
149e75ccac1SShannon Nelson buf_info->page = NULL;
1502b5720f2SShannon Nelson }
1512b5720f2SShannon Nelson
ionic_rx_buf_recycle(struct ionic_queue * q,struct ionic_buf_info * buf_info,u32 used)1524b0a7539SShannon Nelson static bool ionic_rx_buf_recycle(struct ionic_queue *q,
1534b0a7539SShannon Nelson struct ionic_buf_info *buf_info, u32 used)
1544b0a7539SShannon Nelson {
1554b0a7539SShannon Nelson u32 size;
1562b5720f2SShannon Nelson
1574b0a7539SShannon Nelson /* don't re-use pages allocated in low-mem condition */
1584b0a7539SShannon Nelson if (page_is_pfmemalloc(buf_info->page))
1594b0a7539SShannon Nelson return false;
1604b0a7539SShannon Nelson
1614b0a7539SShannon Nelson /* don't re-use buffers from non-local numa nodes */
1624b0a7539SShannon Nelson if (page_to_nid(buf_info->page) != numa_mem_id())
1634b0a7539SShannon Nelson return false;
1644b0a7539SShannon Nelson
1654b0a7539SShannon Nelson size = ALIGN(used, IONIC_PAGE_SPLIT_SZ);
1664b0a7539SShannon Nelson buf_info->page_offset += size;
1674b0a7539SShannon Nelson if (buf_info->page_offset >= IONIC_PAGE_SIZE)
1684b0a7539SShannon Nelson return false;
1694b0a7539SShannon Nelson
1704b0a7539SShannon Nelson get_page(buf_info->page);
1714b0a7539SShannon Nelson
1724b0a7539SShannon Nelson return true;
1732b5720f2SShannon Nelson }
1742b5720f2SShannon Nelson
ionic_rx_frags(struct ionic_queue * q,struct ionic_desc_info * desc_info,struct ionic_rxq_comp * comp)17508f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
17608f2e4b2SShannon Nelson struct ionic_desc_info *desc_info,
177a25edab9SShannon Nelson struct ionic_rxq_comp *comp)
1780f3154e6SShannon Nelson {
17989e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev;
1804b0a7539SShannon Nelson struct ionic_buf_info *buf_info;
18189e572e7SShannon Nelson struct ionic_rx_stats *stats;
182f37bc346SShannon Nelson struct device *dev = q->dev;
18308f2e4b2SShannon Nelson struct sk_buff *skb;
18408f2e4b2SShannon Nelson unsigned int i;
18508f2e4b2SShannon Nelson u16 frag_len;
18608f2e4b2SShannon Nelson u16 len;
1870f3154e6SShannon Nelson
18889e572e7SShannon Nelson stats = q_to_rx_stats(q);
18989e572e7SShannon Nelson
1904b0a7539SShannon Nelson buf_info = &desc_info->bufs[0];
19108f2e4b2SShannon Nelson len = le16_to_cpu(comp->len);
19208f2e4b2SShannon Nelson
193e75ccac1SShannon Nelson prefetchw(buf_info->page);
19408f2e4b2SShannon Nelson
19589e572e7SShannon Nelson skb = napi_get_frags(&q_to_qcq(q)->napi);
19689e572e7SShannon Nelson if (unlikely(!skb)) {
19789e572e7SShannon Nelson net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
19889e572e7SShannon Nelson netdev->name, q->name);
19989e572e7SShannon Nelson stats->alloc_err++;
20008f2e4b2SShannon Nelson return NULL;
20189e572e7SShannon Nelson }
20208f2e4b2SShannon Nelson
20308f2e4b2SShannon Nelson i = comp->num_sg_elems + 1;
20408f2e4b2SShannon Nelson do {
2054b0a7539SShannon Nelson if (unlikely(!buf_info->page)) {
20608f2e4b2SShannon Nelson dev_kfree_skb(skb);
20708f2e4b2SShannon Nelson return NULL;
2080f3154e6SShannon Nelson }
2090f3154e6SShannon Nelson
210*8f6b846bSDavid Christensen frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
211*8f6b846bSDavid Christensen IONIC_PAGE_SIZE - buf_info->page_offset));
21208f2e4b2SShannon Nelson len -= frag_len;
21308f2e4b2SShannon Nelson
2144b0a7539SShannon Nelson dma_sync_single_for_cpu(dev,
2154b0a7539SShannon Nelson buf_info->dma_addr + buf_info->page_offset,
2164b0a7539SShannon Nelson frag_len, DMA_FROM_DEVICE);
2174b0a7539SShannon Nelson
21808f2e4b2SShannon Nelson skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2194b0a7539SShannon Nelson buf_info->page, buf_info->page_offset, frag_len,
2204b0a7539SShannon Nelson IONIC_PAGE_SIZE);
2214b0a7539SShannon Nelson
2224b0a7539SShannon Nelson if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
2234b0a7539SShannon Nelson dma_unmap_page(dev, buf_info->dma_addr,
2244b0a7539SShannon Nelson IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
225e75ccac1SShannon Nelson buf_info->page = NULL;
2264b0a7539SShannon Nelson }
2274b0a7539SShannon Nelson
2284b0a7539SShannon Nelson buf_info++;
2294b0a7539SShannon Nelson
23008f2e4b2SShannon Nelson i--;
23108f2e4b2SShannon Nelson } while (i > 0);
23208f2e4b2SShannon Nelson
23308f2e4b2SShannon Nelson return skb;
2340f3154e6SShannon Nelson }
2350f3154e6SShannon Nelson
ionic_rx_copybreak(struct ionic_queue * q,struct ionic_desc_info * desc_info,struct ionic_rxq_comp * comp)23608f2e4b2SShannon Nelson static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
23708f2e4b2SShannon Nelson struct ionic_desc_info *desc_info,
238a25edab9SShannon Nelson struct ionic_rxq_comp *comp)
23908f2e4b2SShannon Nelson {
24089e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev;
2414b0a7539SShannon Nelson struct ionic_buf_info *buf_info;
24289e572e7SShannon Nelson struct ionic_rx_stats *stats;
243f37bc346SShannon Nelson struct device *dev = q->dev;
24408f2e4b2SShannon Nelson struct sk_buff *skb;
24508f2e4b2SShannon Nelson u16 len;
2460f3154e6SShannon Nelson
24789e572e7SShannon Nelson stats = q_to_rx_stats(q);
24889e572e7SShannon Nelson
2494b0a7539SShannon Nelson buf_info = &desc_info->bufs[0];
25008f2e4b2SShannon Nelson len = le16_to_cpu(comp->len);
2510f3154e6SShannon Nelson
25289e572e7SShannon Nelson skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
25389e572e7SShannon Nelson if (unlikely(!skb)) {
25489e572e7SShannon Nelson net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
25589e572e7SShannon Nelson netdev->name, q->name);
25689e572e7SShannon Nelson stats->alloc_err++;
25708f2e4b2SShannon Nelson return NULL;
25889e572e7SShannon Nelson }
2590f3154e6SShannon Nelson
2604b0a7539SShannon Nelson if (unlikely(!buf_info->page)) {
26108f2e4b2SShannon Nelson dev_kfree_skb(skb);
26208f2e4b2SShannon Nelson return NULL;
26308f2e4b2SShannon Nelson }
26408f2e4b2SShannon Nelson
2654b0a7539SShannon Nelson dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset,
26608f2e4b2SShannon Nelson len, DMA_FROM_DEVICE);
2674b0a7539SShannon Nelson skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len);
2684b0a7539SShannon Nelson dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset,
26908f2e4b2SShannon Nelson len, DMA_FROM_DEVICE);
27008f2e4b2SShannon Nelson
27108f2e4b2SShannon Nelson skb_put(skb, len);
27208f2e4b2SShannon Nelson skb->protocol = eth_type_trans(skb, q->lif->netdev);
27308f2e4b2SShannon Nelson
27408f2e4b2SShannon Nelson return skb;
2750f3154e6SShannon Nelson }
2760f3154e6SShannon Nelson
ionic_rx_clean(struct ionic_queue * q,struct ionic_desc_info * desc_info,struct ionic_cq_info * cq_info,void * cb_arg)2775b3f3f2aSShannon Nelson static void ionic_rx_clean(struct ionic_queue *q,
2785b3f3f2aSShannon Nelson struct ionic_desc_info *desc_info,
2795b3f3f2aSShannon Nelson struct ionic_cq_info *cq_info,
2805b3f3f2aSShannon Nelson void *cb_arg)
2810f3154e6SShannon Nelson {
28289e572e7SShannon Nelson struct net_device *netdev = q->lif->netdev;
2830f3154e6SShannon Nelson struct ionic_qcq *qcq = q_to_qcq(q);
2840f3154e6SShannon Nelson struct ionic_rx_stats *stats;
2850ec9f666SShannon Nelson struct ionic_rxq_comp *comp;
28608f2e4b2SShannon Nelson struct sk_buff *skb;
2870f3154e6SShannon Nelson
2880ec9f666SShannon Nelson comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
2890ec9f666SShannon Nelson
2900f3154e6SShannon Nelson stats = q_to_rx_stats(q);
2910f3154e6SShannon Nelson
29224cfa8c7SShannon Nelson if (comp->status) {
29324cfa8c7SShannon Nelson stats->dropped++;
2940f3154e6SShannon Nelson return;
29524cfa8c7SShannon Nelson }
2960f3154e6SShannon Nelson
2970f3154e6SShannon Nelson stats->pkts++;
2980f3154e6SShannon Nelson stats->bytes += le16_to_cpu(comp->len);
2990f3154e6SShannon Nelson
30008f2e4b2SShannon Nelson if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
301a25edab9SShannon Nelson skb = ionic_rx_copybreak(q, desc_info, comp);
30208f2e4b2SShannon Nelson else
303a25edab9SShannon Nelson skb = ionic_rx_frags(q, desc_info, comp);
3040f3154e6SShannon Nelson
30524cfa8c7SShannon Nelson if (unlikely(!skb)) {
30624cfa8c7SShannon Nelson stats->dropped++;
30708f2e4b2SShannon Nelson return;
30824cfa8c7SShannon Nelson }
3090f3154e6SShannon Nelson
3100f3154e6SShannon Nelson skb_record_rx_queue(skb, q->index);
3110f3154e6SShannon Nelson
31208f2e4b2SShannon Nelson if (likely(netdev->features & NETIF_F_RXHASH)) {
3130f3154e6SShannon Nelson switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
3140f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4:
3150f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6:
3160f3154e6SShannon Nelson skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
3170f3154e6SShannon Nelson PKT_HASH_TYPE_L3);
3180f3154e6SShannon Nelson break;
3190f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4_TCP:
3200f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6_TCP:
3210f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV4_UDP:
3220f3154e6SShannon Nelson case IONIC_PKT_TYPE_IPV6_UDP:
3230f3154e6SShannon Nelson skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
3240f3154e6SShannon Nelson PKT_HASH_TYPE_L4);
3250f3154e6SShannon Nelson break;
3260f3154e6SShannon Nelson }
3270f3154e6SShannon Nelson }
3280f3154e6SShannon Nelson
329f07f9815SShannon Nelson if (likely(netdev->features & NETIF_F_RXCSUM) &&
330f07f9815SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
3310f3154e6SShannon Nelson skb->ip_summed = CHECKSUM_COMPLETE;
332d701ec32SShannon Nelson skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
3330f3154e6SShannon Nelson stats->csum_complete++;
3340f3154e6SShannon Nelson } else {
3350f3154e6SShannon Nelson stats->csum_none++;
3360f3154e6SShannon Nelson }
3370f3154e6SShannon Nelson
33808f2e4b2SShannon Nelson if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
3390f3154e6SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
34008f2e4b2SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
3410f3154e6SShannon Nelson stats->csum_error++;
3420f3154e6SShannon Nelson
343f64e0c56SShannon Nelson if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
344f64e0c56SShannon Nelson (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
3450f3154e6SShannon Nelson __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
3460f3154e6SShannon Nelson le16_to_cpu(comp->vlan_tci));
347f64e0c56SShannon Nelson stats->vlan_stripped++;
3480f3154e6SShannon Nelson }
3490f3154e6SShannon Nelson
350a8771bfeSShannon Nelson if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
351a8771bfeSShannon Nelson __le64 *cq_desc_hwstamp;
352a8771bfeSShannon Nelson u64 hwstamp;
353a8771bfeSShannon Nelson
354a8771bfeSShannon Nelson cq_desc_hwstamp =
355a8771bfeSShannon Nelson cq_info->cq_desc +
356a8771bfeSShannon Nelson qcq->cq.desc_size -
357a8771bfeSShannon Nelson sizeof(struct ionic_rxq_comp) -
358a8771bfeSShannon Nelson IONIC_HWSTAMP_CQ_NEGOFFSET;
359a8771bfeSShannon Nelson
360a8771bfeSShannon Nelson hwstamp = le64_to_cpu(*cq_desc_hwstamp);
361a8771bfeSShannon Nelson
362a8771bfeSShannon Nelson if (hwstamp != IONIC_HWSTAMP_INVALID) {
363a8771bfeSShannon Nelson skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
364a8771bfeSShannon Nelson stats->hwstamp_valid++;
365a8771bfeSShannon Nelson } else {
366a8771bfeSShannon Nelson stats->hwstamp_invalid++;
367a8771bfeSShannon Nelson }
368a8771bfeSShannon Nelson }
369a8771bfeSShannon Nelson
37008f2e4b2SShannon Nelson if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
3710f3154e6SShannon Nelson napi_gro_receive(&qcq->napi, skb);
37208f2e4b2SShannon Nelson else
37308f2e4b2SShannon Nelson napi_gro_frags(&qcq->napi);
3740f3154e6SShannon Nelson }
3750f3154e6SShannon Nelson
ionic_rx_service(struct ionic_cq * cq,struct ionic_cq_info * cq_info)376a8771bfeSShannon Nelson bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
3770f3154e6SShannon Nelson {
3780f3154e6SShannon Nelson struct ionic_queue *q = cq->bound_q;
3790f3154e6SShannon Nelson struct ionic_desc_info *desc_info;
3800ec9f666SShannon Nelson struct ionic_rxq_comp *comp;
3810ec9f666SShannon Nelson
3820ec9f666SShannon Nelson comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
3830f3154e6SShannon Nelson
3840f3154e6SShannon Nelson if (!color_match(comp->pkt_type_color, cq->done_color))
3850f3154e6SShannon Nelson return false;
3860f3154e6SShannon Nelson
3870f3154e6SShannon Nelson /* check for empty queue */
388f1d2e894SShannon Nelson if (q->tail_idx == q->head_idx)
3890f3154e6SShannon Nelson return false;
3900f3154e6SShannon Nelson
391339dcf7fSShannon Nelson if (q->tail_idx != le16_to_cpu(comp->comp_index))
3920f3154e6SShannon Nelson return false;
3930f3154e6SShannon Nelson
394339dcf7fSShannon Nelson desc_info = &q->info[q->tail_idx];
395f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
3960f3154e6SShannon Nelson
3970f3154e6SShannon Nelson /* clean the related q entry, only one per qc completion */
3980f3154e6SShannon Nelson ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
3990f3154e6SShannon Nelson
4000f3154e6SShannon Nelson desc_info->cb = NULL;
4010f3154e6SShannon Nelson desc_info->cb_arg = NULL;
4020f3154e6SShannon Nelson
4030f3154e6SShannon Nelson return true;
4040f3154e6SShannon Nelson }
4050f3154e6SShannon Nelson
ionic_write_cmb_desc(struct ionic_queue * q,void __iomem * cmb_desc,void * desc)40640bc471dSShannon Nelson static inline void ionic_write_cmb_desc(struct ionic_queue *q,
40740bc471dSShannon Nelson void __iomem *cmb_desc,
40840bc471dSShannon Nelson void *desc)
40940bc471dSShannon Nelson {
41040bc471dSShannon Nelson if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)
41140bc471dSShannon Nelson memcpy_toio(cmb_desc, desc, q->desc_size);
41240bc471dSShannon Nelson }
41340bc471dSShannon Nelson
ionic_rx_fill(struct ionic_queue * q)4140f3154e6SShannon Nelson void ionic_rx_fill(struct ionic_queue *q)
4150f3154e6SShannon Nelson {
4160f3154e6SShannon Nelson struct net_device *netdev = q->lif->netdev;
41708f2e4b2SShannon Nelson struct ionic_desc_info *desc_info;
41808f2e4b2SShannon Nelson struct ionic_rxq_sg_desc *sg_desc;
41908f2e4b2SShannon Nelson struct ionic_rxq_sg_elem *sg_elem;
4204b0a7539SShannon Nelson struct ionic_buf_info *buf_info;
421e55f0f5bSNeel Patel unsigned int fill_threshold;
4220f3154e6SShannon Nelson struct ionic_rxq_desc *desc;
423c37d6e3fSShannon Nelson unsigned int remain_len;
4244b0a7539SShannon Nelson unsigned int frag_len;
42508f2e4b2SShannon Nelson unsigned int nfrags;
426e55f0f5bSNeel Patel unsigned int n_fill;
42708f2e4b2SShannon Nelson unsigned int i, j;
4280f3154e6SShannon Nelson unsigned int len;
4290f3154e6SShannon Nelson
430e55f0f5bSNeel Patel n_fill = ionic_q_space_avail(q);
431e55f0f5bSNeel Patel
432e55f0f5bSNeel Patel fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD,
433e55f0f5bSNeel Patel q->num_descs / IONIC_RX_FILL_DIV);
434e55f0f5bSNeel Patel if (n_fill < fill_threshold)
435e55f0f5bSNeel Patel return;
436e55f0f5bSNeel Patel
43783469893SShannon Nelson len = netdev->mtu + ETH_HLEN + VLAN_HLEN;
4380f3154e6SShannon Nelson
439e55f0f5bSNeel Patel for (i = n_fill; i; i--) {
4404b0a7539SShannon Nelson nfrags = 0;
441c37d6e3fSShannon Nelson remain_len = len;
442f1d2e894SShannon Nelson desc_info = &q->info[q->head_idx];
44308f2e4b2SShannon Nelson desc = desc_info->desc;
4444b0a7539SShannon Nelson buf_info = &desc_info->bufs[0];
4450f3154e6SShannon Nelson
4464b0a7539SShannon Nelson if (!buf_info->page) { /* alloc a new buffer? */
4474b0a7539SShannon Nelson if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
44808f2e4b2SShannon Nelson desc->addr = 0;
44908f2e4b2SShannon Nelson desc->len = 0;
45008f2e4b2SShannon Nelson return;
45108f2e4b2SShannon Nelson }
4524b0a7539SShannon Nelson }
45308f2e4b2SShannon Nelson
4544b0a7539SShannon Nelson /* fill main descriptor - buf[0] */
4554b0a7539SShannon Nelson desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
456*8f6b846bSDavid Christensen frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
457*8f6b846bSDavid Christensen IONIC_PAGE_SIZE - buf_info->page_offset));
4584b0a7539SShannon Nelson desc->len = cpu_to_le16(frag_len);
4594b0a7539SShannon Nelson remain_len -= frag_len;
4604b0a7539SShannon Nelson buf_info++;
4614b0a7539SShannon Nelson nfrags++;
46208f2e4b2SShannon Nelson
4634b0a7539SShannon Nelson /* fill sg descriptors - buf[1..n] */
4644b0a7539SShannon Nelson sg_desc = desc_info->sg_desc;
465f37bc346SShannon Nelson for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {
46608f2e4b2SShannon Nelson sg_elem = &sg_desc->elems[j];
4674b0a7539SShannon Nelson if (!buf_info->page) { /* alloc a new sg buffer? */
4684b0a7539SShannon Nelson if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
46908f2e4b2SShannon Nelson sg_elem->addr = 0;
47008f2e4b2SShannon Nelson sg_elem->len = 0;
47108f2e4b2SShannon Nelson return;
47208f2e4b2SShannon Nelson }
47308f2e4b2SShannon Nelson }
4740f3154e6SShannon Nelson
4754b0a7539SShannon Nelson sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
476*8f6b846bSDavid Christensen frag_len = min_t(u16, remain_len, min_t(u32, IONIC_MAX_BUF_LEN,
477*8f6b846bSDavid Christensen IONIC_PAGE_SIZE -
478*8f6b846bSDavid Christensen buf_info->page_offset));
4794b0a7539SShannon Nelson sg_elem->len = cpu_to_le16(frag_len);
4804b0a7539SShannon Nelson remain_len -= frag_len;
4814b0a7539SShannon Nelson buf_info++;
4824b0a7539SShannon Nelson nfrags++;
4834b0a7539SShannon Nelson }
4844b0a7539SShannon Nelson
4854b0a7539SShannon Nelson /* clear end sg element as a sentinel */
486f37bc346SShannon Nelson if (j < q->max_sg_elems) {
4874b0a7539SShannon Nelson sg_elem = &sg_desc->elems[j];
4884b0a7539SShannon Nelson memset(sg_elem, 0, sizeof(*sg_elem));
4894b0a7539SShannon Nelson }
4904b0a7539SShannon Nelson
4914b0a7539SShannon Nelson desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
4924b0a7539SShannon Nelson IONIC_RXQ_DESC_OPCODE_SIMPLE;
4934b0a7539SShannon Nelson desc_info->nbufs = nfrags;
4944b0a7539SShannon Nelson
49540bc471dSShannon Nelson ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
49640bc471dSShannon Nelson
497155f15adSShannon Nelson ionic_rxq_post(q, false, ionic_rx_clean, NULL);
4980f3154e6SShannon Nelson }
499155f15adSShannon Nelson
500155f15adSShannon Nelson ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
501f1d2e894SShannon Nelson q->dbval | q->head_idx);
502b69585bfSAllen Hubbe
503b69585bfSAllen Hubbe q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
504b69585bfSAllen Hubbe q->dbell_jiffies = jiffies;
505b69585bfSAllen Hubbe
506b69585bfSAllen Hubbe mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
507b69585bfSAllen Hubbe jiffies + IONIC_NAPI_DEADLINE);
5080f3154e6SShannon Nelson }
5090f3154e6SShannon Nelson
ionic_rx_empty(struct ionic_queue * q)5100f3154e6SShannon Nelson void ionic_rx_empty(struct ionic_queue *q)
5110f3154e6SShannon Nelson {
512f1d2e894SShannon Nelson struct ionic_desc_info *desc_info;
5134b0a7539SShannon Nelson struct ionic_buf_info *buf_info;
5140c32a28eSShannon Nelson unsigned int i, j;
5150f3154e6SShannon Nelson
5160c32a28eSShannon Nelson for (i = 0; i < q->num_descs; i++) {
5170c32a28eSShannon Nelson desc_info = &q->info[i];
5180c32a28eSShannon Nelson for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) {
5194b0a7539SShannon Nelson buf_info = &desc_info->bufs[j];
5204b0a7539SShannon Nelson if (buf_info->page)
5214b0a7539SShannon Nelson ionic_rx_page_free(q, buf_info);
5220c32a28eSShannon Nelson }
52308f2e4b2SShannon Nelson
5244b0a7539SShannon Nelson desc_info->nbufs = 0;
5250c32a28eSShannon Nelson desc_info->cb = NULL;
526f1d2e894SShannon Nelson desc_info->cb_arg = NULL;
5270f3154e6SShannon Nelson }
5284b0a7539SShannon Nelson
5294b0a7539SShannon Nelson q->head_idx = 0;
5304b0a7539SShannon Nelson q->tail_idx = 0;
5310f3154e6SShannon Nelson }
5320f3154e6SShannon Nelson
ionic_dim_update(struct ionic_qcq * qcq,int napi_mode)53376ed8a4aSShannon Nelson static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
53404a83459SShannon Nelson {
53504a83459SShannon Nelson struct dim_sample dim_sample;
53604a83459SShannon Nelson struct ionic_lif *lif;
53704a83459SShannon Nelson unsigned int qi;
53876ed8a4aSShannon Nelson u64 pkts, bytes;
53904a83459SShannon Nelson
54004a83459SShannon Nelson if (!qcq->intr.dim_coal_hw)
54104a83459SShannon Nelson return;
54204a83459SShannon Nelson
54304a83459SShannon Nelson lif = qcq->q.lif;
54404a83459SShannon Nelson qi = qcq->cq.bound_q->index;
54504a83459SShannon Nelson
54676ed8a4aSShannon Nelson switch (napi_mode) {
54776ed8a4aSShannon Nelson case IONIC_LIF_F_TX_DIM_INTR:
54876ed8a4aSShannon Nelson pkts = lif->txqstats[qi].pkts;
54976ed8a4aSShannon Nelson bytes = lif->txqstats[qi].bytes;
55076ed8a4aSShannon Nelson break;
55176ed8a4aSShannon Nelson case IONIC_LIF_F_RX_DIM_INTR:
55276ed8a4aSShannon Nelson pkts = lif->rxqstats[qi].pkts;
55376ed8a4aSShannon Nelson bytes = lif->rxqstats[qi].bytes;
55476ed8a4aSShannon Nelson break;
55576ed8a4aSShannon Nelson default:
55676ed8a4aSShannon Nelson pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
55776ed8a4aSShannon Nelson bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
55876ed8a4aSShannon Nelson break;
55976ed8a4aSShannon Nelson }
56004a83459SShannon Nelson
56104a83459SShannon Nelson dim_update_sample(qcq->cq.bound_intr->rearm_count,
56276ed8a4aSShannon Nelson pkts, bytes, &dim_sample);
56304a83459SShannon Nelson
56404a83459SShannon Nelson net_dim(&qcq->dim, dim_sample);
56504a83459SShannon Nelson }
56604a83459SShannon Nelson
ionic_tx_napi(struct napi_struct * napi,int budget)567fe8c30b5SShannon Nelson int ionic_tx_napi(struct napi_struct *napi, int budget)
568fe8c30b5SShannon Nelson {
569fe8c30b5SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi);
570fe8c30b5SShannon Nelson struct ionic_cq *cq = napi_to_cq(napi);
571fe8c30b5SShannon Nelson struct ionic_dev *idev;
572fe8c30b5SShannon Nelson struct ionic_lif *lif;
573fe8c30b5SShannon Nelson u32 work_done = 0;
574fe8c30b5SShannon Nelson u32 flags = 0;
575fe8c30b5SShannon Nelson
576fe8c30b5SShannon Nelson lif = cq->bound_q->lif;
577fe8c30b5SShannon Nelson idev = &lif->ionic->idev;
578fe8c30b5SShannon Nelson
579fe8c30b5SShannon Nelson work_done = ionic_cq_service(cq, budget,
580fe8c30b5SShannon Nelson ionic_tx_service, NULL, NULL);
581fe8c30b5SShannon Nelson
582fe8c30b5SShannon Nelson if (work_done < budget && napi_complete_done(napi, work_done)) {
58376ed8a4aSShannon Nelson ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
584fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK;
58504a83459SShannon Nelson cq->bound_intr->rearm_count++;
586fe8c30b5SShannon Nelson }
587fe8c30b5SShannon Nelson
588fe8c30b5SShannon Nelson if (work_done || flags) {
589fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE;
590fe8c30b5SShannon Nelson ionic_intr_credits(idev->intr_ctrl,
591fe8c30b5SShannon Nelson cq->bound_intr->index,
592fe8c30b5SShannon Nelson work_done, flags);
593fe8c30b5SShannon Nelson }
594fe8c30b5SShannon Nelson
595b69585bfSAllen Hubbe if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
596b69585bfSAllen Hubbe mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
597b69585bfSAllen Hubbe
598fe8c30b5SShannon Nelson return work_done;
599fe8c30b5SShannon Nelson }
600fe8c30b5SShannon Nelson
ionic_rx_napi(struct napi_struct * napi,int budget)6010f3154e6SShannon Nelson int ionic_rx_napi(struct napi_struct *napi, int budget)
6020f3154e6SShannon Nelson {
6030f3154e6SShannon Nelson struct ionic_qcq *qcq = napi_to_qcq(napi);
604fe8c30b5SShannon Nelson struct ionic_cq *cq = napi_to_cq(napi);
605fe8c30b5SShannon Nelson struct ionic_dev *idev;
606fe8c30b5SShannon Nelson struct ionic_lif *lif;
607fe8c30b5SShannon Nelson u32 work_done = 0;
608fe8c30b5SShannon Nelson u32 flags = 0;
609fe8c30b5SShannon Nelson
610fe8c30b5SShannon Nelson lif = cq->bound_q->lif;
611fe8c30b5SShannon Nelson idev = &lif->ionic->idev;
612fe8c30b5SShannon Nelson
613fe8c30b5SShannon Nelson work_done = ionic_cq_service(cq, budget,
614fe8c30b5SShannon Nelson ionic_rx_service, NULL, NULL);
615fe8c30b5SShannon Nelson
616fe8c30b5SShannon Nelson ionic_rx_fill(cq->bound_q);
617fe8c30b5SShannon Nelson
618fe8c30b5SShannon Nelson if (work_done < budget && napi_complete_done(napi, work_done)) {
61976ed8a4aSShannon Nelson ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
620fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK;
62104a83459SShannon Nelson cq->bound_intr->rearm_count++;
622fe8c30b5SShannon Nelson }
623fe8c30b5SShannon Nelson
624fe8c30b5SShannon Nelson if (work_done || flags) {
625fe8c30b5SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE;
626fe8c30b5SShannon Nelson ionic_intr_credits(idev->intr_ctrl,
627fe8c30b5SShannon Nelson cq->bound_intr->index,
628fe8c30b5SShannon Nelson work_done, flags);
629fe8c30b5SShannon Nelson }
630fe8c30b5SShannon Nelson
631b69585bfSAllen Hubbe if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
632b69585bfSAllen Hubbe mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
633b69585bfSAllen Hubbe
634fe8c30b5SShannon Nelson return work_done;
635fe8c30b5SShannon Nelson }
636fe8c30b5SShannon Nelson
ionic_txrx_napi(struct napi_struct * napi,int budget)637fe8c30b5SShannon Nelson int ionic_txrx_napi(struct napi_struct *napi, int budget)
638fe8c30b5SShannon Nelson {
639b69585bfSAllen Hubbe struct ionic_qcq *rxqcq = napi_to_qcq(napi);
6400f3154e6SShannon Nelson struct ionic_cq *rxcq = napi_to_cq(napi);
6410f3154e6SShannon Nelson unsigned int qi = rxcq->bound_q->index;
642b69585bfSAllen Hubbe struct ionic_qcq *txqcq;
6430f3154e6SShannon Nelson struct ionic_dev *idev;
6440f3154e6SShannon Nelson struct ionic_lif *lif;
6450f3154e6SShannon Nelson struct ionic_cq *txcq;
646b69585bfSAllen Hubbe bool resched = false;
647b14e4e95SShannon Nelson u32 rx_work_done = 0;
648b14e4e95SShannon Nelson u32 tx_work_done = 0;
6490f3154e6SShannon Nelson u32 flags = 0;
6500f3154e6SShannon Nelson
6510f3154e6SShannon Nelson lif = rxcq->bound_q->lif;
6520f3154e6SShannon Nelson idev = &lif->ionic->idev;
653b69585bfSAllen Hubbe txqcq = lif->txqcqs[qi];
65434dec947SShannon Nelson txcq = &lif->txqcqs[qi]->cq;
6550f3154e6SShannon Nelson
656f37bc346SShannon Nelson tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT,
657b14e4e95SShannon Nelson ionic_tx_service, NULL, NULL);
6580f3154e6SShannon Nelson
659b14e4e95SShannon Nelson rx_work_done = ionic_cq_service(rxcq, budget,
660b14e4e95SShannon Nelson ionic_rx_service, NULL, NULL);
661a8205ab6SShannon Nelson
662a8205ab6SShannon Nelson ionic_rx_fill(rxcq->bound_q);
6630f3154e6SShannon Nelson
6649dda5110SShannon Nelson if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
665b69585bfSAllen Hubbe ionic_dim_update(rxqcq, 0);
6660f3154e6SShannon Nelson flags |= IONIC_INTR_CRED_UNMASK;
66704a83459SShannon Nelson rxcq->bound_intr->rearm_count++;
6680f3154e6SShannon Nelson }
6690f3154e6SShannon Nelson
6709dda5110SShannon Nelson if (rx_work_done || flags) {
6710f3154e6SShannon Nelson flags |= IONIC_INTR_CRED_RESET_COALESCE;
6720f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
673b14e4e95SShannon Nelson tx_work_done + rx_work_done, flags);
6740f3154e6SShannon Nelson }
6750f3154e6SShannon Nelson
676b69585bfSAllen Hubbe if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
677b69585bfSAllen Hubbe resched = true;
678b69585bfSAllen Hubbe if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
679b69585bfSAllen Hubbe resched = true;
680b69585bfSAllen Hubbe if (resched)
681b69585bfSAllen Hubbe mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
682b69585bfSAllen Hubbe
6839dda5110SShannon Nelson return rx_work_done;
6840f3154e6SShannon Nelson }
6850f3154e6SShannon Nelson
ionic_tx_map_single(struct ionic_queue * q,void * data,size_t len)6865b3f3f2aSShannon Nelson static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
6875b3f3f2aSShannon Nelson void *data, size_t len)
6880f3154e6SShannon Nelson {
6890f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
690f37bc346SShannon Nelson struct device *dev = q->dev;
6910f3154e6SShannon Nelson dma_addr_t dma_addr;
6920f3154e6SShannon Nelson
6930f3154e6SShannon Nelson dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
6940f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) {
6950f3154e6SShannon Nelson net_warn_ratelimited("%s: DMA single map failed on %s!\n",
6960f3154e6SShannon Nelson q->lif->netdev->name, q->name);
6970f3154e6SShannon Nelson stats->dma_map_err++;
6980f3154e6SShannon Nelson return 0;
6990f3154e6SShannon Nelson }
7000f3154e6SShannon Nelson return dma_addr;
7010f3154e6SShannon Nelson }
7020f3154e6SShannon Nelson
ionic_tx_map_frag(struct ionic_queue * q,const skb_frag_t * frag,size_t offset,size_t len)7035b3f3f2aSShannon Nelson static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
7045b3f3f2aSShannon Nelson const skb_frag_t *frag,
7050f3154e6SShannon Nelson size_t offset, size_t len)
7060f3154e6SShannon Nelson {
7070f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
708f37bc346SShannon Nelson struct device *dev = q->dev;
7090f3154e6SShannon Nelson dma_addr_t dma_addr;
7100f3154e6SShannon Nelson
7110f3154e6SShannon Nelson dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
7120f3154e6SShannon Nelson if (dma_mapping_error(dev, dma_addr)) {
7130f3154e6SShannon Nelson net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
7140f3154e6SShannon Nelson q->lif->netdev->name, q->name);
7150f3154e6SShannon Nelson stats->dma_map_err++;
7160f3154e6SShannon Nelson }
7170f3154e6SShannon Nelson return dma_addr;
7180f3154e6SShannon Nelson }
7190f3154e6SShannon Nelson
ionic_tx_map_skb(struct ionic_queue * q,struct sk_buff * skb,struct ionic_desc_info * desc_info)7202da479caSShannon Nelson static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
7212da479caSShannon Nelson struct ionic_desc_info *desc_info)
7225b039241SShannon Nelson {
7232da479caSShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs;
7240f4e7f4eSShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
7255b039241SShannon Nelson struct device *dev = q->dev;
7265b039241SShannon Nelson dma_addr_t dma_addr;
7272da479caSShannon Nelson unsigned int nfrags;
7285b039241SShannon Nelson skb_frag_t *frag;
7295b039241SShannon Nelson int frag_idx;
7305b039241SShannon Nelson
7315b039241SShannon Nelson dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
7320f4e7f4eSShannon Nelson if (dma_mapping_error(dev, dma_addr)) {
7330f4e7f4eSShannon Nelson stats->dma_map_err++;
7345b039241SShannon Nelson return -EIO;
7350f4e7f4eSShannon Nelson }
7365b039241SShannon Nelson buf_info->dma_addr = dma_addr;
7375b039241SShannon Nelson buf_info->len = skb_headlen(skb);
7385b039241SShannon Nelson buf_info++;
7395b039241SShannon Nelson
7402da479caSShannon Nelson frag = skb_shinfo(skb)->frags;
7412da479caSShannon Nelson nfrags = skb_shinfo(skb)->nr_frags;
7422da479caSShannon Nelson for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
7435b039241SShannon Nelson dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
7440f4e7f4eSShannon Nelson if (dma_mapping_error(dev, dma_addr)) {
7450f4e7f4eSShannon Nelson stats->dma_map_err++;
7465b039241SShannon Nelson goto dma_fail;
7470f4e7f4eSShannon Nelson }
7485b039241SShannon Nelson buf_info->dma_addr = dma_addr;
7495b039241SShannon Nelson buf_info->len = skb_frag_size(frag);
7502da479caSShannon Nelson buf_info++;
7515b039241SShannon Nelson }
7525b039241SShannon Nelson
7532da479caSShannon Nelson desc_info->nbufs = 1 + nfrags;
7542da479caSShannon Nelson
7555b039241SShannon Nelson return 0;
7565b039241SShannon Nelson
7575b039241SShannon Nelson dma_fail:
7585b039241SShannon Nelson /* unwind the frag mappings and the head mapping */
7595b039241SShannon Nelson while (frag_idx > 0) {
7605b039241SShannon Nelson frag_idx--;
7615b039241SShannon Nelson buf_info--;
7625b039241SShannon Nelson dma_unmap_page(dev, buf_info->dma_addr,
7635b039241SShannon Nelson buf_info->len, DMA_TO_DEVICE);
7645b039241SShannon Nelson }
7655b039241SShannon Nelson dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
7665b039241SShannon Nelson return -EIO;
7675b039241SShannon Nelson }
7685b039241SShannon Nelson
ionic_tx_desc_unmap_bufs(struct ionic_queue * q,struct ionic_desc_info * desc_info)769238a0f7cSBrett Creeley static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
770238a0f7cSBrett Creeley struct ionic_desc_info *desc_info)
7710f3154e6SShannon Nelson {
7725b039241SShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs;
773f37bc346SShannon Nelson struct device *dev = q->dev;
7740f3154e6SShannon Nelson unsigned int i;
7750f3154e6SShannon Nelson
776238a0f7cSBrett Creeley if (!desc_info->nbufs)
777238a0f7cSBrett Creeley return;
778238a0f7cSBrett Creeley
7795b039241SShannon Nelson dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
7805b039241SShannon Nelson buf_info->len, DMA_TO_DEVICE);
7815b039241SShannon Nelson buf_info++;
7825b039241SShannon Nelson for (i = 1; i < desc_info->nbufs; i++, buf_info++)
7835b039241SShannon Nelson dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
7845b039241SShannon Nelson buf_info->len, DMA_TO_DEVICE);
785238a0f7cSBrett Creeley
786238a0f7cSBrett Creeley desc_info->nbufs = 0;
7875b039241SShannon Nelson }
7880f3154e6SShannon Nelson
ionic_tx_clean(struct ionic_queue * q,struct ionic_desc_info * desc_info,struct ionic_cq_info * cq_info,void * cb_arg)789238a0f7cSBrett Creeley static void ionic_tx_clean(struct ionic_queue *q,
790238a0f7cSBrett Creeley struct ionic_desc_info *desc_info,
791238a0f7cSBrett Creeley struct ionic_cq_info *cq_info,
792238a0f7cSBrett Creeley void *cb_arg)
793238a0f7cSBrett Creeley {
794238a0f7cSBrett Creeley struct ionic_tx_stats *stats = q_to_tx_stats(q);
795238a0f7cSBrett Creeley struct ionic_qcq *qcq = q_to_qcq(q);
796238a0f7cSBrett Creeley struct sk_buff *skb = cb_arg;
797238a0f7cSBrett Creeley u16 qi;
798238a0f7cSBrett Creeley
799238a0f7cSBrett Creeley ionic_tx_desc_unmap_bufs(q, desc_info);
800238a0f7cSBrett Creeley
801a8771bfeSShannon Nelson if (!skb)
802a8771bfeSShannon Nelson return;
8030f3154e6SShannon Nelson
804a8771bfeSShannon Nelson qi = skb_get_queue_mapping(skb);
805a8771bfeSShannon Nelson
806a8771bfeSShannon Nelson if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) {
807a8771bfeSShannon Nelson if (cq_info) {
808a8771bfeSShannon Nelson struct skb_shared_hwtstamps hwts = {};
809a8771bfeSShannon Nelson __le64 *cq_desc_hwstamp;
810a8771bfeSShannon Nelson u64 hwstamp;
811a8771bfeSShannon Nelson
812a8771bfeSShannon Nelson cq_desc_hwstamp =
813a8771bfeSShannon Nelson cq_info->cq_desc +
814a8771bfeSShannon Nelson qcq->cq.desc_size -
815a8771bfeSShannon Nelson sizeof(struct ionic_txq_comp) -
816a8771bfeSShannon Nelson IONIC_HWSTAMP_CQ_NEGOFFSET;
817a8771bfeSShannon Nelson
818a8771bfeSShannon Nelson hwstamp = le64_to_cpu(*cq_desc_hwstamp);
819a8771bfeSShannon Nelson
820a8771bfeSShannon Nelson if (hwstamp != IONIC_HWSTAMP_INVALID) {
821a8771bfeSShannon Nelson hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
822a8771bfeSShannon Nelson
823a8771bfeSShannon Nelson skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
824a8771bfeSShannon Nelson skb_tstamp_tx(skb, &hwts);
825a8771bfeSShannon Nelson
826a8771bfeSShannon Nelson stats->hwstamp_valid++;
827a8771bfeSShannon Nelson } else {
828a8771bfeSShannon Nelson stats->hwstamp_invalid++;
829a8771bfeSShannon Nelson }
830a8771bfeSShannon Nelson }
831a8771bfeSShannon Nelson
832a8771bfeSShannon Nelson } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) {
833a8771bfeSShannon Nelson netif_wake_subqueue(q->lif->netdev, qi);
8340f3154e6SShannon Nelson }
835633eddf1SShannon Nelson
836633eddf1SShannon Nelson desc_info->bytes = skb->len;
8370f3154e6SShannon Nelson stats->clean++;
838633eddf1SShannon Nelson
839633eddf1SShannon Nelson dev_consume_skb_any(skb);
8400f3154e6SShannon Nelson }
8410f3154e6SShannon Nelson
ionic_tx_service(struct ionic_cq * cq,struct ionic_cq_info * cq_info)842a8771bfeSShannon Nelson bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
8430f3154e6SShannon Nelson {
8440f3154e6SShannon Nelson struct ionic_queue *q = cq->bound_q;
8450f3154e6SShannon Nelson struct ionic_desc_info *desc_info;
8460ec9f666SShannon Nelson struct ionic_txq_comp *comp;
847633eddf1SShannon Nelson int bytes = 0;
848633eddf1SShannon Nelson int pkts = 0;
849339dcf7fSShannon Nelson u16 index;
8500f3154e6SShannon Nelson
8510ec9f666SShannon Nelson comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
8520ec9f666SShannon Nelson
853b14e4e95SShannon Nelson if (!color_match(comp->color, cq->done_color))
854b14e4e95SShannon Nelson return false;
8550f3154e6SShannon Nelson
8560f3154e6SShannon Nelson /* clean the related q entries, there could be
8570f3154e6SShannon Nelson * several q entries completed for each cq completion
8580f3154e6SShannon Nelson */
8590f3154e6SShannon Nelson do {
860f1d2e894SShannon Nelson desc_info = &q->info[q->tail_idx];
861633eddf1SShannon Nelson desc_info->bytes = 0;
862339dcf7fSShannon Nelson index = q->tail_idx;
863f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
864f1d2e894SShannon Nelson ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);
865633eddf1SShannon Nelson if (desc_info->cb_arg) {
866633eddf1SShannon Nelson pkts++;
867633eddf1SShannon Nelson bytes += desc_info->bytes;
868633eddf1SShannon Nelson }
8690f3154e6SShannon Nelson desc_info->cb = NULL;
8700f3154e6SShannon Nelson desc_info->cb_arg = NULL;
871339dcf7fSShannon Nelson } while (index != le16_to_cpu(comp->comp_index));
8720f3154e6SShannon Nelson
873a8771bfeSShannon Nelson if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
874633eddf1SShannon Nelson netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
875633eddf1SShannon Nelson
876b14e4e95SShannon Nelson return true;
8770f3154e6SShannon Nelson }
8780f3154e6SShannon Nelson
ionic_tx_flush(struct ionic_cq * cq)879b14e4e95SShannon Nelson void ionic_tx_flush(struct ionic_cq *cq)
880b14e4e95SShannon Nelson {
881b14e4e95SShannon Nelson struct ionic_dev *idev = &cq->lif->ionic->idev;
882b14e4e95SShannon Nelson u32 work_done;
883b14e4e95SShannon Nelson
884b14e4e95SShannon Nelson work_done = ionic_cq_service(cq, cq->num_descs,
885b14e4e95SShannon Nelson ionic_tx_service, NULL, NULL);
8860f3154e6SShannon Nelson if (work_done)
8870f3154e6SShannon Nelson ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
888b14e4e95SShannon Nelson work_done, IONIC_INTR_CRED_RESET_COALESCE);
8890f3154e6SShannon Nelson }
8900f3154e6SShannon Nelson
ionic_tx_empty(struct ionic_queue * q)891f9c00e2cSShannon Nelson void ionic_tx_empty(struct ionic_queue *q)
892f9c00e2cSShannon Nelson {
893f9c00e2cSShannon Nelson struct ionic_desc_info *desc_info;
894633eddf1SShannon Nelson int bytes = 0;
895633eddf1SShannon Nelson int pkts = 0;
896f9c00e2cSShannon Nelson
897f9c00e2cSShannon Nelson /* walk the not completed tx entries, if any */
898f1d2e894SShannon Nelson while (q->head_idx != q->tail_idx) {
899f1d2e894SShannon Nelson desc_info = &q->info[q->tail_idx];
900633eddf1SShannon Nelson desc_info->bytes = 0;
901f1d2e894SShannon Nelson q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
902f9c00e2cSShannon Nelson ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
903633eddf1SShannon Nelson if (desc_info->cb_arg) {
904633eddf1SShannon Nelson pkts++;
905633eddf1SShannon Nelson bytes += desc_info->bytes;
906633eddf1SShannon Nelson }
907f9c00e2cSShannon Nelson desc_info->cb = NULL;
908f9c00e2cSShannon Nelson desc_info->cb_arg = NULL;
909f9c00e2cSShannon Nelson }
910633eddf1SShannon Nelson
911a8771bfeSShannon Nelson if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
912633eddf1SShannon Nelson netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
913f9c00e2cSShannon Nelson }
914f9c00e2cSShannon Nelson
ionic_tx_tcp_inner_pseudo_csum(struct sk_buff * skb)9150f3154e6SShannon Nelson static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
9160f3154e6SShannon Nelson {
9170f3154e6SShannon Nelson int err;
9180f3154e6SShannon Nelson
9190f3154e6SShannon Nelson err = skb_cow_head(skb, 0);
9200f3154e6SShannon Nelson if (err)
9210f3154e6SShannon Nelson return err;
9220f3154e6SShannon Nelson
9230f3154e6SShannon Nelson if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
9240f3154e6SShannon Nelson inner_ip_hdr(skb)->check = 0;
9250f3154e6SShannon Nelson inner_tcp_hdr(skb)->check =
9260f3154e6SShannon Nelson ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
9270f3154e6SShannon Nelson inner_ip_hdr(skb)->daddr,
9280f3154e6SShannon Nelson 0, IPPROTO_TCP, 0);
9290f3154e6SShannon Nelson } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
9300f3154e6SShannon Nelson inner_tcp_hdr(skb)->check =
9310f3154e6SShannon Nelson ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
9320f3154e6SShannon Nelson &inner_ipv6_hdr(skb)->daddr,
9330f3154e6SShannon Nelson 0, IPPROTO_TCP, 0);
9340f3154e6SShannon Nelson }
9350f3154e6SShannon Nelson
9360f3154e6SShannon Nelson return 0;
9370f3154e6SShannon Nelson }
9380f3154e6SShannon Nelson
ionic_tx_tcp_pseudo_csum(struct sk_buff * skb)9390f3154e6SShannon Nelson static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
9400f3154e6SShannon Nelson {
9410f3154e6SShannon Nelson int err;
9420f3154e6SShannon Nelson
9430f3154e6SShannon Nelson err = skb_cow_head(skb, 0);
9440f3154e6SShannon Nelson if (err)
9450f3154e6SShannon Nelson return err;
9460f3154e6SShannon Nelson
9470f3154e6SShannon Nelson if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
9480f3154e6SShannon Nelson ip_hdr(skb)->check = 0;
9490f3154e6SShannon Nelson tcp_hdr(skb)->check =
9500f3154e6SShannon Nelson ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
9510f3154e6SShannon Nelson ip_hdr(skb)->daddr,
9520f3154e6SShannon Nelson 0, IPPROTO_TCP, 0);
9530f3154e6SShannon Nelson } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
954fa6b8429SHeiner Kallweit tcp_v6_gso_csum_prep(skb);
9550f3154e6SShannon Nelson }
9560f3154e6SShannon Nelson
9570f3154e6SShannon Nelson return 0;
9580f3154e6SShannon Nelson }
9590f3154e6SShannon Nelson
ionic_tx_tso_post(struct ionic_queue * q,struct ionic_desc_info * desc_info,struct sk_buff * skb,dma_addr_t addr,u8 nsge,u16 len,unsigned int hdrlen,unsigned int mss,bool outer_csum,u16 vlan_tci,bool has_vlan,bool start,bool done)96040bc471dSShannon Nelson static void ionic_tx_tso_post(struct ionic_queue *q,
96140bc471dSShannon Nelson struct ionic_desc_info *desc_info,
9620f3154e6SShannon Nelson struct sk_buff *skb,
9630f3154e6SShannon Nelson dma_addr_t addr, u8 nsge, u16 len,
9640f3154e6SShannon Nelson unsigned int hdrlen, unsigned int mss,
9650f3154e6SShannon Nelson bool outer_csum,
9660f3154e6SShannon Nelson u16 vlan_tci, bool has_vlan,
9670f3154e6SShannon Nelson bool start, bool done)
9680f3154e6SShannon Nelson {
96940bc471dSShannon Nelson struct ionic_txq_desc *desc = desc_info->desc;
9700f3154e6SShannon Nelson u8 flags = 0;
9710f3154e6SShannon Nelson u64 cmd;
9720f3154e6SShannon Nelson
9730f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
9740f3154e6SShannon Nelson flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
9750f3154e6SShannon Nelson flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
9760f3154e6SShannon Nelson flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
9770f3154e6SShannon Nelson
9780f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
9790f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd);
9800f3154e6SShannon Nelson desc->len = cpu_to_le16(len);
9810f3154e6SShannon Nelson desc->vlan_tci = cpu_to_le16(vlan_tci);
9820f3154e6SShannon Nelson desc->hdr_len = cpu_to_le16(hdrlen);
9830f3154e6SShannon Nelson desc->mss = cpu_to_le16(mss);
9840f3154e6SShannon Nelson
98540bc471dSShannon Nelson ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
98640bc471dSShannon Nelson
9872da479caSShannon Nelson if (start) {
9880f3154e6SShannon Nelson skb_tx_timestamp(skb);
989a8771bfeSShannon Nelson if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
9900f3154e6SShannon Nelson netdev_tx_sent_queue(q_to_ndq(q), skb->len);
9912da479caSShannon Nelson ionic_txq_post(q, false, ionic_tx_clean, skb);
9920f3154e6SShannon Nelson } else {
9932da479caSShannon Nelson ionic_txq_post(q, done, NULL, NULL);
9940f3154e6SShannon Nelson }
9950f3154e6SShannon Nelson }
9960f3154e6SShannon Nelson
ionic_tx_tso(struct ionic_queue * q,struct sk_buff * skb)9970f3154e6SShannon Nelson static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
9980f3154e6SShannon Nelson {
9990f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
10002da479caSShannon Nelson struct ionic_desc_info *desc_info;
10012da479caSShannon Nelson struct ionic_buf_info *buf_info;
10020f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem;
10030f3154e6SShannon Nelson struct ionic_txq_desc *desc;
10045b039241SShannon Nelson unsigned int chunk_len;
10055b039241SShannon Nelson unsigned int frag_rem;
10065b039241SShannon Nelson unsigned int tso_rem;
10075b039241SShannon Nelson unsigned int seg_rem;
10080f3154e6SShannon Nelson dma_addr_t desc_addr;
10095b039241SShannon Nelson dma_addr_t frag_addr;
10100f3154e6SShannon Nelson unsigned int hdrlen;
10110f3154e6SShannon Nelson unsigned int len;
10120f3154e6SShannon Nelson unsigned int mss;
10130f3154e6SShannon Nelson bool start, done;
10140f3154e6SShannon Nelson bool outer_csum;
10150f3154e6SShannon Nelson bool has_vlan;
10160f3154e6SShannon Nelson u16 desc_len;
10170f3154e6SShannon Nelson u8 desc_nsge;
10180f3154e6SShannon Nelson u16 vlan_tci;
10190f3154e6SShannon Nelson bool encap;
10200f3154e6SShannon Nelson int err;
10210f3154e6SShannon Nelson
10222da479caSShannon Nelson desc_info = &q->info[q->head_idx];
10232da479caSShannon Nelson buf_info = desc_info->bufs;
10242da479caSShannon Nelson
10252da479caSShannon Nelson if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
10265b039241SShannon Nelson return -EIO;
10275b039241SShannon Nelson
10285b039241SShannon Nelson len = skb->len;
10290f3154e6SShannon Nelson mss = skb_shinfo(skb)->gso_size;
1030cad478c7SNeel Patel outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1031cad478c7SNeel Patel SKB_GSO_GRE_CSUM |
1032cad478c7SNeel Patel SKB_GSO_IPXIP4 |
1033cad478c7SNeel Patel SKB_GSO_IPXIP6 |
1034cad478c7SNeel Patel SKB_GSO_UDP_TUNNEL |
1035cad478c7SNeel Patel SKB_GSO_UDP_TUNNEL_CSUM));
10360f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb);
10370f3154e6SShannon Nelson vlan_tci = skb_vlan_tag_get(skb);
10380f3154e6SShannon Nelson encap = skb->encapsulation;
10390f3154e6SShannon Nelson
10400f3154e6SShannon Nelson /* Preload inner-most TCP csum field with IP pseudo hdr
10410f3154e6SShannon Nelson * calculated with IP length set to zero. HW will later
10420f3154e6SShannon Nelson * add in length to each TCP segment resulting from the TSO.
10430f3154e6SShannon Nelson */
10440f3154e6SShannon Nelson
10450f3154e6SShannon Nelson if (encap)
10460f3154e6SShannon Nelson err = ionic_tx_tcp_inner_pseudo_csum(skb);
10470f3154e6SShannon Nelson else
10480f3154e6SShannon Nelson err = ionic_tx_tcp_pseudo_csum(skb);
1049238a0f7cSBrett Creeley if (err) {
1050238a0f7cSBrett Creeley /* clean up mapping from ionic_tx_map_skb */
1051238a0f7cSBrett Creeley ionic_tx_desc_unmap_bufs(q, desc_info);
10520f3154e6SShannon Nelson return err;
1053238a0f7cSBrett Creeley }
10540f3154e6SShannon Nelson
10550f3154e6SShannon Nelson if (encap)
1056504148feSEric Dumazet hdrlen = skb_inner_tcp_all_headers(skb);
10570f3154e6SShannon Nelson else
1058504148feSEric Dumazet hdrlen = skb_tcp_all_headers(skb);
10590f3154e6SShannon Nelson
10605b039241SShannon Nelson tso_rem = len;
10615b039241SShannon Nelson seg_rem = min(tso_rem, hdrlen + mss);
10620f3154e6SShannon Nelson
10635b039241SShannon Nelson frag_addr = 0;
10645b039241SShannon Nelson frag_rem = 0;
10655b039241SShannon Nelson
10660f3154e6SShannon Nelson start = true;
10670f3154e6SShannon Nelson
10685b039241SShannon Nelson while (tso_rem > 0) {
10695b039241SShannon Nelson desc = NULL;
10705b039241SShannon Nelson elem = NULL;
10715b039241SShannon Nelson desc_addr = 0;
10725b039241SShannon Nelson desc_len = 0;
10730f3154e6SShannon Nelson desc_nsge = 0;
10742da479caSShannon Nelson /* use fragments until we have enough to post a single descriptor */
10755b039241SShannon Nelson while (seg_rem > 0) {
10762da479caSShannon Nelson /* if the fragment is exhausted then move to the next one */
10775b039241SShannon Nelson if (frag_rem == 0) {
10785b039241SShannon Nelson /* grab the next fragment */
10792da479caSShannon Nelson frag_addr = buf_info->dma_addr;
10802da479caSShannon Nelson frag_rem = buf_info->len;
10812da479caSShannon Nelson buf_info++;
10820f3154e6SShannon Nelson }
10835b039241SShannon Nelson chunk_len = min(frag_rem, seg_rem);
10845b039241SShannon Nelson if (!desc) {
10855b039241SShannon Nelson /* fill main descriptor */
10862da479caSShannon Nelson desc = desc_info->txq_desc;
10872da479caSShannon Nelson elem = desc_info->txq_sg_desc->elems;
10885b039241SShannon Nelson desc_addr = frag_addr;
10895b039241SShannon Nelson desc_len = chunk_len;
10905b039241SShannon Nelson } else {
10915b039241SShannon Nelson /* fill sg descriptor */
10925b039241SShannon Nelson elem->addr = cpu_to_le64(frag_addr);
10935b039241SShannon Nelson elem->len = cpu_to_le16(chunk_len);
10940f3154e6SShannon Nelson elem++;
10950f3154e6SShannon Nelson desc_nsge++;
10960f3154e6SShannon Nelson }
10975b039241SShannon Nelson frag_addr += chunk_len;
10985b039241SShannon Nelson frag_rem -= chunk_len;
10995b039241SShannon Nelson tso_rem -= chunk_len;
11005b039241SShannon Nelson seg_rem -= chunk_len;
11010f3154e6SShannon Nelson }
11025b039241SShannon Nelson seg_rem = min(tso_rem, mss);
11035b039241SShannon Nelson done = (tso_rem == 0);
11045b039241SShannon Nelson /* post descriptor */
110540bc471dSShannon Nelson ionic_tx_tso_post(q, desc_info, skb,
11065b039241SShannon Nelson desc_addr, desc_nsge, desc_len,
11075b039241SShannon Nelson hdrlen, mss, outer_csum, vlan_tci, has_vlan,
11085b039241SShannon Nelson start, done);
11095b039241SShannon Nelson start = false;
11102da479caSShannon Nelson /* Buffer information is stored with the first tso descriptor */
11112da479caSShannon Nelson desc_info = &q->info[q->head_idx];
11122da479caSShannon Nelson desc_info->nbufs = 0;
11130f3154e6SShannon Nelson }
11140f3154e6SShannon Nelson
11155b039241SShannon Nelson stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
11165b039241SShannon Nelson stats->bytes += len;
11170f3154e6SShannon Nelson stats->tso++;
11185b039241SShannon Nelson stats->tso_bytes = len;
11190f3154e6SShannon Nelson
11200f3154e6SShannon Nelson return 0;
11210f3154e6SShannon Nelson }
11220f3154e6SShannon Nelson
ionic_tx_calc_csum(struct ionic_queue * q,struct sk_buff * skb,struct ionic_desc_info * desc_info)1123238a0f7cSBrett Creeley static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
11242da479caSShannon Nelson struct ionic_desc_info *desc_info)
11250f3154e6SShannon Nelson {
11262da479caSShannon Nelson struct ionic_txq_desc *desc = desc_info->txq_desc;
11272da479caSShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs;
11280f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
11290f3154e6SShannon Nelson bool has_vlan;
11300f3154e6SShannon Nelson u8 flags = 0;
11310f3154e6SShannon Nelson bool encap;
11320f3154e6SShannon Nelson u64 cmd;
11330f3154e6SShannon Nelson
11340f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb);
11350f3154e6SShannon Nelson encap = skb->encapsulation;
11360f3154e6SShannon Nelson
11370f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
11380f3154e6SShannon Nelson flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
11390f3154e6SShannon Nelson
11400f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
11412da479caSShannon Nelson flags, skb_shinfo(skb)->nr_frags,
11422da479caSShannon Nelson buf_info->dma_addr);
11430f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd);
11442da479caSShannon Nelson desc->len = cpu_to_le16(buf_info->len);
1145f64e0c56SShannon Nelson if (has_vlan) {
1146f64e0c56SShannon Nelson desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1147f64e0c56SShannon Nelson stats->vlan_inserted++;
11482da479caSShannon Nelson } else {
11492da479caSShannon Nelson desc->vlan_tci = 0;
1150f64e0c56SShannon Nelson }
11512da479caSShannon Nelson desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
11522da479caSShannon Nelson desc->csum_offset = cpu_to_le16(skb->csum_offset);
11530f3154e6SShannon Nelson
115440bc471dSShannon Nelson ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
115540bc471dSShannon Nelson
1156fa821170SXin Long if (skb_csum_is_sctp(skb))
11570f3154e6SShannon Nelson stats->crc32_csum++;
11580f3154e6SShannon Nelson else
11590f3154e6SShannon Nelson stats->csum++;
11600f3154e6SShannon Nelson }
11610f3154e6SShannon Nelson
ionic_tx_calc_no_csum(struct ionic_queue * q,struct sk_buff * skb,struct ionic_desc_info * desc_info)1162238a0f7cSBrett Creeley static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
11632da479caSShannon Nelson struct ionic_desc_info *desc_info)
11640f3154e6SShannon Nelson {
11652da479caSShannon Nelson struct ionic_txq_desc *desc = desc_info->txq_desc;
11662da479caSShannon Nelson struct ionic_buf_info *buf_info = desc_info->bufs;
11670f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
11680f3154e6SShannon Nelson bool has_vlan;
11690f3154e6SShannon Nelson u8 flags = 0;
11700f3154e6SShannon Nelson bool encap;
11710f3154e6SShannon Nelson u64 cmd;
11720f3154e6SShannon Nelson
11730f3154e6SShannon Nelson has_vlan = !!skb_vlan_tag_present(skb);
11740f3154e6SShannon Nelson encap = skb->encapsulation;
11750f3154e6SShannon Nelson
11760f3154e6SShannon Nelson flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
11770f3154e6SShannon Nelson flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
11780f3154e6SShannon Nelson
11790f3154e6SShannon Nelson cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
11802da479caSShannon Nelson flags, skb_shinfo(skb)->nr_frags,
11812da479caSShannon Nelson buf_info->dma_addr);
11820f3154e6SShannon Nelson desc->cmd = cpu_to_le64(cmd);
11832da479caSShannon Nelson desc->len = cpu_to_le16(buf_info->len);
1184f64e0c56SShannon Nelson if (has_vlan) {
11850f3154e6SShannon Nelson desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1186f64e0c56SShannon Nelson stats->vlan_inserted++;
11872da479caSShannon Nelson } else {
11882da479caSShannon Nelson desc->vlan_tci = 0;
1189f64e0c56SShannon Nelson }
11902da479caSShannon Nelson desc->csum_start = 0;
11912da479caSShannon Nelson desc->csum_offset = 0;
11920f3154e6SShannon Nelson
119340bc471dSShannon Nelson ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
119440bc471dSShannon Nelson
1195f64e0c56SShannon Nelson stats->csum_none++;
11960f3154e6SShannon Nelson }
11970f3154e6SShannon Nelson
ionic_tx_skb_frags(struct ionic_queue * q,struct sk_buff * skb,struct ionic_desc_info * desc_info)1198238a0f7cSBrett Creeley static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
11992da479caSShannon Nelson struct ionic_desc_info *desc_info)
12000f3154e6SShannon Nelson {
12012da479caSShannon Nelson struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc;
12022da479caSShannon Nelson struct ionic_buf_info *buf_info = &desc_info->bufs[1];
12030f3154e6SShannon Nelson struct ionic_txq_sg_elem *elem = sg_desc->elems;
12040f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
12052da479caSShannon Nelson unsigned int i;
12060f3154e6SShannon Nelson
12072da479caSShannon Nelson for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
12082da479caSShannon Nelson elem->addr = cpu_to_le64(buf_info->dma_addr);
12092da479caSShannon Nelson elem->len = cpu_to_le16(buf_info->len);
12100f3154e6SShannon Nelson }
12110f3154e6SShannon Nelson
12122da479caSShannon Nelson stats->frags += skb_shinfo(skb)->nr_frags;
12130f3154e6SShannon Nelson }
12140f3154e6SShannon Nelson
ionic_tx(struct ionic_queue * q,struct sk_buff * skb)12150f3154e6SShannon Nelson static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
12160f3154e6SShannon Nelson {
12172da479caSShannon Nelson struct ionic_desc_info *desc_info = &q->info[q->head_idx];
12180f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
12190f3154e6SShannon Nelson
12202da479caSShannon Nelson if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
12212da479caSShannon Nelson return -EIO;
12222da479caSShannon Nelson
12230f3154e6SShannon Nelson /* set up the initial descriptor */
12240f3154e6SShannon Nelson if (skb->ip_summed == CHECKSUM_PARTIAL)
1225238a0f7cSBrett Creeley ionic_tx_calc_csum(q, skb, desc_info);
12260f3154e6SShannon Nelson else
1227238a0f7cSBrett Creeley ionic_tx_calc_no_csum(q, skb, desc_info);
12280f3154e6SShannon Nelson
12290f3154e6SShannon Nelson /* add frags */
1230238a0f7cSBrett Creeley ionic_tx_skb_frags(q, skb, desc_info);
12310f3154e6SShannon Nelson
12320f3154e6SShannon Nelson skb_tx_timestamp(skb);
12330f3154e6SShannon Nelson stats->pkts++;
12340f3154e6SShannon Nelson stats->bytes += skb->len;
12350f3154e6SShannon Nelson
1236a8771bfeSShannon Nelson if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
12370f3154e6SShannon Nelson netdev_tx_sent_queue(q_to_ndq(q), skb->len);
12380f3154e6SShannon Nelson ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
12390f3154e6SShannon Nelson
12400f3154e6SShannon Nelson return 0;
12410f3154e6SShannon Nelson }
12420f3154e6SShannon Nelson
ionic_tx_descs_needed(struct ionic_queue * q,struct sk_buff * skb)12430f3154e6SShannon Nelson static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
12440f3154e6SShannon Nelson {
12450f3154e6SShannon Nelson struct ionic_tx_stats *stats = q_to_tx_stats(q);
1246d2c21422SShannon Nelson int ndescs;
12470f3154e6SShannon Nelson int err;
12480f3154e6SShannon Nelson
1249d2c21422SShannon Nelson /* Each desc is mss long max, so a descriptor for each gso_seg */
12500f3154e6SShannon Nelson if (skb_is_gso(skb))
1251d2c21422SShannon Nelson ndescs = skb_shinfo(skb)->gso_segs;
1252d2c21422SShannon Nelson else
1253d2c21422SShannon Nelson ndescs = 1;
12540f3154e6SShannon Nelson
12550f3154e6SShannon Nelson /* If non-TSO, just need 1 desc and nr_frags sg elems */
1256f37bc346SShannon Nelson if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems)
1257d2c21422SShannon Nelson return ndescs;
12580f3154e6SShannon Nelson
12590f3154e6SShannon Nelson /* Too many frags, so linearize */
12600f3154e6SShannon Nelson err = skb_linearize(skb);
12610f3154e6SShannon Nelson if (err)
12620f3154e6SShannon Nelson return err;
12630f3154e6SShannon Nelson
12640f3154e6SShannon Nelson stats->linearize++;
12650f3154e6SShannon Nelson
1266d2c21422SShannon Nelson return ndescs;
12670f3154e6SShannon Nelson }
12680f3154e6SShannon Nelson
ionic_maybe_stop_tx(struct ionic_queue * q,int ndescs)12690f3154e6SShannon Nelson static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
12700f3154e6SShannon Nelson {
12710f3154e6SShannon Nelson int stopped = 0;
12720f3154e6SShannon Nelson
12730f3154e6SShannon Nelson if (unlikely(!ionic_q_has_space(q, ndescs))) {
12740f3154e6SShannon Nelson netif_stop_subqueue(q->lif->netdev, q->index);
12750f3154e6SShannon Nelson stopped = 1;
12760f3154e6SShannon Nelson
12770f3154e6SShannon Nelson /* Might race with ionic_tx_clean, check again */
12780f3154e6SShannon Nelson smp_rmb();
12790f3154e6SShannon Nelson if (ionic_q_has_space(q, ndescs)) {
12800f3154e6SShannon Nelson netif_wake_subqueue(q->lif->netdev, q->index);
12810f3154e6SShannon Nelson stopped = 0;
12820f3154e6SShannon Nelson }
12830f3154e6SShannon Nelson }
12840f3154e6SShannon Nelson
12850f3154e6SShannon Nelson return stopped;
12860f3154e6SShannon Nelson }
12870f3154e6SShannon Nelson
ionic_start_hwstamp_xmit(struct sk_buff * skb,struct net_device * netdev)1288a8771bfeSShannon Nelson static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
1289a8771bfeSShannon Nelson struct net_device *netdev)
1290a8771bfeSShannon Nelson {
1291a8771bfeSShannon Nelson struct ionic_lif *lif = netdev_priv(netdev);
1292a8771bfeSShannon Nelson struct ionic_queue *q = &lif->hwstamp_txq->q;
1293a8771bfeSShannon Nelson int err, ndescs;
1294a8771bfeSShannon Nelson
1295a8771bfeSShannon Nelson /* Does not stop/start txq, because we post to a separate tx queue
1296a8771bfeSShannon Nelson * for timestamping, and if a packet can't be posted immediately to
1297a8771bfeSShannon Nelson * the timestamping queue, it is dropped.
1298a8771bfeSShannon Nelson */
1299a8771bfeSShannon Nelson
1300a8771bfeSShannon Nelson ndescs = ionic_tx_descs_needed(q, skb);
1301a8771bfeSShannon Nelson if (unlikely(ndescs < 0))
1302a8771bfeSShannon Nelson goto err_out_drop;
1303a8771bfeSShannon Nelson
1304a8771bfeSShannon Nelson if (unlikely(!ionic_q_has_space(q, ndescs)))
1305a8771bfeSShannon Nelson goto err_out_drop;
1306a8771bfeSShannon Nelson
1307bd7856bcSShannon Nelson skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
1308a8771bfeSShannon Nelson if (skb_is_gso(skb))
1309a8771bfeSShannon Nelson err = ionic_tx_tso(q, skb);
1310a8771bfeSShannon Nelson else
1311a8771bfeSShannon Nelson err = ionic_tx(q, skb);
1312a8771bfeSShannon Nelson
1313a8771bfeSShannon Nelson if (err)
1314a8771bfeSShannon Nelson goto err_out_drop;
1315a8771bfeSShannon Nelson
1316a8771bfeSShannon Nelson return NETDEV_TX_OK;
1317a8771bfeSShannon Nelson
1318a8771bfeSShannon Nelson err_out_drop:
1319a8771bfeSShannon Nelson q->drop++;
1320a8771bfeSShannon Nelson dev_kfree_skb(skb);
1321a8771bfeSShannon Nelson return NETDEV_TX_OK;
1322a8771bfeSShannon Nelson }
1323a8771bfeSShannon Nelson
ionic_start_xmit(struct sk_buff * skb,struct net_device * netdev)13240f3154e6SShannon Nelson netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
13250f3154e6SShannon Nelson {
13260f3154e6SShannon Nelson u16 queue_index = skb_get_queue_mapping(skb);
13270f3154e6SShannon Nelson struct ionic_lif *lif = netdev_priv(netdev);
13280f3154e6SShannon Nelson struct ionic_queue *q;
13290f3154e6SShannon Nelson int ndescs;
13300f3154e6SShannon Nelson int err;
13310f3154e6SShannon Nelson
1332c6d3d73aSShannon Nelson if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
13330f3154e6SShannon Nelson dev_kfree_skb(skb);
13340f3154e6SShannon Nelson return NETDEV_TX_OK;
13350f3154e6SShannon Nelson }
13360f3154e6SShannon Nelson
1337a8771bfeSShannon Nelson if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1338e2ce148eSShannon Nelson if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
1339a8771bfeSShannon Nelson return ionic_start_hwstamp_xmit(skb, netdev);
1340a8771bfeSShannon Nelson
134134dec947SShannon Nelson if (unlikely(queue_index >= lif->nxqs))
13420f3154e6SShannon Nelson queue_index = 0;
134334dec947SShannon Nelson q = &lif->txqcqs[queue_index]->q;
13440f3154e6SShannon Nelson
13450f3154e6SShannon Nelson ndescs = ionic_tx_descs_needed(q, skb);
13460f3154e6SShannon Nelson if (ndescs < 0)
13470f3154e6SShannon Nelson goto err_out_drop;
13480f3154e6SShannon Nelson
13490f3154e6SShannon Nelson if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
13500f3154e6SShannon Nelson return NETDEV_TX_BUSY;
13510f3154e6SShannon Nelson
13520f3154e6SShannon Nelson if (skb_is_gso(skb))
13530f3154e6SShannon Nelson err = ionic_tx_tso(q, skb);
13540f3154e6SShannon Nelson else
13550f3154e6SShannon Nelson err = ionic_tx(q, skb);
13560f3154e6SShannon Nelson
13570f3154e6SShannon Nelson if (err)
13580f3154e6SShannon Nelson goto err_out_drop;
13590f3154e6SShannon Nelson
13600f3154e6SShannon Nelson /* Stop the queue if there aren't descriptors for the next packet.
13610f3154e6SShannon Nelson * Since our SG lists per descriptor take care of most of the possible
13620f3154e6SShannon Nelson * fragmentation, we don't need to have many descriptors available.
13630f3154e6SShannon Nelson */
13640f3154e6SShannon Nelson ionic_maybe_stop_tx(q, 4);
13650f3154e6SShannon Nelson
13660f3154e6SShannon Nelson return NETDEV_TX_OK;
13670f3154e6SShannon Nelson
13680f3154e6SShannon Nelson err_out_drop:
13690f3154e6SShannon Nelson q->drop++;
13700f3154e6SShannon Nelson dev_kfree_skb(skb);
13710f3154e6SShannon Nelson return NETDEV_TX_OK;
13720f3154e6SShannon Nelson }
1373