1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2874aeea5SJeff Kirsher /****************************************************************************
3f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards
4874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd.
5f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc.
6874aeea5SJeff Kirsher */
7874aeea5SJeff Kirsher
8874aeea5SJeff Kirsher #include <linux/pci.h>
9874aeea5SJeff Kirsher #include <linux/tcp.h>
10874aeea5SJeff Kirsher #include <linux/ip.h>
11874aeea5SJeff Kirsher #include <linux/in.h>
12874aeea5SJeff Kirsher #include <linux/ipv6.h>
13874aeea5SJeff Kirsher #include <linux/slab.h>
14874aeea5SJeff Kirsher #include <net/ipv6.h>
15874aeea5SJeff Kirsher #include <linux/if_ether.h>
16874aeea5SJeff Kirsher #include <linux/highmem.h>
17183233beSBen Hutchings #include <linux/cache.h>
18874aeea5SJeff Kirsher #include "net_driver.h"
19874aeea5SJeff Kirsher #include "efx.h"
20183233beSBen Hutchings #include "io.h"
21874aeea5SJeff Kirsher #include "nic.h"
22e9117e50SBert Kenward #include "tx.h"
23e1253f39SAlex Maftei (amaftei) #include "tx_common.h"
24874aeea5SJeff Kirsher #include "workarounds.h"
25dfa50be9SBen Hutchings #include "ef10_regs.h"
26874aeea5SJeff Kirsher
27183233beSBen Hutchings #ifdef EFX_USE_PIO
28183233beSBen Hutchings
29183233beSBen Hutchings #define EFX_PIOBUF_SIZE_DEF ALIGN(256, L1_CACHE_BYTES)
30183233beSBen Hutchings unsigned int efx_piobuf_size __read_mostly = EFX_PIOBUF_SIZE_DEF;
31183233beSBen Hutchings
32183233beSBen Hutchings #endif /* EFX_USE_PIO */
33183233beSBen Hutchings
efx_tx_get_copy_buffer(struct efx_tx_queue * tx_queue,struct efx_tx_buffer * buffer)34e9117e50SBert Kenward static inline u8 *efx_tx_get_copy_buffer(struct efx_tx_queue *tx_queue,
35e9117e50SBert Kenward struct efx_tx_buffer *buffer)
360fe5565bSBen Hutchings {
37e9117e50SBert Kenward unsigned int index = efx_tx_queue_get_insert_index(tx_queue);
38e9117e50SBert Kenward struct efx_buffer *page_buf =
39e9117e50SBert Kenward &tx_queue->cb_page[index >> (PAGE_SHIFT - EFX_TX_CB_ORDER)];
40e9117e50SBert Kenward unsigned int offset =
41e9117e50SBert Kenward ((index << EFX_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1);
42e9117e50SBert Kenward
43e9117e50SBert Kenward if (unlikely(!page_buf->addr) &&
44e9117e50SBert Kenward efx_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE,
45e9117e50SBert Kenward GFP_ATOMIC))
46e9117e50SBert Kenward return NULL;
47e9117e50SBert Kenward buffer->dma_addr = page_buf->dma_addr + offset;
48e9117e50SBert Kenward buffer->unmap_len = 0;
49e9117e50SBert Kenward return (u8 *)page_buf->addr + offset;
500fe5565bSBen Hutchings }
510fe5565bSBen Hutchings
efx_tx_get_copy_buffer_limited(struct efx_tx_queue * tx_queue,struct efx_tx_buffer * buffer,size_t len)52e9117e50SBert Kenward u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue,
53e9117e50SBert Kenward struct efx_tx_buffer *buffer, size_t len)
540fe5565bSBen Hutchings {
55e9117e50SBert Kenward if (len > EFX_TX_CB_SIZE)
56e9117e50SBert Kenward return NULL;
57e9117e50SBert Kenward return efx_tx_get_copy_buffer(tx_queue, buffer);
580fe5565bSBen Hutchings }
590fe5565bSBen Hutchings
efx_tx_maybe_stop_queue(struct efx_tx_queue * txq1)6014bf718fSBen Hutchings static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1)
6114bf718fSBen Hutchings {
620d8c1229SEdward Cree /* We need to consider all queues that the net core sees as one */
6314bf718fSBen Hutchings struct efx_nic *efx = txq1->efx;
640d8c1229SEdward Cree struct efx_tx_queue *txq2;
6514bf718fSBen Hutchings unsigned int fill_level;
6614bf718fSBen Hutchings
670d8c1229SEdward Cree fill_level = efx_channel_tx_old_fill_level(txq1->channel);
6814bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh))
6914bf718fSBen Hutchings return;
7014bf718fSBen Hutchings
7114bf718fSBen Hutchings /* We used the stale old_read_count above, which gives us a
7214bf718fSBen Hutchings * pessimistic estimate of the fill level (which may even
7314bf718fSBen Hutchings * validly be >= efx->txq_entries). Now try again using
7414bf718fSBen Hutchings * read_count (more likely to be a cache miss).
7514bf718fSBen Hutchings *
7614bf718fSBen Hutchings * If we read read_count and then conditionally stop the
7714bf718fSBen Hutchings * queue, it is possible for the completion path to race with
7814bf718fSBen Hutchings * us and complete all outstanding descriptors in the middle,
7914bf718fSBen Hutchings * after which there will be no more completions to wake it.
8014bf718fSBen Hutchings * Therefore we stop the queue first, then read read_count
8114bf718fSBen Hutchings * (with a memory barrier to ensure the ordering), then
8214bf718fSBen Hutchings * restart the queue if the fill level turns out to be low
8314bf718fSBen Hutchings * enough.
8414bf718fSBen Hutchings */
8514bf718fSBen Hutchings netif_tx_stop_queue(txq1->core_txq);
8614bf718fSBen Hutchings smp_mb();
870d8c1229SEdward Cree efx_for_each_channel_tx_queue(txq2, txq1->channel)
886aa7de05SMark Rutland txq2->old_read_count = READ_ONCE(txq2->read_count);
8914bf718fSBen Hutchings
900d8c1229SEdward Cree fill_level = efx_channel_tx_old_fill_level(txq1->channel);
91e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(fill_level >= efx->txq_entries);
9214bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) {
9314bf718fSBen Hutchings smp_mb();
9414bf718fSBen Hutchings if (likely(!efx->loopback_selftest))
9514bf718fSBen Hutchings netif_tx_start_queue(txq1->core_txq);
9614bf718fSBen Hutchings }
9714bf718fSBen Hutchings }
9814bf718fSBen Hutchings
efx_enqueue_skb_copy(struct efx_tx_queue * tx_queue,struct sk_buff * skb)99e9117e50SBert Kenward static int efx_enqueue_skb_copy(struct efx_tx_queue *tx_queue,
100e9117e50SBert Kenward struct sk_buff *skb)
101e9117e50SBert Kenward {
102e9117e50SBert Kenward unsigned int copy_len = skb->len;
103e9117e50SBert Kenward struct efx_tx_buffer *buffer;
104e9117e50SBert Kenward u8 *copy_buffer;
105e9117e50SBert Kenward int rc;
106e9117e50SBert Kenward
107e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(copy_len > EFX_TX_CB_SIZE);
108e9117e50SBert Kenward
109e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue);
110e9117e50SBert Kenward
111e9117e50SBert Kenward copy_buffer = efx_tx_get_copy_buffer(tx_queue, buffer);
112e9117e50SBert Kenward if (unlikely(!copy_buffer))
113e9117e50SBert Kenward return -ENOMEM;
114e9117e50SBert Kenward
115e9117e50SBert Kenward rc = skb_copy_bits(skb, 0, copy_buffer, copy_len);
116e9117e50SBert Kenward EFX_WARN_ON_PARANOID(rc);
117e9117e50SBert Kenward buffer->len = copy_len;
118e9117e50SBert Kenward
119e9117e50SBert Kenward buffer->skb = skb;
120e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB;
121e9117e50SBert Kenward
122e9117e50SBert Kenward ++tx_queue->insert_count;
123e9117e50SBert Kenward return rc;
124e9117e50SBert Kenward }
125e9117e50SBert Kenward
126ee45fd92SJon Cooper #ifdef EFX_USE_PIO
127ee45fd92SJon Cooper
128ee45fd92SJon Cooper struct efx_short_copy_buffer {
129ee45fd92SJon Cooper int used;
130ee45fd92SJon Cooper u8 buf[L1_CACHE_BYTES];
131ee45fd92SJon Cooper };
132ee45fd92SJon Cooper
133ee45fd92SJon Cooper /* Copy to PIO, respecting that writes to PIO buffers must be dword aligned.
134ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer.
135ee45fd92SJon Cooper */
efx_memcpy_toio_aligned(struct efx_nic * efx,u8 __iomem ** piobuf,u8 * data,int len,struct efx_short_copy_buffer * copy_buf)136ee45fd92SJon Cooper static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf,
137ee45fd92SJon Cooper u8 *data, int len,
138ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf)
139ee45fd92SJon Cooper {
140ee45fd92SJon Cooper int block_len = len & ~(sizeof(copy_buf->buf) - 1);
141ee45fd92SJon Cooper
1424984c237SBen Hutchings __iowrite64_copy(*piobuf, data, block_len >> 3);
143ee45fd92SJon Cooper *piobuf += block_len;
144ee45fd92SJon Cooper len -= block_len;
145ee45fd92SJon Cooper
146ee45fd92SJon Cooper if (len) {
147ee45fd92SJon Cooper data += block_len;
148ee45fd92SJon Cooper BUG_ON(copy_buf->used);
149ee45fd92SJon Cooper BUG_ON(len > sizeof(copy_buf->buf));
150ee45fd92SJon Cooper memcpy(copy_buf->buf, data, len);
151ee45fd92SJon Cooper copy_buf->used = len;
152ee45fd92SJon Cooper }
153ee45fd92SJon Cooper }
154ee45fd92SJon Cooper
155ee45fd92SJon Cooper /* Copy to PIO, respecting dword alignment, popping data from copy buffer first.
156ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer.
157ee45fd92SJon Cooper */
efx_memcpy_toio_aligned_cb(struct efx_nic * efx,u8 __iomem ** piobuf,u8 * data,int len,struct efx_short_copy_buffer * copy_buf)158ee45fd92SJon Cooper static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf,
159ee45fd92SJon Cooper u8 *data, int len,
160ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf)
161ee45fd92SJon Cooper {
162ee45fd92SJon Cooper if (copy_buf->used) {
163ee45fd92SJon Cooper /* if the copy buffer is partially full, fill it up and write */
164ee45fd92SJon Cooper int copy_to_buf =
165ee45fd92SJon Cooper min_t(int, sizeof(copy_buf->buf) - copy_buf->used, len);
166ee45fd92SJon Cooper
167ee45fd92SJon Cooper memcpy(copy_buf->buf + copy_buf->used, data, copy_to_buf);
168ee45fd92SJon Cooper copy_buf->used += copy_to_buf;
169ee45fd92SJon Cooper
170ee45fd92SJon Cooper /* if we didn't fill it up then we're done for now */
171ee45fd92SJon Cooper if (copy_buf->used < sizeof(copy_buf->buf))
172ee45fd92SJon Cooper return;
173ee45fd92SJon Cooper
1744984c237SBen Hutchings __iowrite64_copy(*piobuf, copy_buf->buf,
1754984c237SBen Hutchings sizeof(copy_buf->buf) >> 3);
176ee45fd92SJon Cooper *piobuf += sizeof(copy_buf->buf);
177ee45fd92SJon Cooper data += copy_to_buf;
178ee45fd92SJon Cooper len -= copy_to_buf;
179ee45fd92SJon Cooper copy_buf->used = 0;
180ee45fd92SJon Cooper }
181ee45fd92SJon Cooper
182ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, data, len, copy_buf);
183ee45fd92SJon Cooper }
184ee45fd92SJon Cooper
efx_flush_copy_buffer(struct efx_nic * efx,u8 __iomem * piobuf,struct efx_short_copy_buffer * copy_buf)185ee45fd92SJon Cooper static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf,
186ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf)
187ee45fd92SJon Cooper {
188ee45fd92SJon Cooper /* if there's anything in it, write the whole buffer, including junk */
189ee45fd92SJon Cooper if (copy_buf->used)
1904984c237SBen Hutchings __iowrite64_copy(piobuf, copy_buf->buf,
1914984c237SBen Hutchings sizeof(copy_buf->buf) >> 3);
192ee45fd92SJon Cooper }
193ee45fd92SJon Cooper
194ee45fd92SJon Cooper /* Traverse skb structure and copy fragments in to PIO buffer.
195ee45fd92SJon Cooper * Advances piobuf pointer.
196ee45fd92SJon Cooper */
efx_skb_copy_bits_to_pio(struct efx_nic * efx,struct sk_buff * skb,u8 __iomem ** piobuf,struct efx_short_copy_buffer * copy_buf)197ee45fd92SJon Cooper static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb,
198ee45fd92SJon Cooper u8 __iomem **piobuf,
199ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf)
200ee45fd92SJon Cooper {
201ee45fd92SJon Cooper int i;
202ee45fd92SJon Cooper
203ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, skb->data, skb_headlen(skb),
204ee45fd92SJon Cooper copy_buf);
205ee45fd92SJon Cooper
206ee45fd92SJon Cooper for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
207ee45fd92SJon Cooper skb_frag_t *f = &skb_shinfo(skb)->frags[i];
208ee45fd92SJon Cooper u8 *vaddr;
209ee45fd92SJon Cooper
210f61e6d3cSAnirudh Venkataramanan vaddr = kmap_local_page(skb_frag_page(f));
211ee45fd92SJon Cooper
212b54c9d5bSJonathan Lemon efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f),
213ee45fd92SJon Cooper skb_frag_size(f), copy_buf);
214f61e6d3cSAnirudh Venkataramanan kunmap_local(vaddr);
215ee45fd92SJon Cooper }
216ee45fd92SJon Cooper
217e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list);
218ee45fd92SJon Cooper }
219ee45fd92SJon Cooper
efx_enqueue_skb_pio(struct efx_tx_queue * tx_queue,struct sk_buff * skb)220e9117e50SBert Kenward static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue,
221e9117e50SBert Kenward struct sk_buff *skb)
222ee45fd92SJon Cooper {
223ee45fd92SJon Cooper struct efx_tx_buffer *buffer =
224ee45fd92SJon Cooper efx_tx_queue_get_insert_buffer(tx_queue);
225ee45fd92SJon Cooper u8 __iomem *piobuf = tx_queue->piobuf;
226ee45fd92SJon Cooper
227ee45fd92SJon Cooper /* Copy to PIO buffer. Ensure the writes are padded to the end
228ee45fd92SJon Cooper * of a cache line, as this is required for write-combining to be
229ee45fd92SJon Cooper * effective on at least x86.
230ee45fd92SJon Cooper */
231ee45fd92SJon Cooper
232ee45fd92SJon Cooper if (skb_shinfo(skb)->nr_frags) {
233ee45fd92SJon Cooper /* The size of the copy buffer will ensure all writes
234ee45fd92SJon Cooper * are the size of a cache line.
235ee45fd92SJon Cooper */
236ee45fd92SJon Cooper struct efx_short_copy_buffer copy_buf;
237ee45fd92SJon Cooper
238ee45fd92SJon Cooper copy_buf.used = 0;
239ee45fd92SJon Cooper
240ee45fd92SJon Cooper efx_skb_copy_bits_to_pio(tx_queue->efx, skb,
241ee45fd92SJon Cooper &piobuf, ©_buf);
242ee45fd92SJon Cooper efx_flush_copy_buffer(tx_queue->efx, piobuf, ©_buf);
243ee45fd92SJon Cooper } else {
244ee45fd92SJon Cooper /* Pad the write to the size of a cache line.
245e9117e50SBert Kenward * We can do this because we know the skb_shared_info struct is
246ee45fd92SJon Cooper * after the source, and the destination buffer is big enough.
247ee45fd92SJon Cooper */
248ee45fd92SJon Cooper BUILD_BUG_ON(L1_CACHE_BYTES >
249ee45fd92SJon Cooper SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
2504984c237SBen Hutchings __iowrite64_copy(tx_queue->piobuf, skb->data,
2514984c237SBen Hutchings ALIGN(skb->len, L1_CACHE_BYTES) >> 3);
252ee45fd92SJon Cooper }
253ee45fd92SJon Cooper
254e9117e50SBert Kenward buffer->skb = skb;
255e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | EFX_TX_BUF_OPTION;
256e9117e50SBert Kenward
257ee45fd92SJon Cooper EFX_POPULATE_QWORD_5(buffer->option,
258ee45fd92SJon Cooper ESF_DZ_TX_DESC_IS_OPT, 1,
259ee45fd92SJon Cooper ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_PIO,
260ee45fd92SJon Cooper ESF_DZ_TX_PIO_CONT, 0,
261ee45fd92SJon Cooper ESF_DZ_TX_PIO_BYTE_CNT, skb->len,
262ee45fd92SJon Cooper ESF_DZ_TX_PIO_BUF_ADDR,
263ee45fd92SJon Cooper tx_queue->piobuf_offset);
264ee45fd92SJon Cooper ++tx_queue->insert_count;
265e9117e50SBert Kenward return 0;
266ee45fd92SJon Cooper }
2678be41842SEdward Cree
2688be41842SEdward Cree /* Decide whether we can use TX PIO, ie. write packet data directly into
2698be41842SEdward Cree * a buffer on the device. This can reduce latency at the expense of
2708be41842SEdward Cree * throughput, so we only do this if both hardware and software TX rings
2718be41842SEdward Cree * are empty, including all queues for the channel. This also ensures that
2728be41842SEdward Cree * only one packet at a time can be using the PIO buffer. If the xmit_more
2738be41842SEdward Cree * flag is set then we don't use this - there'll be another packet along
2748be41842SEdward Cree * shortly and we want to hold off the doorbell.
2758be41842SEdward Cree */
efx_tx_may_pio(struct efx_tx_queue * tx_queue)2768be41842SEdward Cree static bool efx_tx_may_pio(struct efx_tx_queue *tx_queue)
2778be41842SEdward Cree {
2788be41842SEdward Cree struct efx_channel *channel = tx_queue->channel;
2798be41842SEdward Cree
2808be41842SEdward Cree if (!tx_queue->piobuf)
2818be41842SEdward Cree return false;
2828be41842SEdward Cree
2838be41842SEdward Cree EFX_WARN_ON_ONCE_PARANOID(!channel->efx->type->option_descriptors);
2848be41842SEdward Cree
2858be41842SEdward Cree efx_for_each_channel_tx_queue(tx_queue, channel)
2868be41842SEdward Cree if (!efx_nic_tx_is_empty(tx_queue, tx_queue->packet_write_count))
2878be41842SEdward Cree return false;
2888be41842SEdward Cree
2898be41842SEdward Cree return true;
2908be41842SEdward Cree }
291ee45fd92SJon Cooper #endif /* EFX_USE_PIO */
292ee45fd92SJon Cooper
2931c0544d2SEdward Cree /* Send any pending traffic for a channel. xmit_more is shared across all
2941c0544d2SEdward Cree * queues for a channel, so we must check all of them.
2951c0544d2SEdward Cree */
efx_tx_send_pending(struct efx_channel * channel)2961c0544d2SEdward Cree static void efx_tx_send_pending(struct efx_channel *channel)
2971c0544d2SEdward Cree {
2981c0544d2SEdward Cree struct efx_tx_queue *q;
2991c0544d2SEdward Cree
3001c0544d2SEdward Cree efx_for_each_channel_tx_queue(q, channel) {
3011c0544d2SEdward Cree if (q->xmit_pending)
3021c0544d2SEdward Cree efx_nic_push_buffers(q);
3031c0544d2SEdward Cree }
3041c0544d2SEdward Cree }
3051c0544d2SEdward Cree
30646d1efd8SEdward Cree /*
307874aeea5SJeff Kirsher * Add a socket buffer to a TX queue
308874aeea5SJeff Kirsher *
309874aeea5SJeff Kirsher * This maps all fragments of a socket buffer for DMA and adds them to
310874aeea5SJeff Kirsher * the TX queue. The queue's insert pointer will be incremented by
311874aeea5SJeff Kirsher * the number of fragments in the socket buffer.
312874aeea5SJeff Kirsher *
313874aeea5SJeff Kirsher * If any DMA mapping fails, any mapped fragments will be unmapped,
314874aeea5SJeff Kirsher * the queue's insert pointer will be restored to its original value.
315874aeea5SJeff Kirsher *
316874aeea5SJeff Kirsher * This function is split out from efx_hard_start_xmit to allow the
317874aeea5SJeff Kirsher * loopback test to direct packets via specific TX queues.
318874aeea5SJeff Kirsher *
31914bf718fSBen Hutchings * Returns NETDEV_TX_OK.
320874aeea5SJeff Kirsher * You must hold netif_tx_lock() to call this function.
321874aeea5SJeff Kirsher */
__efx_enqueue_skb(struct efx_tx_queue * tx_queue,struct sk_buff * skb)32251b35a45SEdward Cree netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
323874aeea5SJeff Kirsher {
3240c235113SMartin Habets unsigned int old_insert_count = tx_queue->insert_count;
325f79c957aSFlorian Westphal bool xmit_more = netdev_xmit_more();
326e9117e50SBert Kenward bool data_mapped = false;
327e9117e50SBert Kenward unsigned int segments;
328e9117e50SBert Kenward unsigned int skb_len;
32946d1efd8SEdward Cree int rc;
330874aeea5SJeff Kirsher
331e9117e50SBert Kenward skb_len = skb->len;
332e9117e50SBert Kenward segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0;
333e9117e50SBert Kenward if (segments == 1)
334e9117e50SBert Kenward segments = 0; /* Don't use TSO for a single segment. */
335874aeea5SJeff Kirsher
336e9117e50SBert Kenward /* Handle TSO first - it's *possible* (although unlikely) that we might
337e9117e50SBert Kenward * be passed a packet to segment that's smaller than the copybreak/PIO
338e9117e50SBert Kenward * size limit.
339874aeea5SJeff Kirsher */
340e9117e50SBert Kenward if (segments) {
3411679c72cSEdward Cree switch (tx_queue->tso_version) {
3421679c72cSEdward Cree case 1:
3431679c72cSEdward Cree rc = efx_enqueue_skb_tso(tx_queue, skb, &data_mapped);
3441679c72cSEdward Cree break;
3451679c72cSEdward Cree case 2:
3461679c72cSEdward Cree rc = efx_ef10_tx_tso_desc(tx_queue, skb, &data_mapped);
3471679c72cSEdward Cree break;
3481679c72cSEdward Cree case 0: /* No TSO on this queue, SW fallback needed */
3491679c72cSEdward Cree default:
3501679c72cSEdward Cree rc = -EINVAL;
3511679c72cSEdward Cree break;
3521679c72cSEdward Cree }
35346d1efd8SEdward Cree if (rc == -EINVAL) {
35446d1efd8SEdward Cree rc = efx_tx_tso_fallback(tx_queue, skb);
35546d1efd8SEdward Cree tx_queue->tso_fallbacks++;
35646d1efd8SEdward Cree if (rc == 0)
35746d1efd8SEdward Cree return 0;
35846d1efd8SEdward Cree }
35946d1efd8SEdward Cree if (rc)
360e9117e50SBert Kenward goto err;
361e9117e50SBert Kenward #ifdef EFX_USE_PIO
362f79c957aSFlorian Westphal } else if (skb_len <= efx_piobuf_size && !xmit_more &&
3638be41842SEdward Cree efx_tx_may_pio(tx_queue)) {
364e9117e50SBert Kenward /* Use PIO for short packets with an empty queue. */
365e9117e50SBert Kenward if (efx_enqueue_skb_pio(tx_queue, skb))
366e9117e50SBert Kenward goto err;
367e9117e50SBert Kenward tx_queue->pio_packets++;
368e9117e50SBert Kenward data_mapped = true;
369e9117e50SBert Kenward #endif
3705a6681e2SEdward Cree } else if (skb->data_len && skb_len <= EFX_TX_CB_SIZE) {
371e9117e50SBert Kenward /* Pad short packets or coalesce short fragmented packets. */
372e9117e50SBert Kenward if (efx_enqueue_skb_copy(tx_queue, skb))
373e9117e50SBert Kenward goto err;
374e9117e50SBert Kenward tx_queue->cb_packets++;
375e9117e50SBert Kenward data_mapped = true;
376874aeea5SJeff Kirsher }
377874aeea5SJeff Kirsher
378e9117e50SBert Kenward /* Map for DMA and create descriptors if we haven't done so already. */
379e9117e50SBert Kenward if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments)))
380e9117e50SBert Kenward goto err;
381874aeea5SJeff Kirsher
3820c235113SMartin Habets efx_tx_maybe_stop_queue(tx_queue);
3830c235113SMartin Habets
3841c0544d2SEdward Cree tx_queue->xmit_pending = true;
3851c0544d2SEdward Cree
386874aeea5SJeff Kirsher /* Pass off to hardware */
3871c0544d2SEdward Cree if (__netdev_tx_sent_queue(tx_queue->core_txq, skb_len, xmit_more))
3881c0544d2SEdward Cree efx_tx_send_pending(tx_queue->channel);
389874aeea5SJeff Kirsher
390e9117e50SBert Kenward if (segments) {
391e9117e50SBert Kenward tx_queue->tso_bursts++;
392e9117e50SBert Kenward tx_queue->tso_packets += segments;
393e9117e50SBert Kenward tx_queue->tx_packets += segments;
394e9117e50SBert Kenward } else {
3958ccf3800SAndrew Rybchenko tx_queue->tx_packets++;
396e9117e50SBert Kenward }
397e9117e50SBert Kenward
398874aeea5SJeff Kirsher return NETDEV_TX_OK;
399874aeea5SJeff Kirsher
400874aeea5SJeff Kirsher
401e9117e50SBert Kenward err:
4020c235113SMartin Habets efx_enqueue_unwind(tx_queue, old_insert_count);
403874aeea5SJeff Kirsher dev_kfree_skb_any(skb);
4040c235113SMartin Habets
4050c235113SMartin Habets /* If we're not expecting another transmit and we had something to push
4060c235113SMartin Habets * on this queue or a partner queue then we need to push here to get the
4070c235113SMartin Habets * previous packets out.
4080c235113SMartin Habets */
4091c0544d2SEdward Cree if (!xmit_more)
4101c0544d2SEdward Cree efx_tx_send_pending(tx_queue->channel);
4110c235113SMartin Habets
41214bf718fSBen Hutchings return NETDEV_TX_OK;
413874aeea5SJeff Kirsher }
414874aeea5SJeff Kirsher
415dfe44c1fSCharles McLachlan /* Transmit a packet from an XDP buffer
416dfe44c1fSCharles McLachlan *
417dfe44c1fSCharles McLachlan * Returns number of packets sent on success, error code otherwise.
418dfe44c1fSCharles McLachlan * Runs in NAPI context, either in our poll (for XDP TX) or a different NIC
419dfe44c1fSCharles McLachlan * (for XDP redirect).
420dfe44c1fSCharles McLachlan */
efx_xdp_tx_buffers(struct efx_nic * efx,int n,struct xdp_frame ** xdpfs,bool flush)421dfe44c1fSCharles McLachlan int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs,
422dfe44c1fSCharles McLachlan bool flush)
423dfe44c1fSCharles McLachlan {
424dfe44c1fSCharles McLachlan struct efx_tx_buffer *tx_buffer;
425dfe44c1fSCharles McLachlan struct efx_tx_queue *tx_queue;
426dfe44c1fSCharles McLachlan struct xdp_frame *xdpf;
427dfe44c1fSCharles McLachlan dma_addr_t dma_addr;
428dfe44c1fSCharles McLachlan unsigned int len;
429dfe44c1fSCharles McLachlan int space;
430dfe44c1fSCharles McLachlan int cpu;
4316215b608SÍñigo Huguet int i = 0;
432dfe44c1fSCharles McLachlan
43341544618SÍñigo Huguet if (unlikely(n && !xdpfs))
43441544618SÍñigo Huguet return -EINVAL;
43541544618SÍñigo Huguet if (unlikely(!n))
43641544618SÍñigo Huguet return 0;
437dfe44c1fSCharles McLachlan
43841544618SÍñigo Huguet cpu = raw_smp_processor_id();
43941544618SÍñigo Huguet if (unlikely(cpu >= efx->xdp_tx_queue_count))
440dfe44c1fSCharles McLachlan return -EINVAL;
441dfe44c1fSCharles McLachlan
442dfe44c1fSCharles McLachlan tx_queue = efx->xdp_tx_queues[cpu];
443dfe44c1fSCharles McLachlan if (unlikely(!tx_queue))
444dfe44c1fSCharles McLachlan return -EINVAL;
445dfe44c1fSCharles McLachlan
446fb5833d8STaehee Yoo if (!tx_queue->initialised)
447fb5833d8STaehee Yoo return -EINVAL;
448fb5833d8STaehee Yoo
44941544618SÍñigo Huguet if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED)
45041544618SÍñigo Huguet HARD_TX_LOCK(efx->net_dev, tx_queue->core_txq, cpu);
451dfe44c1fSCharles McLachlan
4526215b608SÍñigo Huguet /* If we're borrowing net stack queues we have to handle stop-restart
4536215b608SÍñigo Huguet * or we might block the queue and it will be considered as frozen
4546215b608SÍñigo Huguet */
4556215b608SÍñigo Huguet if (efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_BORROWED) {
4566215b608SÍñigo Huguet if (netif_tx_queue_stopped(tx_queue->core_txq))
4576215b608SÍñigo Huguet goto unlock;
4586215b608SÍñigo Huguet efx_tx_maybe_stop_queue(tx_queue);
4596215b608SÍñigo Huguet }
4606215b608SÍñigo Huguet
461dfe44c1fSCharles McLachlan /* Check for available space. We should never need multiple
462dfe44c1fSCharles McLachlan * descriptors per frame.
463dfe44c1fSCharles McLachlan */
464dfe44c1fSCharles McLachlan space = efx->txq_entries +
465dfe44c1fSCharles McLachlan tx_queue->read_count - tx_queue->insert_count;
466dfe44c1fSCharles McLachlan
467dfe44c1fSCharles McLachlan for (i = 0; i < n; i++) {
468dfe44c1fSCharles McLachlan xdpf = xdpfs[i];
469dfe44c1fSCharles McLachlan
470dfe44c1fSCharles McLachlan if (i >= space)
471dfe44c1fSCharles McLachlan break;
472dfe44c1fSCharles McLachlan
473dfe44c1fSCharles McLachlan /* We'll want a descriptor for this tx. */
474dfe44c1fSCharles McLachlan prefetchw(__efx_tx_queue_get_insert_buffer(tx_queue));
475dfe44c1fSCharles McLachlan
476dfe44c1fSCharles McLachlan len = xdpf->len;
477dfe44c1fSCharles McLachlan
478dfe44c1fSCharles McLachlan /* Map for DMA. */
479dfe44c1fSCharles McLachlan dma_addr = dma_map_single(&efx->pci_dev->dev,
480dfe44c1fSCharles McLachlan xdpf->data, len,
481dfe44c1fSCharles McLachlan DMA_TO_DEVICE);
482dfe44c1fSCharles McLachlan if (dma_mapping_error(&efx->pci_dev->dev, dma_addr))
483dfe44c1fSCharles McLachlan break;
484dfe44c1fSCharles McLachlan
485dfe44c1fSCharles McLachlan /* Create descriptor and set up for unmapping DMA. */
486dfe44c1fSCharles McLachlan tx_buffer = efx_tx_map_chunk(tx_queue, dma_addr, len);
487dfe44c1fSCharles McLachlan tx_buffer->xdpf = xdpf;
488dfe44c1fSCharles McLachlan tx_buffer->flags = EFX_TX_BUF_XDP |
489dfe44c1fSCharles McLachlan EFX_TX_BUF_MAP_SINGLE;
490dfe44c1fSCharles McLachlan tx_buffer->dma_offset = 0;
491dfe44c1fSCharles McLachlan tx_buffer->unmap_len = len;
492dfe44c1fSCharles McLachlan tx_queue->tx_packets++;
493dfe44c1fSCharles McLachlan }
494dfe44c1fSCharles McLachlan
495dfe44c1fSCharles McLachlan /* Pass mapped frames to hardware. */
496dfe44c1fSCharles McLachlan if (flush && i > 0)
497dfe44c1fSCharles McLachlan efx_nic_push_buffers(tx_queue);
498dfe44c1fSCharles McLachlan
4996215b608SÍñigo Huguet unlock:
50041544618SÍñigo Huguet if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED)
50141544618SÍñigo Huguet HARD_TX_UNLOCK(efx->net_dev, tx_queue->core_txq);
50241544618SÍñigo Huguet
503fdc13979SLorenzo Bianconi return i == 0 ? -EIO : i;
504dfe44c1fSCharles McLachlan }
505dfe44c1fSCharles McLachlan
506874aeea5SJeff Kirsher /* Initiate a packet transmission. We use one channel per CPU
50712804793SEdward Cree * (sharing when we have more CPUs than channels).
508874aeea5SJeff Kirsher *
509874aeea5SJeff Kirsher * Context: non-blocking.
51012804793SEdward Cree * Should always return NETDEV_TX_OK and consume the skb.
511874aeea5SJeff Kirsher */
efx_hard_start_xmit(struct sk_buff * skb,struct net_device * net_dev)512874aeea5SJeff Kirsher netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
513874aeea5SJeff Kirsher struct net_device *net_dev)
514874aeea5SJeff Kirsher {
5158cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(net_dev);
516874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue;
517874aeea5SJeff Kirsher unsigned index, type;
518874aeea5SJeff Kirsher
519874aeea5SJeff Kirsher EFX_WARN_ON_PARANOID(!netif_device_present(net_dev));
520874aeea5SJeff Kirsher index = skb_get_queue_mapping(skb);
521fe0c4060SEdward Cree type = efx_tx_csum_type_skb(skb);
5221c0544d2SEdward Cree
5231c0544d2SEdward Cree /* PTP "event" packet */
5241c0544d2SEdward Cree if (unlikely(efx_xmit_with_hwtstamp(skb)) &&
525bd4a2697SBert Kenward ((efx_ptp_use_mac_tx_timestamps(efx) && efx->ptp_data) ||
526bd4a2697SBert Kenward unlikely(efx_ptp_is_ptp_tx(efx, skb)))) {
5271c0544d2SEdward Cree /* There may be existing transmits on the channel that are
5281c0544d2SEdward Cree * waiting for this packet to trigger the doorbell write.
5291c0544d2SEdward Cree * We need to send the packets at this point.
5301c0544d2SEdward Cree */
5311c0544d2SEdward Cree efx_tx_send_pending(efx_get_tx_channel(efx, index));
5321c0544d2SEdward Cree return efx_ptp_tx(efx, skb);
5331c0544d2SEdward Cree }
5341c0544d2SEdward Cree
535874aeea5SJeff Kirsher tx_queue = efx_get_tx_queue(efx, index, type);
53612804793SEdward Cree if (WARN_ON_ONCE(!tx_queue)) {
53712804793SEdward Cree /* We don't have a TXQ of the right type.
53812804793SEdward Cree * This should never happen, as we don't advertise offload
53912804793SEdward Cree * features unless we can support them.
54012804793SEdward Cree */
54112804793SEdward Cree dev_kfree_skb_any(skb);
54212804793SEdward Cree /* If we're not expecting another transmit and we had something to push
54312804793SEdward Cree * on this queue or a partner queue then we need to push here to get the
54412804793SEdward Cree * previous packets out.
54512804793SEdward Cree */
54612804793SEdward Cree if (!netdev_xmit_more())
5470a242eb2SÍñigo Huguet efx_tx_send_pending(efx_get_tx_channel(efx, index));
54812804793SEdward Cree return NETDEV_TX_OK;
54912804793SEdward Cree }
550874aeea5SJeff Kirsher
55151b35a45SEdward Cree return __efx_enqueue_skb(tx_queue, skb);
552874aeea5SJeff Kirsher }
553874aeea5SJeff Kirsher
efx_xmit_done_single(struct efx_tx_queue * tx_queue)5543b4f06c7STom Zhao void efx_xmit_done_single(struct efx_tx_queue *tx_queue)
5553b4f06c7STom Zhao {
5563b4f06c7STom Zhao unsigned int pkts_compl = 0, bytes_compl = 0;
55702443ab8SEdward Cree unsigned int efv_pkts_compl = 0;
5583b4f06c7STom Zhao unsigned int read_ptr;
5593b4f06c7STom Zhao bool finished = false;
5603b4f06c7STom Zhao
5613b4f06c7STom Zhao read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
5623b4f06c7STom Zhao
5633b4f06c7STom Zhao while (!finished) {
5643b4f06c7STom Zhao struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
5653b4f06c7STom Zhao
5663b4f06c7STom Zhao if (!efx_tx_buffer_in_use(buffer)) {
5673b4f06c7STom Zhao struct efx_nic *efx = tx_queue->efx;
5683b4f06c7STom Zhao
5693b4f06c7STom Zhao netif_err(efx, hw, efx->net_dev,
5703b4f06c7STom Zhao "TX queue %d spurious single TX completion\n",
5713b4f06c7STom Zhao tx_queue->queue);
5723b4f06c7STom Zhao efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
5733b4f06c7STom Zhao return;
5743b4f06c7STom Zhao }
5753b4f06c7STom Zhao
5763b4f06c7STom Zhao /* Need to check the flag before dequeueing. */
5773b4f06c7STom Zhao if (buffer->flags & EFX_TX_BUF_SKB)
5783b4f06c7STom Zhao finished = true;
57902443ab8SEdward Cree efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl,
58002443ab8SEdward Cree &efv_pkts_compl);
5813b4f06c7STom Zhao
5823b4f06c7STom Zhao ++tx_queue->read_count;
5833b4f06c7STom Zhao read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
5843b4f06c7STom Zhao }
5853b4f06c7STom Zhao
5863b4f06c7STom Zhao tx_queue->pkts_compl += pkts_compl;
5873b4f06c7STom Zhao tx_queue->bytes_compl += bytes_compl;
5883b4f06c7STom Zhao
58902443ab8SEdward Cree EFX_WARN_ON_PARANOID(pkts_compl + efv_pkts_compl != 1);
5903b4f06c7STom Zhao
5913b4f06c7STom Zhao efx_xmit_done_check_empty(tx_queue);
5923b4f06c7STom Zhao }
5933b4f06c7STom Zhao
efx_init_tx_queue_core_txq(struct efx_tx_queue * tx_queue)594874aeea5SJeff Kirsher void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue)
595874aeea5SJeff Kirsher {
596874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx;
597874aeea5SJeff Kirsher
598874aeea5SJeff Kirsher /* Must be inverse of queue lookup in efx_hard_start_xmit() */
599874aeea5SJeff Kirsher tx_queue->core_txq =
600874aeea5SJeff Kirsher netdev_get_tx_queue(efx->net_dev,
601*f294c1f7SMartin Habets tx_queue->channel->channel);
602874aeea5SJeff Kirsher }
603