1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2874aeea5SJeff Kirsher /**************************************************************************** 3f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 4874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 5f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 6874aeea5SJeff Kirsher */ 7874aeea5SJeff Kirsher 8874aeea5SJeff Kirsher #include <linux/pci.h> 9874aeea5SJeff Kirsher #include <linux/tcp.h> 10874aeea5SJeff Kirsher #include <linux/ip.h> 11874aeea5SJeff Kirsher #include <linux/in.h> 12874aeea5SJeff Kirsher #include <linux/ipv6.h> 13874aeea5SJeff Kirsher #include <linux/slab.h> 14874aeea5SJeff Kirsher #include <net/ipv6.h> 15874aeea5SJeff Kirsher #include <linux/if_ether.h> 16874aeea5SJeff Kirsher #include <linux/highmem.h> 17183233beSBen Hutchings #include <linux/cache.h> 18874aeea5SJeff Kirsher #include "net_driver.h" 19874aeea5SJeff Kirsher #include "efx.h" 20183233beSBen Hutchings #include "io.h" 21874aeea5SJeff Kirsher #include "nic.h" 22e9117e50SBert Kenward #include "tx.h" 23e1253f39SAlex Maftei (amaftei) #include "tx_common.h" 24874aeea5SJeff Kirsher #include "workarounds.h" 25dfa50be9SBen Hutchings #include "ef10_regs.h" 26874aeea5SJeff Kirsher 27183233beSBen Hutchings #ifdef EFX_USE_PIO 28183233beSBen Hutchings 29183233beSBen Hutchings #define EFX_PIOBUF_SIZE_DEF ALIGN(256, L1_CACHE_BYTES) 30183233beSBen Hutchings unsigned int efx_piobuf_size __read_mostly = EFX_PIOBUF_SIZE_DEF; 31183233beSBen Hutchings 32183233beSBen Hutchings #endif /* EFX_USE_PIO */ 33183233beSBen Hutchings 34e9117e50SBert Kenward static inline u8 *efx_tx_get_copy_buffer(struct efx_tx_queue *tx_queue, 35e9117e50SBert Kenward struct efx_tx_buffer *buffer) 360fe5565bSBen Hutchings { 37e9117e50SBert Kenward unsigned int index = efx_tx_queue_get_insert_index(tx_queue); 38e9117e50SBert Kenward struct efx_buffer *page_buf = 39e9117e50SBert Kenward &tx_queue->cb_page[index >> (PAGE_SHIFT - EFX_TX_CB_ORDER)]; 40e9117e50SBert Kenward unsigned int offset = 41e9117e50SBert Kenward ((index << EFX_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); 42e9117e50SBert Kenward 43e9117e50SBert Kenward if (unlikely(!page_buf->addr) && 44e9117e50SBert Kenward efx_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, 45e9117e50SBert Kenward GFP_ATOMIC)) 46e9117e50SBert Kenward return NULL; 47e9117e50SBert Kenward buffer->dma_addr = page_buf->dma_addr + offset; 48e9117e50SBert Kenward buffer->unmap_len = 0; 49e9117e50SBert Kenward return (u8 *)page_buf->addr + offset; 500fe5565bSBen Hutchings } 510fe5565bSBen Hutchings 52e9117e50SBert Kenward u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue, 53e9117e50SBert Kenward struct efx_tx_buffer *buffer, size_t len) 540fe5565bSBen Hutchings { 55e9117e50SBert Kenward if (len > EFX_TX_CB_SIZE) 56e9117e50SBert Kenward return NULL; 57e9117e50SBert Kenward return efx_tx_get_copy_buffer(tx_queue, buffer); 580fe5565bSBen Hutchings } 590fe5565bSBen Hutchings 6014bf718fSBen Hutchings static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1) 6114bf718fSBen Hutchings { 620d8c1229SEdward Cree /* We need to consider all queues that the net core sees as one */ 6314bf718fSBen Hutchings struct efx_nic *efx = txq1->efx; 640d8c1229SEdward Cree struct efx_tx_queue *txq2; 6514bf718fSBen Hutchings unsigned int fill_level; 6614bf718fSBen Hutchings 670d8c1229SEdward Cree fill_level = efx_channel_tx_old_fill_level(txq1->channel); 6814bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) 6914bf718fSBen Hutchings return; 7014bf718fSBen Hutchings 7114bf718fSBen Hutchings /* We used the stale old_read_count above, which gives us a 7214bf718fSBen Hutchings * pessimistic estimate of the fill level (which may even 7314bf718fSBen Hutchings * validly be >= efx->txq_entries). Now try again using 7414bf718fSBen Hutchings * read_count (more likely to be a cache miss). 7514bf718fSBen Hutchings * 7614bf718fSBen Hutchings * If we read read_count and then conditionally stop the 7714bf718fSBen Hutchings * queue, it is possible for the completion path to race with 7814bf718fSBen Hutchings * us and complete all outstanding descriptors in the middle, 7914bf718fSBen Hutchings * after which there will be no more completions to wake it. 8014bf718fSBen Hutchings * Therefore we stop the queue first, then read read_count 8114bf718fSBen Hutchings * (with a memory barrier to ensure the ordering), then 8214bf718fSBen Hutchings * restart the queue if the fill level turns out to be low 8314bf718fSBen Hutchings * enough. 8414bf718fSBen Hutchings */ 8514bf718fSBen Hutchings netif_tx_stop_queue(txq1->core_txq); 8614bf718fSBen Hutchings smp_mb(); 870d8c1229SEdward Cree efx_for_each_channel_tx_queue(txq2, txq1->channel) 886aa7de05SMark Rutland txq2->old_read_count = READ_ONCE(txq2->read_count); 8914bf718fSBen Hutchings 900d8c1229SEdward Cree fill_level = efx_channel_tx_old_fill_level(txq1->channel); 91e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(fill_level >= efx->txq_entries); 9214bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) { 9314bf718fSBen Hutchings smp_mb(); 9414bf718fSBen Hutchings if (likely(!efx->loopback_selftest)) 9514bf718fSBen Hutchings netif_tx_start_queue(txq1->core_txq); 9614bf718fSBen Hutchings } 9714bf718fSBen Hutchings } 9814bf718fSBen Hutchings 99e9117e50SBert Kenward static int efx_enqueue_skb_copy(struct efx_tx_queue *tx_queue, 100e9117e50SBert Kenward struct sk_buff *skb) 101e9117e50SBert Kenward { 102e9117e50SBert Kenward unsigned int copy_len = skb->len; 103e9117e50SBert Kenward struct efx_tx_buffer *buffer; 104e9117e50SBert Kenward u8 *copy_buffer; 105e9117e50SBert Kenward int rc; 106e9117e50SBert Kenward 107e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(copy_len > EFX_TX_CB_SIZE); 108e9117e50SBert Kenward 109e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue); 110e9117e50SBert Kenward 111e9117e50SBert Kenward copy_buffer = efx_tx_get_copy_buffer(tx_queue, buffer); 112e9117e50SBert Kenward if (unlikely(!copy_buffer)) 113e9117e50SBert Kenward return -ENOMEM; 114e9117e50SBert Kenward 115e9117e50SBert Kenward rc = skb_copy_bits(skb, 0, copy_buffer, copy_len); 116e9117e50SBert Kenward EFX_WARN_ON_PARANOID(rc); 117e9117e50SBert Kenward buffer->len = copy_len; 118e9117e50SBert Kenward 119e9117e50SBert Kenward buffer->skb = skb; 120e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB; 121e9117e50SBert Kenward 122e9117e50SBert Kenward ++tx_queue->insert_count; 123e9117e50SBert Kenward return rc; 124e9117e50SBert Kenward } 125e9117e50SBert Kenward 126ee45fd92SJon Cooper #ifdef EFX_USE_PIO 127ee45fd92SJon Cooper 128ee45fd92SJon Cooper struct efx_short_copy_buffer { 129ee45fd92SJon Cooper int used; 130ee45fd92SJon Cooper u8 buf[L1_CACHE_BYTES]; 131ee45fd92SJon Cooper }; 132ee45fd92SJon Cooper 133ee45fd92SJon Cooper /* Copy to PIO, respecting that writes to PIO buffers must be dword aligned. 134ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 135ee45fd92SJon Cooper */ 136ee45fd92SJon Cooper static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf, 137ee45fd92SJon Cooper u8 *data, int len, 138ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 139ee45fd92SJon Cooper { 140ee45fd92SJon Cooper int block_len = len & ~(sizeof(copy_buf->buf) - 1); 141ee45fd92SJon Cooper 1424984c237SBen Hutchings __iowrite64_copy(*piobuf, data, block_len >> 3); 143ee45fd92SJon Cooper *piobuf += block_len; 144ee45fd92SJon Cooper len -= block_len; 145ee45fd92SJon Cooper 146ee45fd92SJon Cooper if (len) { 147ee45fd92SJon Cooper data += block_len; 148ee45fd92SJon Cooper BUG_ON(copy_buf->used); 149ee45fd92SJon Cooper BUG_ON(len > sizeof(copy_buf->buf)); 150ee45fd92SJon Cooper memcpy(copy_buf->buf, data, len); 151ee45fd92SJon Cooper copy_buf->used = len; 152ee45fd92SJon Cooper } 153ee45fd92SJon Cooper } 154ee45fd92SJon Cooper 155ee45fd92SJon Cooper /* Copy to PIO, respecting dword alignment, popping data from copy buffer first. 156ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 157ee45fd92SJon Cooper */ 158ee45fd92SJon Cooper static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf, 159ee45fd92SJon Cooper u8 *data, int len, 160ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 161ee45fd92SJon Cooper { 162ee45fd92SJon Cooper if (copy_buf->used) { 163ee45fd92SJon Cooper /* if the copy buffer is partially full, fill it up and write */ 164ee45fd92SJon Cooper int copy_to_buf = 165ee45fd92SJon Cooper min_t(int, sizeof(copy_buf->buf) - copy_buf->used, len); 166ee45fd92SJon Cooper 167ee45fd92SJon Cooper memcpy(copy_buf->buf + copy_buf->used, data, copy_to_buf); 168ee45fd92SJon Cooper copy_buf->used += copy_to_buf; 169ee45fd92SJon Cooper 170ee45fd92SJon Cooper /* if we didn't fill it up then we're done for now */ 171ee45fd92SJon Cooper if (copy_buf->used < sizeof(copy_buf->buf)) 172ee45fd92SJon Cooper return; 173ee45fd92SJon Cooper 1744984c237SBen Hutchings __iowrite64_copy(*piobuf, copy_buf->buf, 1754984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 176ee45fd92SJon Cooper *piobuf += sizeof(copy_buf->buf); 177ee45fd92SJon Cooper data += copy_to_buf; 178ee45fd92SJon Cooper len -= copy_to_buf; 179ee45fd92SJon Cooper copy_buf->used = 0; 180ee45fd92SJon Cooper } 181ee45fd92SJon Cooper 182ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, data, len, copy_buf); 183ee45fd92SJon Cooper } 184ee45fd92SJon Cooper 185ee45fd92SJon Cooper static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf, 186ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 187ee45fd92SJon Cooper { 188ee45fd92SJon Cooper /* if there's anything in it, write the whole buffer, including junk */ 189ee45fd92SJon Cooper if (copy_buf->used) 1904984c237SBen Hutchings __iowrite64_copy(piobuf, copy_buf->buf, 1914984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 192ee45fd92SJon Cooper } 193ee45fd92SJon Cooper 194ee45fd92SJon Cooper /* Traverse skb structure and copy fragments in to PIO buffer. 195ee45fd92SJon Cooper * Advances piobuf pointer. 196ee45fd92SJon Cooper */ 197ee45fd92SJon Cooper static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb, 198ee45fd92SJon Cooper u8 __iomem **piobuf, 199ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 200ee45fd92SJon Cooper { 201ee45fd92SJon Cooper int i; 202ee45fd92SJon Cooper 203ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, skb->data, skb_headlen(skb), 204ee45fd92SJon Cooper copy_buf); 205ee45fd92SJon Cooper 206ee45fd92SJon Cooper for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) { 207ee45fd92SJon Cooper skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 208ee45fd92SJon Cooper u8 *vaddr; 209ee45fd92SJon Cooper 210ee45fd92SJon Cooper vaddr = kmap_atomic(skb_frag_page(f)); 211ee45fd92SJon Cooper 212b54c9d5bSJonathan Lemon efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f), 213ee45fd92SJon Cooper skb_frag_size(f), copy_buf); 214ee45fd92SJon Cooper kunmap_atomic(vaddr); 215ee45fd92SJon Cooper } 216ee45fd92SJon Cooper 217e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list); 218ee45fd92SJon Cooper } 219ee45fd92SJon Cooper 220e9117e50SBert Kenward static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, 221e9117e50SBert Kenward struct sk_buff *skb) 222ee45fd92SJon Cooper { 223ee45fd92SJon Cooper struct efx_tx_buffer *buffer = 224ee45fd92SJon Cooper efx_tx_queue_get_insert_buffer(tx_queue); 225ee45fd92SJon Cooper u8 __iomem *piobuf = tx_queue->piobuf; 226ee45fd92SJon Cooper 227ee45fd92SJon Cooper /* Copy to PIO buffer. Ensure the writes are padded to the end 228ee45fd92SJon Cooper * of a cache line, as this is required for write-combining to be 229ee45fd92SJon Cooper * effective on at least x86. 230ee45fd92SJon Cooper */ 231ee45fd92SJon Cooper 232ee45fd92SJon Cooper if (skb_shinfo(skb)->nr_frags) { 233ee45fd92SJon Cooper /* The size of the copy buffer will ensure all writes 234ee45fd92SJon Cooper * are the size of a cache line. 235ee45fd92SJon Cooper */ 236ee45fd92SJon Cooper struct efx_short_copy_buffer copy_buf; 237ee45fd92SJon Cooper 238ee45fd92SJon Cooper copy_buf.used = 0; 239ee45fd92SJon Cooper 240ee45fd92SJon Cooper efx_skb_copy_bits_to_pio(tx_queue->efx, skb, 241ee45fd92SJon Cooper &piobuf, ©_buf); 242ee45fd92SJon Cooper efx_flush_copy_buffer(tx_queue->efx, piobuf, ©_buf); 243ee45fd92SJon Cooper } else { 244ee45fd92SJon Cooper /* Pad the write to the size of a cache line. 245e9117e50SBert Kenward * We can do this because we know the skb_shared_info struct is 246ee45fd92SJon Cooper * after the source, and the destination buffer is big enough. 247ee45fd92SJon Cooper */ 248ee45fd92SJon Cooper BUILD_BUG_ON(L1_CACHE_BYTES > 249ee45fd92SJon Cooper SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 2504984c237SBen Hutchings __iowrite64_copy(tx_queue->piobuf, skb->data, 2514984c237SBen Hutchings ALIGN(skb->len, L1_CACHE_BYTES) >> 3); 252ee45fd92SJon Cooper } 253ee45fd92SJon Cooper 254e9117e50SBert Kenward buffer->skb = skb; 255e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | EFX_TX_BUF_OPTION; 256e9117e50SBert Kenward 257ee45fd92SJon Cooper EFX_POPULATE_QWORD_5(buffer->option, 258ee45fd92SJon Cooper ESF_DZ_TX_DESC_IS_OPT, 1, 259ee45fd92SJon Cooper ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_PIO, 260ee45fd92SJon Cooper ESF_DZ_TX_PIO_CONT, 0, 261ee45fd92SJon Cooper ESF_DZ_TX_PIO_BYTE_CNT, skb->len, 262ee45fd92SJon Cooper ESF_DZ_TX_PIO_BUF_ADDR, 263ee45fd92SJon Cooper tx_queue->piobuf_offset); 264ee45fd92SJon Cooper ++tx_queue->insert_count; 265e9117e50SBert Kenward return 0; 266ee45fd92SJon Cooper } 2678be41842SEdward Cree 2688be41842SEdward Cree /* Decide whether we can use TX PIO, ie. write packet data directly into 2698be41842SEdward Cree * a buffer on the device. This can reduce latency at the expense of 2708be41842SEdward Cree * throughput, so we only do this if both hardware and software TX rings 2718be41842SEdward Cree * are empty, including all queues for the channel. This also ensures that 2728be41842SEdward Cree * only one packet at a time can be using the PIO buffer. If the xmit_more 2738be41842SEdward Cree * flag is set then we don't use this - there'll be another packet along 2748be41842SEdward Cree * shortly and we want to hold off the doorbell. 2758be41842SEdward Cree */ 2768be41842SEdward Cree static bool efx_tx_may_pio(struct efx_tx_queue *tx_queue) 2778be41842SEdward Cree { 2788be41842SEdward Cree struct efx_channel *channel = tx_queue->channel; 2798be41842SEdward Cree 2808be41842SEdward Cree if (!tx_queue->piobuf) 2818be41842SEdward Cree return false; 2828be41842SEdward Cree 2838be41842SEdward Cree EFX_WARN_ON_ONCE_PARANOID(!channel->efx->type->option_descriptors); 2848be41842SEdward Cree 2858be41842SEdward Cree efx_for_each_channel_tx_queue(tx_queue, channel) 2868be41842SEdward Cree if (!efx_nic_tx_is_empty(tx_queue, tx_queue->packet_write_count)) 2878be41842SEdward Cree return false; 2888be41842SEdward Cree 2898be41842SEdward Cree return true; 2908be41842SEdward Cree } 291ee45fd92SJon Cooper #endif /* EFX_USE_PIO */ 292ee45fd92SJon Cooper 2931c0544d2SEdward Cree /* Send any pending traffic for a channel. xmit_more is shared across all 2941c0544d2SEdward Cree * queues for a channel, so we must check all of them. 2951c0544d2SEdward Cree */ 2961c0544d2SEdward Cree static void efx_tx_send_pending(struct efx_channel *channel) 2971c0544d2SEdward Cree { 2981c0544d2SEdward Cree struct efx_tx_queue *q; 2991c0544d2SEdward Cree 3001c0544d2SEdward Cree efx_for_each_channel_tx_queue(q, channel) { 3011c0544d2SEdward Cree if (q->xmit_pending) 3021c0544d2SEdward Cree efx_nic_push_buffers(q); 3031c0544d2SEdward Cree } 3041c0544d2SEdward Cree } 3051c0544d2SEdward Cree 30646d1efd8SEdward Cree /* 307874aeea5SJeff Kirsher * Add a socket buffer to a TX queue 308874aeea5SJeff Kirsher * 309874aeea5SJeff Kirsher * This maps all fragments of a socket buffer for DMA and adds them to 310874aeea5SJeff Kirsher * the TX queue. The queue's insert pointer will be incremented by 311874aeea5SJeff Kirsher * the number of fragments in the socket buffer. 312874aeea5SJeff Kirsher * 313874aeea5SJeff Kirsher * If any DMA mapping fails, any mapped fragments will be unmapped, 314874aeea5SJeff Kirsher * the queue's insert pointer will be restored to its original value. 315874aeea5SJeff Kirsher * 316874aeea5SJeff Kirsher * This function is split out from efx_hard_start_xmit to allow the 317874aeea5SJeff Kirsher * loopback test to direct packets via specific TX queues. 318874aeea5SJeff Kirsher * 31914bf718fSBen Hutchings * Returns NETDEV_TX_OK. 320874aeea5SJeff Kirsher * You must hold netif_tx_lock() to call this function. 321874aeea5SJeff Kirsher */ 32251b35a45SEdward Cree netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) 323874aeea5SJeff Kirsher { 3240c235113SMartin Habets unsigned int old_insert_count = tx_queue->insert_count; 325f79c957aSFlorian Westphal bool xmit_more = netdev_xmit_more(); 326e9117e50SBert Kenward bool data_mapped = false; 327e9117e50SBert Kenward unsigned int segments; 328e9117e50SBert Kenward unsigned int skb_len; 32946d1efd8SEdward Cree int rc; 330874aeea5SJeff Kirsher 331e9117e50SBert Kenward skb_len = skb->len; 332e9117e50SBert Kenward segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0; 333e9117e50SBert Kenward if (segments == 1) 334e9117e50SBert Kenward segments = 0; /* Don't use TSO for a single segment. */ 335874aeea5SJeff Kirsher 336e9117e50SBert Kenward /* Handle TSO first - it's *possible* (although unlikely) that we might 337e9117e50SBert Kenward * be passed a packet to segment that's smaller than the copybreak/PIO 338e9117e50SBert Kenward * size limit. 339874aeea5SJeff Kirsher */ 340e9117e50SBert Kenward if (segments) { 3411679c72cSEdward Cree switch (tx_queue->tso_version) { 3421679c72cSEdward Cree case 1: 3431679c72cSEdward Cree rc = efx_enqueue_skb_tso(tx_queue, skb, &data_mapped); 3441679c72cSEdward Cree break; 3451679c72cSEdward Cree case 2: 3461679c72cSEdward Cree rc = efx_ef10_tx_tso_desc(tx_queue, skb, &data_mapped); 3471679c72cSEdward Cree break; 3481679c72cSEdward Cree case 0: /* No TSO on this queue, SW fallback needed */ 3491679c72cSEdward Cree default: 3501679c72cSEdward Cree rc = -EINVAL; 3511679c72cSEdward Cree break; 3521679c72cSEdward Cree } 35346d1efd8SEdward Cree if (rc == -EINVAL) { 35446d1efd8SEdward Cree rc = efx_tx_tso_fallback(tx_queue, skb); 35546d1efd8SEdward Cree tx_queue->tso_fallbacks++; 35646d1efd8SEdward Cree if (rc == 0) 35746d1efd8SEdward Cree return 0; 35846d1efd8SEdward Cree } 35946d1efd8SEdward Cree if (rc) 360e9117e50SBert Kenward goto err; 361e9117e50SBert Kenward #ifdef EFX_USE_PIO 362f79c957aSFlorian Westphal } else if (skb_len <= efx_piobuf_size && !xmit_more && 3638be41842SEdward Cree efx_tx_may_pio(tx_queue)) { 364e9117e50SBert Kenward /* Use PIO for short packets with an empty queue. */ 365e9117e50SBert Kenward if (efx_enqueue_skb_pio(tx_queue, skb)) 366e9117e50SBert Kenward goto err; 367e9117e50SBert Kenward tx_queue->pio_packets++; 368e9117e50SBert Kenward data_mapped = true; 369e9117e50SBert Kenward #endif 3705a6681e2SEdward Cree } else if (skb->data_len && skb_len <= EFX_TX_CB_SIZE) { 371e9117e50SBert Kenward /* Pad short packets or coalesce short fragmented packets. */ 372e9117e50SBert Kenward if (efx_enqueue_skb_copy(tx_queue, skb)) 373e9117e50SBert Kenward goto err; 374e9117e50SBert Kenward tx_queue->cb_packets++; 375e9117e50SBert Kenward data_mapped = true; 376874aeea5SJeff Kirsher } 377874aeea5SJeff Kirsher 378e9117e50SBert Kenward /* Map for DMA and create descriptors if we haven't done so already. */ 379e9117e50SBert Kenward if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments))) 380e9117e50SBert Kenward goto err; 381874aeea5SJeff Kirsher 3820c235113SMartin Habets efx_tx_maybe_stop_queue(tx_queue); 3830c235113SMartin Habets 3841c0544d2SEdward Cree tx_queue->xmit_pending = true; 3851c0544d2SEdward Cree 386874aeea5SJeff Kirsher /* Pass off to hardware */ 3871c0544d2SEdward Cree if (__netdev_tx_sent_queue(tx_queue->core_txq, skb_len, xmit_more)) 3881c0544d2SEdward Cree efx_tx_send_pending(tx_queue->channel); 389874aeea5SJeff Kirsher 390e9117e50SBert Kenward if (segments) { 391e9117e50SBert Kenward tx_queue->tso_bursts++; 392e9117e50SBert Kenward tx_queue->tso_packets += segments; 393e9117e50SBert Kenward tx_queue->tx_packets += segments; 394e9117e50SBert Kenward } else { 3958ccf3800SAndrew Rybchenko tx_queue->tx_packets++; 396e9117e50SBert Kenward } 397e9117e50SBert Kenward 398874aeea5SJeff Kirsher return NETDEV_TX_OK; 399874aeea5SJeff Kirsher 400874aeea5SJeff Kirsher 401e9117e50SBert Kenward err: 4020c235113SMartin Habets efx_enqueue_unwind(tx_queue, old_insert_count); 403874aeea5SJeff Kirsher dev_kfree_skb_any(skb); 4040c235113SMartin Habets 4050c235113SMartin Habets /* If we're not expecting another transmit and we had something to push 4060c235113SMartin Habets * on this queue or a partner queue then we need to push here to get the 4070c235113SMartin Habets * previous packets out. 4080c235113SMartin Habets */ 4091c0544d2SEdward Cree if (!xmit_more) 4101c0544d2SEdward Cree efx_tx_send_pending(tx_queue->channel); 4110c235113SMartin Habets 41214bf718fSBen Hutchings return NETDEV_TX_OK; 413874aeea5SJeff Kirsher } 414874aeea5SJeff Kirsher 415dfe44c1fSCharles McLachlan /* Transmit a packet from an XDP buffer 416dfe44c1fSCharles McLachlan * 417dfe44c1fSCharles McLachlan * Returns number of packets sent on success, error code otherwise. 418dfe44c1fSCharles McLachlan * Runs in NAPI context, either in our poll (for XDP TX) or a different NIC 419dfe44c1fSCharles McLachlan * (for XDP redirect). 420dfe44c1fSCharles McLachlan */ 421dfe44c1fSCharles McLachlan int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs, 422dfe44c1fSCharles McLachlan bool flush) 423dfe44c1fSCharles McLachlan { 424dfe44c1fSCharles McLachlan struct efx_tx_buffer *tx_buffer; 425dfe44c1fSCharles McLachlan struct efx_tx_queue *tx_queue; 426dfe44c1fSCharles McLachlan struct xdp_frame *xdpf; 427dfe44c1fSCharles McLachlan dma_addr_t dma_addr; 428dfe44c1fSCharles McLachlan unsigned int len; 429dfe44c1fSCharles McLachlan int space; 430dfe44c1fSCharles McLachlan int cpu; 4316215b608SÍñigo Huguet int i = 0; 432dfe44c1fSCharles McLachlan 43341544618SÍñigo Huguet if (unlikely(n && !xdpfs)) 43441544618SÍñigo Huguet return -EINVAL; 43541544618SÍñigo Huguet if (unlikely(!n)) 43641544618SÍñigo Huguet return 0; 437dfe44c1fSCharles McLachlan 43841544618SÍñigo Huguet cpu = raw_smp_processor_id(); 43941544618SÍñigo Huguet if (unlikely(cpu >= efx->xdp_tx_queue_count)) 440dfe44c1fSCharles McLachlan return -EINVAL; 441dfe44c1fSCharles McLachlan 442dfe44c1fSCharles McLachlan tx_queue = efx->xdp_tx_queues[cpu]; 443dfe44c1fSCharles McLachlan if (unlikely(!tx_queue)) 444dfe44c1fSCharles McLachlan return -EINVAL; 445dfe44c1fSCharles McLachlan 446fb5833d8STaehee Yoo if (!tx_queue->initialised) 447fb5833d8STaehee Yoo return -EINVAL; 448fb5833d8STaehee Yoo 44941544618SÍñigo Huguet if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED) 45041544618SÍñigo Huguet HARD_TX_LOCK(efx->net_dev, tx_queue->core_txq, cpu); 451dfe44c1fSCharles McLachlan 4526215b608SÍñigo Huguet /* If we're borrowing net stack queues we have to handle stop-restart 4536215b608SÍñigo Huguet * or we might block the queue and it will be considered as frozen 4546215b608SÍñigo Huguet */ 4556215b608SÍñigo Huguet if (efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_BORROWED) { 4566215b608SÍñigo Huguet if (netif_tx_queue_stopped(tx_queue->core_txq)) 4576215b608SÍñigo Huguet goto unlock; 4586215b608SÍñigo Huguet efx_tx_maybe_stop_queue(tx_queue); 4596215b608SÍñigo Huguet } 4606215b608SÍñigo Huguet 461dfe44c1fSCharles McLachlan /* Check for available space. We should never need multiple 462dfe44c1fSCharles McLachlan * descriptors per frame. 463dfe44c1fSCharles McLachlan */ 464dfe44c1fSCharles McLachlan space = efx->txq_entries + 465dfe44c1fSCharles McLachlan tx_queue->read_count - tx_queue->insert_count; 466dfe44c1fSCharles McLachlan 467dfe44c1fSCharles McLachlan for (i = 0; i < n; i++) { 468dfe44c1fSCharles McLachlan xdpf = xdpfs[i]; 469dfe44c1fSCharles McLachlan 470dfe44c1fSCharles McLachlan if (i >= space) 471dfe44c1fSCharles McLachlan break; 472dfe44c1fSCharles McLachlan 473dfe44c1fSCharles McLachlan /* We'll want a descriptor for this tx. */ 474dfe44c1fSCharles McLachlan prefetchw(__efx_tx_queue_get_insert_buffer(tx_queue)); 475dfe44c1fSCharles McLachlan 476dfe44c1fSCharles McLachlan len = xdpf->len; 477dfe44c1fSCharles McLachlan 478dfe44c1fSCharles McLachlan /* Map for DMA. */ 479dfe44c1fSCharles McLachlan dma_addr = dma_map_single(&efx->pci_dev->dev, 480dfe44c1fSCharles McLachlan xdpf->data, len, 481dfe44c1fSCharles McLachlan DMA_TO_DEVICE); 482dfe44c1fSCharles McLachlan if (dma_mapping_error(&efx->pci_dev->dev, dma_addr)) 483dfe44c1fSCharles McLachlan break; 484dfe44c1fSCharles McLachlan 485dfe44c1fSCharles McLachlan /* Create descriptor and set up for unmapping DMA. */ 486dfe44c1fSCharles McLachlan tx_buffer = efx_tx_map_chunk(tx_queue, dma_addr, len); 487dfe44c1fSCharles McLachlan tx_buffer->xdpf = xdpf; 488dfe44c1fSCharles McLachlan tx_buffer->flags = EFX_TX_BUF_XDP | 489dfe44c1fSCharles McLachlan EFX_TX_BUF_MAP_SINGLE; 490dfe44c1fSCharles McLachlan tx_buffer->dma_offset = 0; 491dfe44c1fSCharles McLachlan tx_buffer->unmap_len = len; 492dfe44c1fSCharles McLachlan tx_queue->tx_packets++; 493dfe44c1fSCharles McLachlan } 494dfe44c1fSCharles McLachlan 495dfe44c1fSCharles McLachlan /* Pass mapped frames to hardware. */ 496dfe44c1fSCharles McLachlan if (flush && i > 0) 497dfe44c1fSCharles McLachlan efx_nic_push_buffers(tx_queue); 498dfe44c1fSCharles McLachlan 4996215b608SÍñigo Huguet unlock: 50041544618SÍñigo Huguet if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED) 50141544618SÍñigo Huguet HARD_TX_UNLOCK(efx->net_dev, tx_queue->core_txq); 50241544618SÍñigo Huguet 503fdc13979SLorenzo Bianconi return i == 0 ? -EIO : i; 504dfe44c1fSCharles McLachlan } 505dfe44c1fSCharles McLachlan 506874aeea5SJeff Kirsher /* Initiate a packet transmission. We use one channel per CPU 50712804793SEdward Cree * (sharing when we have more CPUs than channels). 508874aeea5SJeff Kirsher * 509874aeea5SJeff Kirsher * Context: non-blocking. 51012804793SEdward Cree * Should always return NETDEV_TX_OK and consume the skb. 511874aeea5SJeff Kirsher */ 512874aeea5SJeff Kirsher netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, 513874aeea5SJeff Kirsher struct net_device *net_dev) 514874aeea5SJeff Kirsher { 515*8cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(net_dev); 516874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 517874aeea5SJeff Kirsher unsigned index, type; 518874aeea5SJeff Kirsher 519874aeea5SJeff Kirsher EFX_WARN_ON_PARANOID(!netif_device_present(net_dev)); 520874aeea5SJeff Kirsher 521874aeea5SJeff Kirsher index = skb_get_queue_mapping(skb); 522fe0c4060SEdward Cree type = efx_tx_csum_type_skb(skb); 523874aeea5SJeff Kirsher if (index >= efx->n_tx_channels) { 524874aeea5SJeff Kirsher index -= efx->n_tx_channels; 525874aeea5SJeff Kirsher type |= EFX_TXQ_TYPE_HIGHPRI; 526874aeea5SJeff Kirsher } 5271c0544d2SEdward Cree 5281c0544d2SEdward Cree /* PTP "event" packet */ 5291c0544d2SEdward Cree if (unlikely(efx_xmit_with_hwtstamp(skb)) && 530bd4a2697SBert Kenward ((efx_ptp_use_mac_tx_timestamps(efx) && efx->ptp_data) || 531bd4a2697SBert Kenward unlikely(efx_ptp_is_ptp_tx(efx, skb)))) { 5321c0544d2SEdward Cree /* There may be existing transmits on the channel that are 5331c0544d2SEdward Cree * waiting for this packet to trigger the doorbell write. 5341c0544d2SEdward Cree * We need to send the packets at this point. 5351c0544d2SEdward Cree */ 5361c0544d2SEdward Cree efx_tx_send_pending(efx_get_tx_channel(efx, index)); 5371c0544d2SEdward Cree return efx_ptp_tx(efx, skb); 5381c0544d2SEdward Cree } 5391c0544d2SEdward Cree 540874aeea5SJeff Kirsher tx_queue = efx_get_tx_queue(efx, index, type); 54112804793SEdward Cree if (WARN_ON_ONCE(!tx_queue)) { 54212804793SEdward Cree /* We don't have a TXQ of the right type. 54312804793SEdward Cree * This should never happen, as we don't advertise offload 54412804793SEdward Cree * features unless we can support them. 54512804793SEdward Cree */ 54612804793SEdward Cree dev_kfree_skb_any(skb); 54712804793SEdward Cree /* If we're not expecting another transmit and we had something to push 54812804793SEdward Cree * on this queue or a partner queue then we need to push here to get the 54912804793SEdward Cree * previous packets out. 55012804793SEdward Cree */ 55112804793SEdward Cree if (!netdev_xmit_more()) 55212804793SEdward Cree efx_tx_send_pending(tx_queue->channel); 55312804793SEdward Cree return NETDEV_TX_OK; 55412804793SEdward Cree } 555874aeea5SJeff Kirsher 55651b35a45SEdward Cree return __efx_enqueue_skb(tx_queue, skb); 557874aeea5SJeff Kirsher } 558874aeea5SJeff Kirsher 5593b4f06c7STom Zhao void efx_xmit_done_single(struct efx_tx_queue *tx_queue) 5603b4f06c7STom Zhao { 5613b4f06c7STom Zhao unsigned int pkts_compl = 0, bytes_compl = 0; 5623b4f06c7STom Zhao unsigned int read_ptr; 5633b4f06c7STom Zhao bool finished = false; 5643b4f06c7STom Zhao 5653b4f06c7STom Zhao read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 5663b4f06c7STom Zhao 5673b4f06c7STom Zhao while (!finished) { 5683b4f06c7STom Zhao struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; 5693b4f06c7STom Zhao 5703b4f06c7STom Zhao if (!efx_tx_buffer_in_use(buffer)) { 5713b4f06c7STom Zhao struct efx_nic *efx = tx_queue->efx; 5723b4f06c7STom Zhao 5733b4f06c7STom Zhao netif_err(efx, hw, efx->net_dev, 5743b4f06c7STom Zhao "TX queue %d spurious single TX completion\n", 5753b4f06c7STom Zhao tx_queue->queue); 5763b4f06c7STom Zhao efx_schedule_reset(efx, RESET_TYPE_TX_SKIP); 5773b4f06c7STom Zhao return; 5783b4f06c7STom Zhao } 5793b4f06c7STom Zhao 5803b4f06c7STom Zhao /* Need to check the flag before dequeueing. */ 5813b4f06c7STom Zhao if (buffer->flags & EFX_TX_BUF_SKB) 5823b4f06c7STom Zhao finished = true; 5833b4f06c7STom Zhao efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); 5843b4f06c7STom Zhao 5853b4f06c7STom Zhao ++tx_queue->read_count; 5863b4f06c7STom Zhao read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 5873b4f06c7STom Zhao } 5883b4f06c7STom Zhao 5893b4f06c7STom Zhao tx_queue->pkts_compl += pkts_compl; 5903b4f06c7STom Zhao tx_queue->bytes_compl += bytes_compl; 5913b4f06c7STom Zhao 5923b4f06c7STom Zhao EFX_WARN_ON_PARANOID(pkts_compl != 1); 5933b4f06c7STom Zhao 5943b4f06c7STom Zhao efx_xmit_done_check_empty(tx_queue); 5953b4f06c7STom Zhao } 5963b4f06c7STom Zhao 597874aeea5SJeff Kirsher void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue) 598874aeea5SJeff Kirsher { 599874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 600874aeea5SJeff Kirsher 601874aeea5SJeff Kirsher /* Must be inverse of queue lookup in efx_hard_start_xmit() */ 602874aeea5SJeff Kirsher tx_queue->core_txq = 603874aeea5SJeff Kirsher netdev_get_tx_queue(efx->net_dev, 604a81dcd85SEdward Cree tx_queue->channel->channel + 60512804793SEdward Cree ((tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ? 606874aeea5SJeff Kirsher efx->n_tx_channels : 0)); 607874aeea5SJeff Kirsher } 608874aeea5SJeff Kirsher 6092572ac53SJiri Pirko int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type, 610de4784caSJiri Pirko void *type_data) 611874aeea5SJeff Kirsher { 612*8cb03f4eSJonathan Cooper struct efx_nic *efx = efx_netdev_priv(net_dev); 613de4784caSJiri Pirko struct tc_mqprio_qopt *mqprio = type_data; 61416e5cc64SJohn Fastabend unsigned tc, num_tc; 615874aeea5SJeff Kirsher 616575ed7d3SNogah Frankel if (type != TC_SETUP_QDISC_MQPRIO) 61738cf0426SJiri Pirko return -EOPNOTSUPP; 618e4c6734eSJohn Fastabend 619f9cac93eSEdward Cree /* Only Siena supported highpri queues */ 620f9cac93eSEdward Cree if (efx_nic_rev(efx) > EFX_REV_SIENA_A0) 621f9cac93eSEdward Cree return -EOPNOTSUPP; 622f9cac93eSEdward Cree 623de4784caSJiri Pirko num_tc = mqprio->num_tc; 62416e5cc64SJohn Fastabend 6255a6681e2SEdward Cree if (num_tc > EFX_MAX_TX_TC) 626874aeea5SJeff Kirsher return -EINVAL; 627874aeea5SJeff Kirsher 628de4784caSJiri Pirko mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 62956f36acdSAmritha Nambiar 630874aeea5SJeff Kirsher if (num_tc == net_dev->num_tc) 631874aeea5SJeff Kirsher return 0; 632874aeea5SJeff Kirsher 633874aeea5SJeff Kirsher for (tc = 0; tc < num_tc; tc++) { 634874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; 635874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].count = efx->n_tx_channels; 636874aeea5SJeff Kirsher } 637874aeea5SJeff Kirsher 638874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 639874aeea5SJeff Kirsher 640a81dcd85SEdward Cree return netif_set_real_num_tx_queues(net_dev, 641874aeea5SJeff Kirsher max_t(int, num_tc, 1) * 642874aeea5SJeff Kirsher efx->n_tx_channels); 643874aeea5SJeff Kirsher } 644