1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2874aeea5SJeff Kirsher /**************************************************************************** 3f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 4874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 5f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 6874aeea5SJeff Kirsher */ 7874aeea5SJeff Kirsher 8874aeea5SJeff Kirsher #include <linux/pci.h> 9874aeea5SJeff Kirsher #include <linux/tcp.h> 10874aeea5SJeff Kirsher #include <linux/ip.h> 11874aeea5SJeff Kirsher #include <linux/in.h> 12874aeea5SJeff Kirsher #include <linux/ipv6.h> 13874aeea5SJeff Kirsher #include <linux/slab.h> 14874aeea5SJeff Kirsher #include <net/ipv6.h> 15874aeea5SJeff Kirsher #include <linux/if_ether.h> 16874aeea5SJeff Kirsher #include <linux/highmem.h> 17183233beSBen Hutchings #include <linux/cache.h> 18874aeea5SJeff Kirsher #include "net_driver.h" 19874aeea5SJeff Kirsher #include "efx.h" 20183233beSBen Hutchings #include "io.h" 21874aeea5SJeff Kirsher #include "nic.h" 22e9117e50SBert Kenward #include "tx.h" 23e1253f39SAlex Maftei (amaftei) #include "tx_common.h" 24874aeea5SJeff Kirsher #include "workarounds.h" 25dfa50be9SBen Hutchings #include "ef10_regs.h" 26874aeea5SJeff Kirsher 27183233beSBen Hutchings #ifdef EFX_USE_PIO 28183233beSBen Hutchings 29183233beSBen Hutchings #define EFX_PIOBUF_SIZE_DEF ALIGN(256, L1_CACHE_BYTES) 30183233beSBen Hutchings unsigned int efx_piobuf_size __read_mostly = EFX_PIOBUF_SIZE_DEF; 31183233beSBen Hutchings 32183233beSBen Hutchings #endif /* EFX_USE_PIO */ 33183233beSBen Hutchings 34e9117e50SBert Kenward static inline u8 *efx_tx_get_copy_buffer(struct efx_tx_queue *tx_queue, 35e9117e50SBert Kenward struct efx_tx_buffer *buffer) 360fe5565bSBen Hutchings { 37e9117e50SBert Kenward unsigned int index = efx_tx_queue_get_insert_index(tx_queue); 38e9117e50SBert Kenward struct efx_buffer *page_buf = 39e9117e50SBert Kenward &tx_queue->cb_page[index >> (PAGE_SHIFT - EFX_TX_CB_ORDER)]; 40e9117e50SBert Kenward unsigned int offset = 41e9117e50SBert Kenward ((index << EFX_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); 42e9117e50SBert Kenward 43e9117e50SBert Kenward if (unlikely(!page_buf->addr) && 44e9117e50SBert Kenward efx_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, 45e9117e50SBert Kenward GFP_ATOMIC)) 46e9117e50SBert Kenward return NULL; 47e9117e50SBert Kenward buffer->dma_addr = page_buf->dma_addr + offset; 48e9117e50SBert Kenward buffer->unmap_len = 0; 49e9117e50SBert Kenward return (u8 *)page_buf->addr + offset; 500fe5565bSBen Hutchings } 510fe5565bSBen Hutchings 52e9117e50SBert Kenward u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue, 53e9117e50SBert Kenward struct efx_tx_buffer *buffer, size_t len) 540fe5565bSBen Hutchings { 55e9117e50SBert Kenward if (len > EFX_TX_CB_SIZE) 56e9117e50SBert Kenward return NULL; 57e9117e50SBert Kenward return efx_tx_get_copy_buffer(tx_queue, buffer); 580fe5565bSBen Hutchings } 590fe5565bSBen Hutchings 6014bf718fSBen Hutchings static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1) 6114bf718fSBen Hutchings { 6214bf718fSBen Hutchings /* We need to consider both queues that the net core sees as one */ 6314bf718fSBen Hutchings struct efx_tx_queue *txq2 = efx_tx_queue_partner(txq1); 6414bf718fSBen Hutchings struct efx_nic *efx = txq1->efx; 6514bf718fSBen Hutchings unsigned int fill_level; 6614bf718fSBen Hutchings 6714bf718fSBen Hutchings fill_level = max(txq1->insert_count - txq1->old_read_count, 6814bf718fSBen Hutchings txq2->insert_count - txq2->old_read_count); 6914bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) 7014bf718fSBen Hutchings return; 7114bf718fSBen Hutchings 7214bf718fSBen Hutchings /* We used the stale old_read_count above, which gives us a 7314bf718fSBen Hutchings * pessimistic estimate of the fill level (which may even 7414bf718fSBen Hutchings * validly be >= efx->txq_entries). Now try again using 7514bf718fSBen Hutchings * read_count (more likely to be a cache miss). 7614bf718fSBen Hutchings * 7714bf718fSBen Hutchings * If we read read_count and then conditionally stop the 7814bf718fSBen Hutchings * queue, it is possible for the completion path to race with 7914bf718fSBen Hutchings * us and complete all outstanding descriptors in the middle, 8014bf718fSBen Hutchings * after which there will be no more completions to wake it. 8114bf718fSBen Hutchings * Therefore we stop the queue first, then read read_count 8214bf718fSBen Hutchings * (with a memory barrier to ensure the ordering), then 8314bf718fSBen Hutchings * restart the queue if the fill level turns out to be low 8414bf718fSBen Hutchings * enough. 8514bf718fSBen Hutchings */ 8614bf718fSBen Hutchings netif_tx_stop_queue(txq1->core_txq); 8714bf718fSBen Hutchings smp_mb(); 886aa7de05SMark Rutland txq1->old_read_count = READ_ONCE(txq1->read_count); 896aa7de05SMark Rutland txq2->old_read_count = READ_ONCE(txq2->read_count); 9014bf718fSBen Hutchings 9114bf718fSBen Hutchings fill_level = max(txq1->insert_count - txq1->old_read_count, 9214bf718fSBen Hutchings txq2->insert_count - txq2->old_read_count); 93e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(fill_level >= efx->txq_entries); 9414bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) { 9514bf718fSBen Hutchings smp_mb(); 9614bf718fSBen Hutchings if (likely(!efx->loopback_selftest)) 9714bf718fSBen Hutchings netif_tx_start_queue(txq1->core_txq); 9814bf718fSBen Hutchings } 9914bf718fSBen Hutchings } 10014bf718fSBen Hutchings 101e9117e50SBert Kenward static int efx_enqueue_skb_copy(struct efx_tx_queue *tx_queue, 102e9117e50SBert Kenward struct sk_buff *skb) 103e9117e50SBert Kenward { 104e9117e50SBert Kenward unsigned int copy_len = skb->len; 105e9117e50SBert Kenward struct efx_tx_buffer *buffer; 106e9117e50SBert Kenward u8 *copy_buffer; 107e9117e50SBert Kenward int rc; 108e9117e50SBert Kenward 109e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(copy_len > EFX_TX_CB_SIZE); 110e9117e50SBert Kenward 111e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue); 112e9117e50SBert Kenward 113e9117e50SBert Kenward copy_buffer = efx_tx_get_copy_buffer(tx_queue, buffer); 114e9117e50SBert Kenward if (unlikely(!copy_buffer)) 115e9117e50SBert Kenward return -ENOMEM; 116e9117e50SBert Kenward 117e9117e50SBert Kenward rc = skb_copy_bits(skb, 0, copy_buffer, copy_len); 118e9117e50SBert Kenward EFX_WARN_ON_PARANOID(rc); 119e9117e50SBert Kenward buffer->len = copy_len; 120e9117e50SBert Kenward 121e9117e50SBert Kenward buffer->skb = skb; 122e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB; 123e9117e50SBert Kenward 124e9117e50SBert Kenward ++tx_queue->insert_count; 125e9117e50SBert Kenward return rc; 126e9117e50SBert Kenward } 127e9117e50SBert Kenward 128ee45fd92SJon Cooper #ifdef EFX_USE_PIO 129ee45fd92SJon Cooper 130ee45fd92SJon Cooper struct efx_short_copy_buffer { 131ee45fd92SJon Cooper int used; 132ee45fd92SJon Cooper u8 buf[L1_CACHE_BYTES]; 133ee45fd92SJon Cooper }; 134ee45fd92SJon Cooper 135ee45fd92SJon Cooper /* Copy to PIO, respecting that writes to PIO buffers must be dword aligned. 136ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 137ee45fd92SJon Cooper */ 138ee45fd92SJon Cooper static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf, 139ee45fd92SJon Cooper u8 *data, int len, 140ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 141ee45fd92SJon Cooper { 142ee45fd92SJon Cooper int block_len = len & ~(sizeof(copy_buf->buf) - 1); 143ee45fd92SJon Cooper 1444984c237SBen Hutchings __iowrite64_copy(*piobuf, data, block_len >> 3); 145ee45fd92SJon Cooper *piobuf += block_len; 146ee45fd92SJon Cooper len -= block_len; 147ee45fd92SJon Cooper 148ee45fd92SJon Cooper if (len) { 149ee45fd92SJon Cooper data += block_len; 150ee45fd92SJon Cooper BUG_ON(copy_buf->used); 151ee45fd92SJon Cooper BUG_ON(len > sizeof(copy_buf->buf)); 152ee45fd92SJon Cooper memcpy(copy_buf->buf, data, len); 153ee45fd92SJon Cooper copy_buf->used = len; 154ee45fd92SJon Cooper } 155ee45fd92SJon Cooper } 156ee45fd92SJon Cooper 157ee45fd92SJon Cooper /* Copy to PIO, respecting dword alignment, popping data from copy buffer first. 158ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 159ee45fd92SJon Cooper */ 160ee45fd92SJon Cooper static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf, 161ee45fd92SJon Cooper u8 *data, int len, 162ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 163ee45fd92SJon Cooper { 164ee45fd92SJon Cooper if (copy_buf->used) { 165ee45fd92SJon Cooper /* if the copy buffer is partially full, fill it up and write */ 166ee45fd92SJon Cooper int copy_to_buf = 167ee45fd92SJon Cooper min_t(int, sizeof(copy_buf->buf) - copy_buf->used, len); 168ee45fd92SJon Cooper 169ee45fd92SJon Cooper memcpy(copy_buf->buf + copy_buf->used, data, copy_to_buf); 170ee45fd92SJon Cooper copy_buf->used += copy_to_buf; 171ee45fd92SJon Cooper 172ee45fd92SJon Cooper /* if we didn't fill it up then we're done for now */ 173ee45fd92SJon Cooper if (copy_buf->used < sizeof(copy_buf->buf)) 174ee45fd92SJon Cooper return; 175ee45fd92SJon Cooper 1764984c237SBen Hutchings __iowrite64_copy(*piobuf, copy_buf->buf, 1774984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 178ee45fd92SJon Cooper *piobuf += sizeof(copy_buf->buf); 179ee45fd92SJon Cooper data += copy_to_buf; 180ee45fd92SJon Cooper len -= copy_to_buf; 181ee45fd92SJon Cooper copy_buf->used = 0; 182ee45fd92SJon Cooper } 183ee45fd92SJon Cooper 184ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, data, len, copy_buf); 185ee45fd92SJon Cooper } 186ee45fd92SJon Cooper 187ee45fd92SJon Cooper static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf, 188ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 189ee45fd92SJon Cooper { 190ee45fd92SJon Cooper /* if there's anything in it, write the whole buffer, including junk */ 191ee45fd92SJon Cooper if (copy_buf->used) 1924984c237SBen Hutchings __iowrite64_copy(piobuf, copy_buf->buf, 1934984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 194ee45fd92SJon Cooper } 195ee45fd92SJon Cooper 196ee45fd92SJon Cooper /* Traverse skb structure and copy fragments in to PIO buffer. 197ee45fd92SJon Cooper * Advances piobuf pointer. 198ee45fd92SJon Cooper */ 199ee45fd92SJon Cooper static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb, 200ee45fd92SJon Cooper u8 __iomem **piobuf, 201ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 202ee45fd92SJon Cooper { 203ee45fd92SJon Cooper int i; 204ee45fd92SJon Cooper 205ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, skb->data, skb_headlen(skb), 206ee45fd92SJon Cooper copy_buf); 207ee45fd92SJon Cooper 208ee45fd92SJon Cooper for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) { 209ee45fd92SJon Cooper skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 210ee45fd92SJon Cooper u8 *vaddr; 211ee45fd92SJon Cooper 212ee45fd92SJon Cooper vaddr = kmap_atomic(skb_frag_page(f)); 213ee45fd92SJon Cooper 214b54c9d5bSJonathan Lemon efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f), 215ee45fd92SJon Cooper skb_frag_size(f), copy_buf); 216ee45fd92SJon Cooper kunmap_atomic(vaddr); 217ee45fd92SJon Cooper } 218ee45fd92SJon Cooper 219e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list); 220ee45fd92SJon Cooper } 221ee45fd92SJon Cooper 222e9117e50SBert Kenward static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, 223e9117e50SBert Kenward struct sk_buff *skb) 224ee45fd92SJon Cooper { 225ee45fd92SJon Cooper struct efx_tx_buffer *buffer = 226ee45fd92SJon Cooper efx_tx_queue_get_insert_buffer(tx_queue); 227ee45fd92SJon Cooper u8 __iomem *piobuf = tx_queue->piobuf; 228ee45fd92SJon Cooper 229ee45fd92SJon Cooper /* Copy to PIO buffer. Ensure the writes are padded to the end 230ee45fd92SJon Cooper * of a cache line, as this is required for write-combining to be 231ee45fd92SJon Cooper * effective on at least x86. 232ee45fd92SJon Cooper */ 233ee45fd92SJon Cooper 234ee45fd92SJon Cooper if (skb_shinfo(skb)->nr_frags) { 235ee45fd92SJon Cooper /* The size of the copy buffer will ensure all writes 236ee45fd92SJon Cooper * are the size of a cache line. 237ee45fd92SJon Cooper */ 238ee45fd92SJon Cooper struct efx_short_copy_buffer copy_buf; 239ee45fd92SJon Cooper 240ee45fd92SJon Cooper copy_buf.used = 0; 241ee45fd92SJon Cooper 242ee45fd92SJon Cooper efx_skb_copy_bits_to_pio(tx_queue->efx, skb, 243ee45fd92SJon Cooper &piobuf, ©_buf); 244ee45fd92SJon Cooper efx_flush_copy_buffer(tx_queue->efx, piobuf, ©_buf); 245ee45fd92SJon Cooper } else { 246ee45fd92SJon Cooper /* Pad the write to the size of a cache line. 247e9117e50SBert Kenward * We can do this because we know the skb_shared_info struct is 248ee45fd92SJon Cooper * after the source, and the destination buffer is big enough. 249ee45fd92SJon Cooper */ 250ee45fd92SJon Cooper BUILD_BUG_ON(L1_CACHE_BYTES > 251ee45fd92SJon Cooper SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 2524984c237SBen Hutchings __iowrite64_copy(tx_queue->piobuf, skb->data, 2534984c237SBen Hutchings ALIGN(skb->len, L1_CACHE_BYTES) >> 3); 254ee45fd92SJon Cooper } 255ee45fd92SJon Cooper 256e9117e50SBert Kenward buffer->skb = skb; 257e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | EFX_TX_BUF_OPTION; 258e9117e50SBert Kenward 259ee45fd92SJon Cooper EFX_POPULATE_QWORD_5(buffer->option, 260ee45fd92SJon Cooper ESF_DZ_TX_DESC_IS_OPT, 1, 261ee45fd92SJon Cooper ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_PIO, 262ee45fd92SJon Cooper ESF_DZ_TX_PIO_CONT, 0, 263ee45fd92SJon Cooper ESF_DZ_TX_PIO_BYTE_CNT, skb->len, 264ee45fd92SJon Cooper ESF_DZ_TX_PIO_BUF_ADDR, 265ee45fd92SJon Cooper tx_queue->piobuf_offset); 266ee45fd92SJon Cooper ++tx_queue->insert_count; 267e9117e50SBert Kenward return 0; 268ee45fd92SJon Cooper } 269ee45fd92SJon Cooper #endif /* EFX_USE_PIO */ 270ee45fd92SJon Cooper 27146d1efd8SEdward Cree /* 272874aeea5SJeff Kirsher * Add a socket buffer to a TX queue 273874aeea5SJeff Kirsher * 274874aeea5SJeff Kirsher * This maps all fragments of a socket buffer for DMA and adds them to 275874aeea5SJeff Kirsher * the TX queue. The queue's insert pointer will be incremented by 276874aeea5SJeff Kirsher * the number of fragments in the socket buffer. 277874aeea5SJeff Kirsher * 278874aeea5SJeff Kirsher * If any DMA mapping fails, any mapped fragments will be unmapped, 279874aeea5SJeff Kirsher * the queue's insert pointer will be restored to its original value. 280874aeea5SJeff Kirsher * 281874aeea5SJeff Kirsher * This function is split out from efx_hard_start_xmit to allow the 282874aeea5SJeff Kirsher * loopback test to direct packets via specific TX queues. 283874aeea5SJeff Kirsher * 28414bf718fSBen Hutchings * Returns NETDEV_TX_OK. 285874aeea5SJeff Kirsher * You must hold netif_tx_lock() to call this function. 286874aeea5SJeff Kirsher */ 287874aeea5SJeff Kirsher netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) 288874aeea5SJeff Kirsher { 2890c235113SMartin Habets unsigned int old_insert_count = tx_queue->insert_count; 290f79c957aSFlorian Westphal bool xmit_more = netdev_xmit_more(); 291e9117e50SBert Kenward bool data_mapped = false; 292e9117e50SBert Kenward unsigned int segments; 293e9117e50SBert Kenward unsigned int skb_len; 29446d1efd8SEdward Cree int rc; 295874aeea5SJeff Kirsher 296e9117e50SBert Kenward skb_len = skb->len; 297e9117e50SBert Kenward segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0; 298e9117e50SBert Kenward if (segments == 1) 299e9117e50SBert Kenward segments = 0; /* Don't use TSO for a single segment. */ 300874aeea5SJeff Kirsher 301e9117e50SBert Kenward /* Handle TSO first - it's *possible* (although unlikely) that we might 302e9117e50SBert Kenward * be passed a packet to segment that's smaller than the copybreak/PIO 303e9117e50SBert Kenward * size limit. 304874aeea5SJeff Kirsher */ 305e9117e50SBert Kenward if (segments) { 306e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(!tx_queue->handle_tso); 30746d1efd8SEdward Cree rc = tx_queue->handle_tso(tx_queue, skb, &data_mapped); 30846d1efd8SEdward Cree if (rc == -EINVAL) { 30946d1efd8SEdward Cree rc = efx_tx_tso_fallback(tx_queue, skb); 31046d1efd8SEdward Cree tx_queue->tso_fallbacks++; 31146d1efd8SEdward Cree if (rc == 0) 31246d1efd8SEdward Cree return 0; 31346d1efd8SEdward Cree } 31446d1efd8SEdward Cree if (rc) 315e9117e50SBert Kenward goto err; 316e9117e50SBert Kenward #ifdef EFX_USE_PIO 317f79c957aSFlorian Westphal } else if (skb_len <= efx_piobuf_size && !xmit_more && 318e9117e50SBert Kenward efx_nic_may_tx_pio(tx_queue)) { 319e9117e50SBert Kenward /* Use PIO for short packets with an empty queue. */ 320e9117e50SBert Kenward if (efx_enqueue_skb_pio(tx_queue, skb)) 321e9117e50SBert Kenward goto err; 322e9117e50SBert Kenward tx_queue->pio_packets++; 323e9117e50SBert Kenward data_mapped = true; 324e9117e50SBert Kenward #endif 3255a6681e2SEdward Cree } else if (skb->data_len && skb_len <= EFX_TX_CB_SIZE) { 326e9117e50SBert Kenward /* Pad short packets or coalesce short fragmented packets. */ 327e9117e50SBert Kenward if (efx_enqueue_skb_copy(tx_queue, skb)) 328e9117e50SBert Kenward goto err; 329e9117e50SBert Kenward tx_queue->cb_packets++; 330e9117e50SBert Kenward data_mapped = true; 331874aeea5SJeff Kirsher } 332874aeea5SJeff Kirsher 333e9117e50SBert Kenward /* Map for DMA and create descriptors if we haven't done so already. */ 334e9117e50SBert Kenward if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments))) 335e9117e50SBert Kenward goto err; 336874aeea5SJeff Kirsher 3370c235113SMartin Habets efx_tx_maybe_stop_queue(tx_queue); 3380c235113SMartin Habets 339874aeea5SJeff Kirsher /* Pass off to hardware */ 34029e12207SEdward Cree if (__netdev_tx_sent_queue(tx_queue->core_txq, skb_len, xmit_more)) { 341b2663a4fSMartin Habets struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue); 342b2663a4fSMartin Habets 343f79c957aSFlorian Westphal /* There could be packets left on the partner queue if 344f79c957aSFlorian Westphal * xmit_more was set. If we do not push those they 345b2663a4fSMartin Habets * could be left for a long time and cause a netdev watchdog. 346b2663a4fSMartin Habets */ 347b2663a4fSMartin Habets if (txq2->xmit_more_available) 348b2663a4fSMartin Habets efx_nic_push_buffers(txq2); 349b2663a4fSMartin Habets 350874aeea5SJeff Kirsher efx_nic_push_buffers(tx_queue); 351b2663a4fSMartin Habets } else { 352f79c957aSFlorian Westphal tx_queue->xmit_more_available = xmit_more; 353b2663a4fSMartin Habets } 354874aeea5SJeff Kirsher 355e9117e50SBert Kenward if (segments) { 356e9117e50SBert Kenward tx_queue->tso_bursts++; 357e9117e50SBert Kenward tx_queue->tso_packets += segments; 358e9117e50SBert Kenward tx_queue->tx_packets += segments; 359e9117e50SBert Kenward } else { 3608ccf3800SAndrew Rybchenko tx_queue->tx_packets++; 361e9117e50SBert Kenward } 362e9117e50SBert Kenward 363874aeea5SJeff Kirsher return NETDEV_TX_OK; 364874aeea5SJeff Kirsher 365874aeea5SJeff Kirsher 366e9117e50SBert Kenward err: 3670c235113SMartin Habets efx_enqueue_unwind(tx_queue, old_insert_count); 368874aeea5SJeff Kirsher dev_kfree_skb_any(skb); 3690c235113SMartin Habets 3700c235113SMartin Habets /* If we're not expecting another transmit and we had something to push 3710c235113SMartin Habets * on this queue or a partner queue then we need to push here to get the 3720c235113SMartin Habets * previous packets out. 3730c235113SMartin Habets */ 3740c235113SMartin Habets if (!xmit_more) { 3750c235113SMartin Habets struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue); 3760c235113SMartin Habets 3770c235113SMartin Habets if (txq2->xmit_more_available) 3780c235113SMartin Habets efx_nic_push_buffers(txq2); 3790c235113SMartin Habets 3800c235113SMartin Habets efx_nic_push_buffers(tx_queue); 3810c235113SMartin Habets } 3820c235113SMartin Habets 38314bf718fSBen Hutchings return NETDEV_TX_OK; 384874aeea5SJeff Kirsher } 385874aeea5SJeff Kirsher 386dfe44c1fSCharles McLachlan static void efx_xdp_return_frames(int n, struct xdp_frame **xdpfs) 387dfe44c1fSCharles McLachlan { 388dfe44c1fSCharles McLachlan int i; 389dfe44c1fSCharles McLachlan 390dfe44c1fSCharles McLachlan for (i = 0; i < n; i++) 391dfe44c1fSCharles McLachlan xdp_return_frame_rx_napi(xdpfs[i]); 392dfe44c1fSCharles McLachlan } 393dfe44c1fSCharles McLachlan 394dfe44c1fSCharles McLachlan /* Transmit a packet from an XDP buffer 395dfe44c1fSCharles McLachlan * 396dfe44c1fSCharles McLachlan * Returns number of packets sent on success, error code otherwise. 397dfe44c1fSCharles McLachlan * Runs in NAPI context, either in our poll (for XDP TX) or a different NIC 398dfe44c1fSCharles McLachlan * (for XDP redirect). 399dfe44c1fSCharles McLachlan */ 400dfe44c1fSCharles McLachlan int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs, 401dfe44c1fSCharles McLachlan bool flush) 402dfe44c1fSCharles McLachlan { 403dfe44c1fSCharles McLachlan struct efx_tx_buffer *tx_buffer; 404dfe44c1fSCharles McLachlan struct efx_tx_queue *tx_queue; 405dfe44c1fSCharles McLachlan struct xdp_frame *xdpf; 406dfe44c1fSCharles McLachlan dma_addr_t dma_addr; 407dfe44c1fSCharles McLachlan unsigned int len; 408dfe44c1fSCharles McLachlan int space; 409dfe44c1fSCharles McLachlan int cpu; 410dfe44c1fSCharles McLachlan int i; 411dfe44c1fSCharles McLachlan 412dfe44c1fSCharles McLachlan cpu = raw_smp_processor_id(); 413dfe44c1fSCharles McLachlan 414dfe44c1fSCharles McLachlan if (!efx->xdp_tx_queue_count || 415dfe44c1fSCharles McLachlan unlikely(cpu >= efx->xdp_tx_queue_count)) 416dfe44c1fSCharles McLachlan return -EINVAL; 417dfe44c1fSCharles McLachlan 418dfe44c1fSCharles McLachlan tx_queue = efx->xdp_tx_queues[cpu]; 419dfe44c1fSCharles McLachlan if (unlikely(!tx_queue)) 420dfe44c1fSCharles McLachlan return -EINVAL; 421dfe44c1fSCharles McLachlan 422dfe44c1fSCharles McLachlan if (unlikely(n && !xdpfs)) 423dfe44c1fSCharles McLachlan return -EINVAL; 424dfe44c1fSCharles McLachlan 425dfe44c1fSCharles McLachlan if (!n) 426dfe44c1fSCharles McLachlan return 0; 427dfe44c1fSCharles McLachlan 428dfe44c1fSCharles McLachlan /* Check for available space. We should never need multiple 429dfe44c1fSCharles McLachlan * descriptors per frame. 430dfe44c1fSCharles McLachlan */ 431dfe44c1fSCharles McLachlan space = efx->txq_entries + 432dfe44c1fSCharles McLachlan tx_queue->read_count - tx_queue->insert_count; 433dfe44c1fSCharles McLachlan 434dfe44c1fSCharles McLachlan for (i = 0; i < n; i++) { 435dfe44c1fSCharles McLachlan xdpf = xdpfs[i]; 436dfe44c1fSCharles McLachlan 437dfe44c1fSCharles McLachlan if (i >= space) 438dfe44c1fSCharles McLachlan break; 439dfe44c1fSCharles McLachlan 440dfe44c1fSCharles McLachlan /* We'll want a descriptor for this tx. */ 441dfe44c1fSCharles McLachlan prefetchw(__efx_tx_queue_get_insert_buffer(tx_queue)); 442dfe44c1fSCharles McLachlan 443dfe44c1fSCharles McLachlan len = xdpf->len; 444dfe44c1fSCharles McLachlan 445dfe44c1fSCharles McLachlan /* Map for DMA. */ 446dfe44c1fSCharles McLachlan dma_addr = dma_map_single(&efx->pci_dev->dev, 447dfe44c1fSCharles McLachlan xdpf->data, len, 448dfe44c1fSCharles McLachlan DMA_TO_DEVICE); 449dfe44c1fSCharles McLachlan if (dma_mapping_error(&efx->pci_dev->dev, dma_addr)) 450dfe44c1fSCharles McLachlan break; 451dfe44c1fSCharles McLachlan 452dfe44c1fSCharles McLachlan /* Create descriptor and set up for unmapping DMA. */ 453dfe44c1fSCharles McLachlan tx_buffer = efx_tx_map_chunk(tx_queue, dma_addr, len); 454dfe44c1fSCharles McLachlan tx_buffer->xdpf = xdpf; 455dfe44c1fSCharles McLachlan tx_buffer->flags = EFX_TX_BUF_XDP | 456dfe44c1fSCharles McLachlan EFX_TX_BUF_MAP_SINGLE; 457dfe44c1fSCharles McLachlan tx_buffer->dma_offset = 0; 458dfe44c1fSCharles McLachlan tx_buffer->unmap_len = len; 459dfe44c1fSCharles McLachlan tx_queue->tx_packets++; 460dfe44c1fSCharles McLachlan } 461dfe44c1fSCharles McLachlan 462dfe44c1fSCharles McLachlan /* Pass mapped frames to hardware. */ 463dfe44c1fSCharles McLachlan if (flush && i > 0) 464dfe44c1fSCharles McLachlan efx_nic_push_buffers(tx_queue); 465dfe44c1fSCharles McLachlan 466dfe44c1fSCharles McLachlan if (i == 0) 467dfe44c1fSCharles McLachlan return -EIO; 468dfe44c1fSCharles McLachlan 469dfe44c1fSCharles McLachlan efx_xdp_return_frames(n - i, xdpfs + i); 470dfe44c1fSCharles McLachlan 471dfe44c1fSCharles McLachlan return i; 472dfe44c1fSCharles McLachlan } 473dfe44c1fSCharles McLachlan 474874aeea5SJeff Kirsher /* Initiate a packet transmission. We use one channel per CPU 475874aeea5SJeff Kirsher * (sharing when we have more CPUs than channels). On Falcon, the TX 476874aeea5SJeff Kirsher * completion events will be directed back to the CPU that transmitted 477874aeea5SJeff Kirsher * the packet, which should be cache-efficient. 478874aeea5SJeff Kirsher * 479874aeea5SJeff Kirsher * Context: non-blocking. 480874aeea5SJeff Kirsher * Note that returning anything other than NETDEV_TX_OK will cause the 481874aeea5SJeff Kirsher * OS to free the skb. 482874aeea5SJeff Kirsher */ 483874aeea5SJeff Kirsher netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, 484874aeea5SJeff Kirsher struct net_device *net_dev) 485874aeea5SJeff Kirsher { 486874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 487874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 488874aeea5SJeff Kirsher unsigned index, type; 489874aeea5SJeff Kirsher 490874aeea5SJeff Kirsher EFX_WARN_ON_PARANOID(!netif_device_present(net_dev)); 491874aeea5SJeff Kirsher 4927c236c43SStuart Hodgson /* PTP "event" packet */ 4937c236c43SStuart Hodgson if (unlikely(efx_xmit_with_hwtstamp(skb)) && 4947c236c43SStuart Hodgson unlikely(efx_ptp_is_ptp_tx(efx, skb))) { 4957c236c43SStuart Hodgson return efx_ptp_tx(efx, skb); 4967c236c43SStuart Hodgson } 4977c236c43SStuart Hodgson 498874aeea5SJeff Kirsher index = skb_get_queue_mapping(skb); 499874aeea5SJeff Kirsher type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0; 500874aeea5SJeff Kirsher if (index >= efx->n_tx_channels) { 501874aeea5SJeff Kirsher index -= efx->n_tx_channels; 502874aeea5SJeff Kirsher type |= EFX_TXQ_TYPE_HIGHPRI; 503874aeea5SJeff Kirsher } 504874aeea5SJeff Kirsher tx_queue = efx_get_tx_queue(efx, index, type); 505874aeea5SJeff Kirsher 506874aeea5SJeff Kirsher return efx_enqueue_skb(tx_queue, skb); 507874aeea5SJeff Kirsher } 508874aeea5SJeff Kirsher 5093b4f06c7STom Zhao void efx_xmit_done_single(struct efx_tx_queue *tx_queue) 5103b4f06c7STom Zhao { 5113b4f06c7STom Zhao unsigned int pkts_compl = 0, bytes_compl = 0; 5123b4f06c7STom Zhao unsigned int read_ptr; 5133b4f06c7STom Zhao bool finished = false; 5143b4f06c7STom Zhao 5153b4f06c7STom Zhao read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 5163b4f06c7STom Zhao 5173b4f06c7STom Zhao while (!finished) { 5183b4f06c7STom Zhao struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; 5193b4f06c7STom Zhao 5203b4f06c7STom Zhao if (!efx_tx_buffer_in_use(buffer)) { 5213b4f06c7STom Zhao struct efx_nic *efx = tx_queue->efx; 5223b4f06c7STom Zhao 5233b4f06c7STom Zhao netif_err(efx, hw, efx->net_dev, 5243b4f06c7STom Zhao "TX queue %d spurious single TX completion\n", 5253b4f06c7STom Zhao tx_queue->queue); 5263b4f06c7STom Zhao efx_schedule_reset(efx, RESET_TYPE_TX_SKIP); 5273b4f06c7STom Zhao return; 5283b4f06c7STom Zhao } 5293b4f06c7STom Zhao 5303b4f06c7STom Zhao /* Need to check the flag before dequeueing. */ 5313b4f06c7STom Zhao if (buffer->flags & EFX_TX_BUF_SKB) 5323b4f06c7STom Zhao finished = true; 5333b4f06c7STom Zhao efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); 5343b4f06c7STom Zhao 5353b4f06c7STom Zhao ++tx_queue->read_count; 5363b4f06c7STom Zhao read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 5373b4f06c7STom Zhao } 5383b4f06c7STom Zhao 5393b4f06c7STom Zhao tx_queue->pkts_compl += pkts_compl; 5403b4f06c7STom Zhao tx_queue->bytes_compl += bytes_compl; 5413b4f06c7STom Zhao 5423b4f06c7STom Zhao EFX_WARN_ON_PARANOID(pkts_compl != 1); 5433b4f06c7STom Zhao 5443b4f06c7STom Zhao efx_xmit_done_check_empty(tx_queue); 5453b4f06c7STom Zhao } 5463b4f06c7STom Zhao 547874aeea5SJeff Kirsher void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue) 548874aeea5SJeff Kirsher { 549874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 550874aeea5SJeff Kirsher 551874aeea5SJeff Kirsher /* Must be inverse of queue lookup in efx_hard_start_xmit() */ 552874aeea5SJeff Kirsher tx_queue->core_txq = 553874aeea5SJeff Kirsher netdev_get_tx_queue(efx->net_dev, 554874aeea5SJeff Kirsher tx_queue->queue / EFX_TXQ_TYPES + 555874aeea5SJeff Kirsher ((tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ? 556874aeea5SJeff Kirsher efx->n_tx_channels : 0)); 557874aeea5SJeff Kirsher } 558874aeea5SJeff Kirsher 5592572ac53SJiri Pirko int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type, 560de4784caSJiri Pirko void *type_data) 561874aeea5SJeff Kirsher { 562874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 563de4784caSJiri Pirko struct tc_mqprio_qopt *mqprio = type_data; 564874aeea5SJeff Kirsher struct efx_channel *channel; 565874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 56616e5cc64SJohn Fastabend unsigned tc, num_tc; 567874aeea5SJeff Kirsher int rc; 568874aeea5SJeff Kirsher 569575ed7d3SNogah Frankel if (type != TC_SETUP_QDISC_MQPRIO) 57038cf0426SJiri Pirko return -EOPNOTSUPP; 571e4c6734eSJohn Fastabend 572f9cac93eSEdward Cree /* Only Siena supported highpri queues */ 573f9cac93eSEdward Cree if (efx_nic_rev(efx) > EFX_REV_SIENA_A0) 574f9cac93eSEdward Cree return -EOPNOTSUPP; 575f9cac93eSEdward Cree 576de4784caSJiri Pirko num_tc = mqprio->num_tc; 57716e5cc64SJohn Fastabend 5785a6681e2SEdward Cree if (num_tc > EFX_MAX_TX_TC) 579874aeea5SJeff Kirsher return -EINVAL; 580874aeea5SJeff Kirsher 581de4784caSJiri Pirko mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 58256f36acdSAmritha Nambiar 583874aeea5SJeff Kirsher if (num_tc == net_dev->num_tc) 584874aeea5SJeff Kirsher return 0; 585874aeea5SJeff Kirsher 586874aeea5SJeff Kirsher for (tc = 0; tc < num_tc; tc++) { 587874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; 588874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].count = efx->n_tx_channels; 589874aeea5SJeff Kirsher } 590874aeea5SJeff Kirsher 591874aeea5SJeff Kirsher if (num_tc > net_dev->num_tc) { 592f9cac93eSEdward Cree efx->tx_queues_per_channel = 4; 593874aeea5SJeff Kirsher /* Initialise high-priority queues as necessary */ 594874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 595f9cac93eSEdward Cree efx_for_each_channel_tx_queue(tx_queue, channel) { 596874aeea5SJeff Kirsher if (!(tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI)) 597874aeea5SJeff Kirsher continue; 598874aeea5SJeff Kirsher if (!tx_queue->buffer) { 599874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 600874aeea5SJeff Kirsher if (rc) 601874aeea5SJeff Kirsher return rc; 602874aeea5SJeff Kirsher } 603874aeea5SJeff Kirsher if (!tx_queue->initialised) 604874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 605874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 606874aeea5SJeff Kirsher } 607874aeea5SJeff Kirsher } 608874aeea5SJeff Kirsher } else { 609874aeea5SJeff Kirsher /* Reduce number of classes before number of queues */ 610874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 611874aeea5SJeff Kirsher } 612874aeea5SJeff Kirsher 613874aeea5SJeff Kirsher rc = netif_set_real_num_tx_queues(net_dev, 614874aeea5SJeff Kirsher max_t(int, num_tc, 1) * 615874aeea5SJeff Kirsher efx->n_tx_channels); 616874aeea5SJeff Kirsher if (rc) 617874aeea5SJeff Kirsher return rc; 618874aeea5SJeff Kirsher 619874aeea5SJeff Kirsher /* Do not destroy high-priority queues when they become 620874aeea5SJeff Kirsher * unused. We would have to flush them first, and it is 621874aeea5SJeff Kirsher * fairly difficult to flush a subset of TX queues. Leave 622874aeea5SJeff Kirsher * it to efx_fini_channels(). 623874aeea5SJeff Kirsher */ 624874aeea5SJeff Kirsher 625874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 626874aeea5SJeff Kirsher return 0; 627874aeea5SJeff Kirsher } 628