1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2874aeea5SJeff Kirsher /**************************************************************************** 3f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 4874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 5f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 6874aeea5SJeff Kirsher */ 7874aeea5SJeff Kirsher 8874aeea5SJeff Kirsher #include <linux/pci.h> 9874aeea5SJeff Kirsher #include <linux/tcp.h> 10874aeea5SJeff Kirsher #include <linux/ip.h> 11874aeea5SJeff Kirsher #include <linux/in.h> 12874aeea5SJeff Kirsher #include <linux/ipv6.h> 13874aeea5SJeff Kirsher #include <linux/slab.h> 14874aeea5SJeff Kirsher #include <net/ipv6.h> 15874aeea5SJeff Kirsher #include <linux/if_ether.h> 16874aeea5SJeff Kirsher #include <linux/highmem.h> 17183233beSBen Hutchings #include <linux/cache.h> 18874aeea5SJeff Kirsher #include "net_driver.h" 19874aeea5SJeff Kirsher #include "efx.h" 20183233beSBen Hutchings #include "io.h" 21874aeea5SJeff Kirsher #include "nic.h" 22e9117e50SBert Kenward #include "tx.h" 23874aeea5SJeff Kirsher #include "workarounds.h" 24dfa50be9SBen Hutchings #include "ef10_regs.h" 25874aeea5SJeff Kirsher 26183233beSBen Hutchings #ifdef EFX_USE_PIO 27183233beSBen Hutchings 28183233beSBen Hutchings #define EFX_PIOBUF_SIZE_DEF ALIGN(256, L1_CACHE_BYTES) 29183233beSBen Hutchings unsigned int efx_piobuf_size __read_mostly = EFX_PIOBUF_SIZE_DEF; 30183233beSBen Hutchings 31183233beSBen Hutchings #endif /* EFX_USE_PIO */ 32183233beSBen Hutchings 33e9117e50SBert Kenward static inline u8 *efx_tx_get_copy_buffer(struct efx_tx_queue *tx_queue, 34e9117e50SBert Kenward struct efx_tx_buffer *buffer) 350fe5565bSBen Hutchings { 36e9117e50SBert Kenward unsigned int index = efx_tx_queue_get_insert_index(tx_queue); 37e9117e50SBert Kenward struct efx_buffer *page_buf = 38e9117e50SBert Kenward &tx_queue->cb_page[index >> (PAGE_SHIFT - EFX_TX_CB_ORDER)]; 39e9117e50SBert Kenward unsigned int offset = 40e9117e50SBert Kenward ((index << EFX_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); 41e9117e50SBert Kenward 42e9117e50SBert Kenward if (unlikely(!page_buf->addr) && 43e9117e50SBert Kenward efx_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, 44e9117e50SBert Kenward GFP_ATOMIC)) 45e9117e50SBert Kenward return NULL; 46e9117e50SBert Kenward buffer->dma_addr = page_buf->dma_addr + offset; 47e9117e50SBert Kenward buffer->unmap_len = 0; 48e9117e50SBert Kenward return (u8 *)page_buf->addr + offset; 490fe5565bSBen Hutchings } 500fe5565bSBen Hutchings 51e9117e50SBert Kenward u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue, 52e9117e50SBert Kenward struct efx_tx_buffer *buffer, size_t len) 530fe5565bSBen Hutchings { 54e9117e50SBert Kenward if (len > EFX_TX_CB_SIZE) 55e9117e50SBert Kenward return NULL; 56e9117e50SBert Kenward return efx_tx_get_copy_buffer(tx_queue, buffer); 570fe5565bSBen Hutchings } 580fe5565bSBen Hutchings 59874aeea5SJeff Kirsher static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, 60c3940999STom Herbert struct efx_tx_buffer *buffer, 61c3940999STom Herbert unsigned int *pkts_compl, 62c3940999STom Herbert unsigned int *bytes_compl) 63874aeea5SJeff Kirsher { 64874aeea5SJeff Kirsher if (buffer->unmap_len) { 650e33d870SBen Hutchings struct device *dma_dev = &tx_queue->efx->pci_dev->dev; 662acdb92eSAlexandre Rames dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset; 677668ff9cSBen Hutchings if (buffer->flags & EFX_TX_BUF_MAP_SINGLE) 680e33d870SBen Hutchings dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len, 690e33d870SBen Hutchings DMA_TO_DEVICE); 70874aeea5SJeff Kirsher else 710e33d870SBen Hutchings dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len, 720e33d870SBen Hutchings DMA_TO_DEVICE); 73874aeea5SJeff Kirsher buffer->unmap_len = 0; 74874aeea5SJeff Kirsher } 75874aeea5SJeff Kirsher 767668ff9cSBen Hutchings if (buffer->flags & EFX_TX_BUF_SKB) { 77b9b603d4SMartin Habets struct sk_buff *skb = (struct sk_buff *)buffer->skb; 78b9b603d4SMartin Habets 79d4a7a889SBert Kenward EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl); 80c3940999STom Herbert (*pkts_compl)++; 81b9b603d4SMartin Habets (*bytes_compl) += skb->len; 82b9b603d4SMartin Habets if (tx_queue->timestamping && 83b9b603d4SMartin Habets (tx_queue->completed_timestamp_major || 84b9b603d4SMartin Habets tx_queue->completed_timestamp_minor)) { 85b9b603d4SMartin Habets struct skb_shared_hwtstamps hwtstamp; 86b9b603d4SMartin Habets 87b9b603d4SMartin Habets hwtstamp.hwtstamp = 88b9b603d4SMartin Habets efx_ptp_nic_to_kernel_time(tx_queue); 89b9b603d4SMartin Habets skb_tstamp_tx(skb, &hwtstamp); 90b9b603d4SMartin Habets 91b9b603d4SMartin Habets tx_queue->completed_timestamp_major = 0; 92b9b603d4SMartin Habets tx_queue->completed_timestamp_minor = 0; 93b9b603d4SMartin Habets } 944ef6dae4SRick Jones dev_consume_skb_any((struct sk_buff *)buffer->skb); 95874aeea5SJeff Kirsher netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev, 96874aeea5SJeff Kirsher "TX queue %d transmission id %x complete\n", 97874aeea5SJeff Kirsher tx_queue->queue, tx_queue->read_count); 98874aeea5SJeff Kirsher } 997668ff9cSBen Hutchings 100f7251a9cSBen Hutchings buffer->len = 0; 101f7251a9cSBen Hutchings buffer->flags = 0; 102874aeea5SJeff Kirsher } 103874aeea5SJeff Kirsher 1047e6d06f0SBen Hutchings unsigned int efx_tx_max_skb_descs(struct efx_nic *efx) 1057e6d06f0SBen Hutchings { 1067e6d06f0SBen Hutchings /* Header and payload descriptor for each output segment, plus 1077e6d06f0SBen Hutchings * one for every input fragment boundary within a segment 1087e6d06f0SBen Hutchings */ 1097e6d06f0SBen Hutchings unsigned int max_descs = EFX_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS; 1107e6d06f0SBen Hutchings 1115a6681e2SEdward Cree /* Possibly one more per segment for option descriptors */ 1125a6681e2SEdward Cree if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 1137e6d06f0SBen Hutchings max_descs += EFX_TSO_MAX_SEGS; 1147e6d06f0SBen Hutchings 1157e6d06f0SBen Hutchings /* Possibly more for PCIe page boundaries within input fragments */ 1167e6d06f0SBen Hutchings if (PAGE_SIZE > EFX_PAGE_SIZE) 1177e6d06f0SBen Hutchings max_descs += max_t(unsigned int, MAX_SKB_FRAGS, 1187e6d06f0SBen Hutchings DIV_ROUND_UP(GSO_MAX_SIZE, EFX_PAGE_SIZE)); 1197e6d06f0SBen Hutchings 1207e6d06f0SBen Hutchings return max_descs; 1217e6d06f0SBen Hutchings } 1227e6d06f0SBen Hutchings 12314bf718fSBen Hutchings static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1) 12414bf718fSBen Hutchings { 12514bf718fSBen Hutchings /* We need to consider both queues that the net core sees as one */ 12614bf718fSBen Hutchings struct efx_tx_queue *txq2 = efx_tx_queue_partner(txq1); 12714bf718fSBen Hutchings struct efx_nic *efx = txq1->efx; 12814bf718fSBen Hutchings unsigned int fill_level; 12914bf718fSBen Hutchings 13014bf718fSBen Hutchings fill_level = max(txq1->insert_count - txq1->old_read_count, 13114bf718fSBen Hutchings txq2->insert_count - txq2->old_read_count); 13214bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) 13314bf718fSBen Hutchings return; 13414bf718fSBen Hutchings 13514bf718fSBen Hutchings /* We used the stale old_read_count above, which gives us a 13614bf718fSBen Hutchings * pessimistic estimate of the fill level (which may even 13714bf718fSBen Hutchings * validly be >= efx->txq_entries). Now try again using 13814bf718fSBen Hutchings * read_count (more likely to be a cache miss). 13914bf718fSBen Hutchings * 14014bf718fSBen Hutchings * If we read read_count and then conditionally stop the 14114bf718fSBen Hutchings * queue, it is possible for the completion path to race with 14214bf718fSBen Hutchings * us and complete all outstanding descriptors in the middle, 14314bf718fSBen Hutchings * after which there will be no more completions to wake it. 14414bf718fSBen Hutchings * Therefore we stop the queue first, then read read_count 14514bf718fSBen Hutchings * (with a memory barrier to ensure the ordering), then 14614bf718fSBen Hutchings * restart the queue if the fill level turns out to be low 14714bf718fSBen Hutchings * enough. 14814bf718fSBen Hutchings */ 14914bf718fSBen Hutchings netif_tx_stop_queue(txq1->core_txq); 15014bf718fSBen Hutchings smp_mb(); 1516aa7de05SMark Rutland txq1->old_read_count = READ_ONCE(txq1->read_count); 1526aa7de05SMark Rutland txq2->old_read_count = READ_ONCE(txq2->read_count); 15314bf718fSBen Hutchings 15414bf718fSBen Hutchings fill_level = max(txq1->insert_count - txq1->old_read_count, 15514bf718fSBen Hutchings txq2->insert_count - txq2->old_read_count); 156e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(fill_level >= efx->txq_entries); 15714bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) { 15814bf718fSBen Hutchings smp_mb(); 15914bf718fSBen Hutchings if (likely(!efx->loopback_selftest)) 16014bf718fSBen Hutchings netif_tx_start_queue(txq1->core_txq); 16114bf718fSBen Hutchings } 16214bf718fSBen Hutchings } 16314bf718fSBen Hutchings 164e9117e50SBert Kenward static int efx_enqueue_skb_copy(struct efx_tx_queue *tx_queue, 165e9117e50SBert Kenward struct sk_buff *skb) 166e9117e50SBert Kenward { 167e9117e50SBert Kenward unsigned int copy_len = skb->len; 168e9117e50SBert Kenward struct efx_tx_buffer *buffer; 169e9117e50SBert Kenward u8 *copy_buffer; 170e9117e50SBert Kenward int rc; 171e9117e50SBert Kenward 172e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(copy_len > EFX_TX_CB_SIZE); 173e9117e50SBert Kenward 174e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue); 175e9117e50SBert Kenward 176e9117e50SBert Kenward copy_buffer = efx_tx_get_copy_buffer(tx_queue, buffer); 177e9117e50SBert Kenward if (unlikely(!copy_buffer)) 178e9117e50SBert Kenward return -ENOMEM; 179e9117e50SBert Kenward 180e9117e50SBert Kenward rc = skb_copy_bits(skb, 0, copy_buffer, copy_len); 181e9117e50SBert Kenward EFX_WARN_ON_PARANOID(rc); 182e9117e50SBert Kenward buffer->len = copy_len; 183e9117e50SBert Kenward 184e9117e50SBert Kenward buffer->skb = skb; 185e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB; 186e9117e50SBert Kenward 187e9117e50SBert Kenward ++tx_queue->insert_count; 188e9117e50SBert Kenward return rc; 189e9117e50SBert Kenward } 190e9117e50SBert Kenward 191ee45fd92SJon Cooper #ifdef EFX_USE_PIO 192ee45fd92SJon Cooper 193ee45fd92SJon Cooper struct efx_short_copy_buffer { 194ee45fd92SJon Cooper int used; 195ee45fd92SJon Cooper u8 buf[L1_CACHE_BYTES]; 196ee45fd92SJon Cooper }; 197ee45fd92SJon Cooper 198ee45fd92SJon Cooper /* Copy to PIO, respecting that writes to PIO buffers must be dword aligned. 199ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 200ee45fd92SJon Cooper */ 201ee45fd92SJon Cooper static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf, 202ee45fd92SJon Cooper u8 *data, int len, 203ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 204ee45fd92SJon Cooper { 205ee45fd92SJon Cooper int block_len = len & ~(sizeof(copy_buf->buf) - 1); 206ee45fd92SJon Cooper 2074984c237SBen Hutchings __iowrite64_copy(*piobuf, data, block_len >> 3); 208ee45fd92SJon Cooper *piobuf += block_len; 209ee45fd92SJon Cooper len -= block_len; 210ee45fd92SJon Cooper 211ee45fd92SJon Cooper if (len) { 212ee45fd92SJon Cooper data += block_len; 213ee45fd92SJon Cooper BUG_ON(copy_buf->used); 214ee45fd92SJon Cooper BUG_ON(len > sizeof(copy_buf->buf)); 215ee45fd92SJon Cooper memcpy(copy_buf->buf, data, len); 216ee45fd92SJon Cooper copy_buf->used = len; 217ee45fd92SJon Cooper } 218ee45fd92SJon Cooper } 219ee45fd92SJon Cooper 220ee45fd92SJon Cooper /* Copy to PIO, respecting dword alignment, popping data from copy buffer first. 221ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 222ee45fd92SJon Cooper */ 223ee45fd92SJon Cooper static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf, 224ee45fd92SJon Cooper u8 *data, int len, 225ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 226ee45fd92SJon Cooper { 227ee45fd92SJon Cooper if (copy_buf->used) { 228ee45fd92SJon Cooper /* if the copy buffer is partially full, fill it up and write */ 229ee45fd92SJon Cooper int copy_to_buf = 230ee45fd92SJon Cooper min_t(int, sizeof(copy_buf->buf) - copy_buf->used, len); 231ee45fd92SJon Cooper 232ee45fd92SJon Cooper memcpy(copy_buf->buf + copy_buf->used, data, copy_to_buf); 233ee45fd92SJon Cooper copy_buf->used += copy_to_buf; 234ee45fd92SJon Cooper 235ee45fd92SJon Cooper /* if we didn't fill it up then we're done for now */ 236ee45fd92SJon Cooper if (copy_buf->used < sizeof(copy_buf->buf)) 237ee45fd92SJon Cooper return; 238ee45fd92SJon Cooper 2394984c237SBen Hutchings __iowrite64_copy(*piobuf, copy_buf->buf, 2404984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 241ee45fd92SJon Cooper *piobuf += sizeof(copy_buf->buf); 242ee45fd92SJon Cooper data += copy_to_buf; 243ee45fd92SJon Cooper len -= copy_to_buf; 244ee45fd92SJon Cooper copy_buf->used = 0; 245ee45fd92SJon Cooper } 246ee45fd92SJon Cooper 247ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, data, len, copy_buf); 248ee45fd92SJon Cooper } 249ee45fd92SJon Cooper 250ee45fd92SJon Cooper static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf, 251ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 252ee45fd92SJon Cooper { 253ee45fd92SJon Cooper /* if there's anything in it, write the whole buffer, including junk */ 254ee45fd92SJon Cooper if (copy_buf->used) 2554984c237SBen Hutchings __iowrite64_copy(piobuf, copy_buf->buf, 2564984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 257ee45fd92SJon Cooper } 258ee45fd92SJon Cooper 259ee45fd92SJon Cooper /* Traverse skb structure and copy fragments in to PIO buffer. 260ee45fd92SJon Cooper * Advances piobuf pointer. 261ee45fd92SJon Cooper */ 262ee45fd92SJon Cooper static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb, 263ee45fd92SJon Cooper u8 __iomem **piobuf, 264ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 265ee45fd92SJon Cooper { 266ee45fd92SJon Cooper int i; 267ee45fd92SJon Cooper 268ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, skb->data, skb_headlen(skb), 269ee45fd92SJon Cooper copy_buf); 270ee45fd92SJon Cooper 271ee45fd92SJon Cooper for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) { 272ee45fd92SJon Cooper skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 273ee45fd92SJon Cooper u8 *vaddr; 274ee45fd92SJon Cooper 275ee45fd92SJon Cooper vaddr = kmap_atomic(skb_frag_page(f)); 276ee45fd92SJon Cooper 277ee45fd92SJon Cooper efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + f->page_offset, 278ee45fd92SJon Cooper skb_frag_size(f), copy_buf); 279ee45fd92SJon Cooper kunmap_atomic(vaddr); 280ee45fd92SJon Cooper } 281ee45fd92SJon Cooper 282e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list); 283ee45fd92SJon Cooper } 284ee45fd92SJon Cooper 285e9117e50SBert Kenward static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, 286e9117e50SBert Kenward struct sk_buff *skb) 287ee45fd92SJon Cooper { 288ee45fd92SJon Cooper struct efx_tx_buffer *buffer = 289ee45fd92SJon Cooper efx_tx_queue_get_insert_buffer(tx_queue); 290ee45fd92SJon Cooper u8 __iomem *piobuf = tx_queue->piobuf; 291ee45fd92SJon Cooper 292ee45fd92SJon Cooper /* Copy to PIO buffer. Ensure the writes are padded to the end 293ee45fd92SJon Cooper * of a cache line, as this is required for write-combining to be 294ee45fd92SJon Cooper * effective on at least x86. 295ee45fd92SJon Cooper */ 296ee45fd92SJon Cooper 297ee45fd92SJon Cooper if (skb_shinfo(skb)->nr_frags) { 298ee45fd92SJon Cooper /* The size of the copy buffer will ensure all writes 299ee45fd92SJon Cooper * are the size of a cache line. 300ee45fd92SJon Cooper */ 301ee45fd92SJon Cooper struct efx_short_copy_buffer copy_buf; 302ee45fd92SJon Cooper 303ee45fd92SJon Cooper copy_buf.used = 0; 304ee45fd92SJon Cooper 305ee45fd92SJon Cooper efx_skb_copy_bits_to_pio(tx_queue->efx, skb, 306ee45fd92SJon Cooper &piobuf, ©_buf); 307ee45fd92SJon Cooper efx_flush_copy_buffer(tx_queue->efx, piobuf, ©_buf); 308ee45fd92SJon Cooper } else { 309ee45fd92SJon Cooper /* Pad the write to the size of a cache line. 310e9117e50SBert Kenward * We can do this because we know the skb_shared_info struct is 311ee45fd92SJon Cooper * after the source, and the destination buffer is big enough. 312ee45fd92SJon Cooper */ 313ee45fd92SJon Cooper BUILD_BUG_ON(L1_CACHE_BYTES > 314ee45fd92SJon Cooper SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 3154984c237SBen Hutchings __iowrite64_copy(tx_queue->piobuf, skb->data, 3164984c237SBen Hutchings ALIGN(skb->len, L1_CACHE_BYTES) >> 3); 317ee45fd92SJon Cooper } 318ee45fd92SJon Cooper 319e9117e50SBert Kenward buffer->skb = skb; 320e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | EFX_TX_BUF_OPTION; 321e9117e50SBert Kenward 322ee45fd92SJon Cooper EFX_POPULATE_QWORD_5(buffer->option, 323ee45fd92SJon Cooper ESF_DZ_TX_DESC_IS_OPT, 1, 324ee45fd92SJon Cooper ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_PIO, 325ee45fd92SJon Cooper ESF_DZ_TX_PIO_CONT, 0, 326ee45fd92SJon Cooper ESF_DZ_TX_PIO_BYTE_CNT, skb->len, 327ee45fd92SJon Cooper ESF_DZ_TX_PIO_BUF_ADDR, 328ee45fd92SJon Cooper tx_queue->piobuf_offset); 329ee45fd92SJon Cooper ++tx_queue->insert_count; 330e9117e50SBert Kenward return 0; 331ee45fd92SJon Cooper } 332ee45fd92SJon Cooper #endif /* EFX_USE_PIO */ 333ee45fd92SJon Cooper 334e9117e50SBert Kenward static struct efx_tx_buffer *efx_tx_map_chunk(struct efx_tx_queue *tx_queue, 335e9117e50SBert Kenward dma_addr_t dma_addr, 336e9117e50SBert Kenward size_t len) 337e9117e50SBert Kenward { 338e9117e50SBert Kenward const struct efx_nic_type *nic_type = tx_queue->efx->type; 339e9117e50SBert Kenward struct efx_tx_buffer *buffer; 340e9117e50SBert Kenward unsigned int dma_len; 341e9117e50SBert Kenward 342e9117e50SBert Kenward /* Map the fragment taking account of NIC-dependent DMA limits. */ 343e9117e50SBert Kenward do { 344e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue); 345e9117e50SBert Kenward dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len); 346e9117e50SBert Kenward 347e9117e50SBert Kenward buffer->len = dma_len; 348e9117e50SBert Kenward buffer->dma_addr = dma_addr; 349e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_CONT; 350e9117e50SBert Kenward len -= dma_len; 351e9117e50SBert Kenward dma_addr += dma_len; 352e9117e50SBert Kenward ++tx_queue->insert_count; 353e9117e50SBert Kenward } while (len); 354e9117e50SBert Kenward 355e9117e50SBert Kenward return buffer; 356e9117e50SBert Kenward } 357e9117e50SBert Kenward 358e9117e50SBert Kenward /* Map all data from an SKB for DMA and create descriptors on the queue. 359e9117e50SBert Kenward */ 360e9117e50SBert Kenward static int efx_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb, 361e9117e50SBert Kenward unsigned int segment_count) 362e9117e50SBert Kenward { 363e9117e50SBert Kenward struct efx_nic *efx = tx_queue->efx; 364e9117e50SBert Kenward struct device *dma_dev = &efx->pci_dev->dev; 365e9117e50SBert Kenward unsigned int frag_index, nr_frags; 366e9117e50SBert Kenward dma_addr_t dma_addr, unmap_addr; 367e9117e50SBert Kenward unsigned short dma_flags; 368e9117e50SBert Kenward size_t len, unmap_len; 369e9117e50SBert Kenward 370e9117e50SBert Kenward nr_frags = skb_shinfo(skb)->nr_frags; 371e9117e50SBert Kenward frag_index = 0; 372e9117e50SBert Kenward 373e9117e50SBert Kenward /* Map header data. */ 374e9117e50SBert Kenward len = skb_headlen(skb); 375e9117e50SBert Kenward dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE); 376e9117e50SBert Kenward dma_flags = EFX_TX_BUF_MAP_SINGLE; 377e9117e50SBert Kenward unmap_len = len; 378e9117e50SBert Kenward unmap_addr = dma_addr; 379e9117e50SBert Kenward 380e9117e50SBert Kenward if (unlikely(dma_mapping_error(dma_dev, dma_addr))) 381e9117e50SBert Kenward return -EIO; 382e9117e50SBert Kenward 383e9117e50SBert Kenward if (segment_count) { 384e9117e50SBert Kenward /* For TSO we need to put the header in to a separate 385e9117e50SBert Kenward * descriptor. Map this separately if necessary. 386e9117e50SBert Kenward */ 387e9117e50SBert Kenward size_t header_len = skb_transport_header(skb) - skb->data + 388e9117e50SBert Kenward (tcp_hdr(skb)->doff << 2u); 389e9117e50SBert Kenward 390e9117e50SBert Kenward if (header_len != len) { 391e9117e50SBert Kenward tx_queue->tso_long_headers++; 392e9117e50SBert Kenward efx_tx_map_chunk(tx_queue, dma_addr, header_len); 393e9117e50SBert Kenward len -= header_len; 394e9117e50SBert Kenward dma_addr += header_len; 395e9117e50SBert Kenward } 396e9117e50SBert Kenward } 397e9117e50SBert Kenward 398e9117e50SBert Kenward /* Add descriptors for each fragment. */ 399e9117e50SBert Kenward do { 400e9117e50SBert Kenward struct efx_tx_buffer *buffer; 401e9117e50SBert Kenward skb_frag_t *fragment; 402e9117e50SBert Kenward 403e9117e50SBert Kenward buffer = efx_tx_map_chunk(tx_queue, dma_addr, len); 404e9117e50SBert Kenward 405e9117e50SBert Kenward /* The final descriptor for a fragment is responsible for 406e9117e50SBert Kenward * unmapping the whole fragment. 407e9117e50SBert Kenward */ 408e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_CONT | dma_flags; 409e9117e50SBert Kenward buffer->unmap_len = unmap_len; 410e9117e50SBert Kenward buffer->dma_offset = buffer->dma_addr - unmap_addr; 411e9117e50SBert Kenward 412e9117e50SBert Kenward if (frag_index >= nr_frags) { 413e9117e50SBert Kenward /* Store SKB details with the final buffer for 414e9117e50SBert Kenward * the completion. 415e9117e50SBert Kenward */ 416e9117e50SBert Kenward buffer->skb = skb; 417e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | dma_flags; 418e9117e50SBert Kenward return 0; 419e9117e50SBert Kenward } 420e9117e50SBert Kenward 421e9117e50SBert Kenward /* Move on to the next fragment. */ 422e9117e50SBert Kenward fragment = &skb_shinfo(skb)->frags[frag_index++]; 423e9117e50SBert Kenward len = skb_frag_size(fragment); 424e9117e50SBert Kenward dma_addr = skb_frag_dma_map(dma_dev, fragment, 425e9117e50SBert Kenward 0, len, DMA_TO_DEVICE); 426e9117e50SBert Kenward dma_flags = 0; 427e9117e50SBert Kenward unmap_len = len; 428e9117e50SBert Kenward unmap_addr = dma_addr; 429e9117e50SBert Kenward 430e9117e50SBert Kenward if (unlikely(dma_mapping_error(dma_dev, dma_addr))) 431e9117e50SBert Kenward return -EIO; 432e9117e50SBert Kenward } while (1); 433e9117e50SBert Kenward } 434e9117e50SBert Kenward 4350c235113SMartin Habets /* Remove buffers put into a tx_queue for the current packet. 4360c235113SMartin Habets * None of the buffers must have an skb attached. 437e9117e50SBert Kenward */ 4380c235113SMartin Habets static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue, 4390c235113SMartin Habets unsigned int insert_count) 440e9117e50SBert Kenward { 441e9117e50SBert Kenward struct efx_tx_buffer *buffer; 442d4a7a889SBert Kenward unsigned int bytes_compl = 0; 443d4a7a889SBert Kenward unsigned int pkts_compl = 0; 444e9117e50SBert Kenward 445e9117e50SBert Kenward /* Work backwards until we hit the original insert pointer value */ 4460c235113SMartin Habets while (tx_queue->insert_count != insert_count) { 447e9117e50SBert Kenward --tx_queue->insert_count; 448e9117e50SBert Kenward buffer = __efx_tx_queue_get_insert_buffer(tx_queue); 449d4a7a889SBert Kenward efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); 450e9117e50SBert Kenward } 451e9117e50SBert Kenward } 452e9117e50SBert Kenward 45346d1efd8SEdward Cree /* 45446d1efd8SEdward Cree * Fallback to software TSO. 45546d1efd8SEdward Cree * 45646d1efd8SEdward Cree * This is used if we are unable to send a GSO packet through hardware TSO. 45746d1efd8SEdward Cree * This should only ever happen due to per-queue restrictions - unsupported 45846d1efd8SEdward Cree * packets should first be filtered by the feature flags. 45946d1efd8SEdward Cree * 46046d1efd8SEdward Cree * Returns 0 on success, error code otherwise. 46146d1efd8SEdward Cree */ 46246d1efd8SEdward Cree static int efx_tx_tso_fallback(struct efx_tx_queue *tx_queue, 46346d1efd8SEdward Cree struct sk_buff *skb) 464e9117e50SBert Kenward { 46546d1efd8SEdward Cree struct sk_buff *segments, *next; 46646d1efd8SEdward Cree 46746d1efd8SEdward Cree segments = skb_gso_segment(skb, 0); 46846d1efd8SEdward Cree if (IS_ERR(segments)) 46946d1efd8SEdward Cree return PTR_ERR(segments); 47046d1efd8SEdward Cree 471f694be27SHuang Zijiang dev_consume_skb_any(skb); 47246d1efd8SEdward Cree skb = segments; 47346d1efd8SEdward Cree 47446d1efd8SEdward Cree while (skb) { 47546d1efd8SEdward Cree next = skb->next; 47646d1efd8SEdward Cree skb->next = NULL; 47746d1efd8SEdward Cree 47846d1efd8SEdward Cree efx_enqueue_skb(tx_queue, skb); 47946d1efd8SEdward Cree skb = next; 48046d1efd8SEdward Cree } 48146d1efd8SEdward Cree 48246d1efd8SEdward Cree return 0; 483e9117e50SBert Kenward } 484e9117e50SBert Kenward 485874aeea5SJeff Kirsher /* 486874aeea5SJeff Kirsher * Add a socket buffer to a TX queue 487874aeea5SJeff Kirsher * 488874aeea5SJeff Kirsher * This maps all fragments of a socket buffer for DMA and adds them to 489874aeea5SJeff Kirsher * the TX queue. The queue's insert pointer will be incremented by 490874aeea5SJeff Kirsher * the number of fragments in the socket buffer. 491874aeea5SJeff Kirsher * 492874aeea5SJeff Kirsher * If any DMA mapping fails, any mapped fragments will be unmapped, 493874aeea5SJeff Kirsher * the queue's insert pointer will be restored to its original value. 494874aeea5SJeff Kirsher * 495874aeea5SJeff Kirsher * This function is split out from efx_hard_start_xmit to allow the 496874aeea5SJeff Kirsher * loopback test to direct packets via specific TX queues. 497874aeea5SJeff Kirsher * 49814bf718fSBen Hutchings * Returns NETDEV_TX_OK. 499874aeea5SJeff Kirsher * You must hold netif_tx_lock() to call this function. 500874aeea5SJeff Kirsher */ 501874aeea5SJeff Kirsher netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) 502874aeea5SJeff Kirsher { 5030c235113SMartin Habets unsigned int old_insert_count = tx_queue->insert_count; 504f79c957aSFlorian Westphal bool xmit_more = netdev_xmit_more(); 505e9117e50SBert Kenward bool data_mapped = false; 506e9117e50SBert Kenward unsigned int segments; 507e9117e50SBert Kenward unsigned int skb_len; 50846d1efd8SEdward Cree int rc; 509874aeea5SJeff Kirsher 510e9117e50SBert Kenward skb_len = skb->len; 511e9117e50SBert Kenward segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0; 512e9117e50SBert Kenward if (segments == 1) 513e9117e50SBert Kenward segments = 0; /* Don't use TSO for a single segment. */ 514874aeea5SJeff Kirsher 515e9117e50SBert Kenward /* Handle TSO first - it's *possible* (although unlikely) that we might 516e9117e50SBert Kenward * be passed a packet to segment that's smaller than the copybreak/PIO 517e9117e50SBert Kenward * size limit. 518874aeea5SJeff Kirsher */ 519e9117e50SBert Kenward if (segments) { 520e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(!tx_queue->handle_tso); 52146d1efd8SEdward Cree rc = tx_queue->handle_tso(tx_queue, skb, &data_mapped); 52246d1efd8SEdward Cree if (rc == -EINVAL) { 52346d1efd8SEdward Cree rc = efx_tx_tso_fallback(tx_queue, skb); 52446d1efd8SEdward Cree tx_queue->tso_fallbacks++; 52546d1efd8SEdward Cree if (rc == 0) 52646d1efd8SEdward Cree return 0; 52746d1efd8SEdward Cree } 52846d1efd8SEdward Cree if (rc) 529e9117e50SBert Kenward goto err; 530e9117e50SBert Kenward #ifdef EFX_USE_PIO 531f79c957aSFlorian Westphal } else if (skb_len <= efx_piobuf_size && !xmit_more && 532e9117e50SBert Kenward efx_nic_may_tx_pio(tx_queue)) { 533e9117e50SBert Kenward /* Use PIO for short packets with an empty queue. */ 534e9117e50SBert Kenward if (efx_enqueue_skb_pio(tx_queue, skb)) 535e9117e50SBert Kenward goto err; 536e9117e50SBert Kenward tx_queue->pio_packets++; 537e9117e50SBert Kenward data_mapped = true; 538e9117e50SBert Kenward #endif 5395a6681e2SEdward Cree } else if (skb->data_len && skb_len <= EFX_TX_CB_SIZE) { 540e9117e50SBert Kenward /* Pad short packets or coalesce short fragmented packets. */ 541e9117e50SBert Kenward if (efx_enqueue_skb_copy(tx_queue, skb)) 542e9117e50SBert Kenward goto err; 543e9117e50SBert Kenward tx_queue->cb_packets++; 544e9117e50SBert Kenward data_mapped = true; 545874aeea5SJeff Kirsher } 546874aeea5SJeff Kirsher 547e9117e50SBert Kenward /* Map for DMA and create descriptors if we haven't done so already. */ 548e9117e50SBert Kenward if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments))) 549e9117e50SBert Kenward goto err; 550874aeea5SJeff Kirsher 5510c235113SMartin Habets efx_tx_maybe_stop_queue(tx_queue); 5520c235113SMartin Habets 553874aeea5SJeff Kirsher /* Pass off to hardware */ 55429e12207SEdward Cree if (__netdev_tx_sent_queue(tx_queue->core_txq, skb_len, xmit_more)) { 555b2663a4fSMartin Habets struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue); 556b2663a4fSMartin Habets 557f79c957aSFlorian Westphal /* There could be packets left on the partner queue if 558f79c957aSFlorian Westphal * xmit_more was set. If we do not push those they 559b2663a4fSMartin Habets * could be left for a long time and cause a netdev watchdog. 560b2663a4fSMartin Habets */ 561b2663a4fSMartin Habets if (txq2->xmit_more_available) 562b2663a4fSMartin Habets efx_nic_push_buffers(txq2); 563b2663a4fSMartin Habets 564874aeea5SJeff Kirsher efx_nic_push_buffers(tx_queue); 565b2663a4fSMartin Habets } else { 566f79c957aSFlorian Westphal tx_queue->xmit_more_available = xmit_more; 567b2663a4fSMartin Habets } 568874aeea5SJeff Kirsher 569e9117e50SBert Kenward if (segments) { 570e9117e50SBert Kenward tx_queue->tso_bursts++; 571e9117e50SBert Kenward tx_queue->tso_packets += segments; 572e9117e50SBert Kenward tx_queue->tx_packets += segments; 573e9117e50SBert Kenward } else { 5748ccf3800SAndrew Rybchenko tx_queue->tx_packets++; 575e9117e50SBert Kenward } 576e9117e50SBert Kenward 577874aeea5SJeff Kirsher return NETDEV_TX_OK; 578874aeea5SJeff Kirsher 579874aeea5SJeff Kirsher 580e9117e50SBert Kenward err: 5810c235113SMartin Habets efx_enqueue_unwind(tx_queue, old_insert_count); 582874aeea5SJeff Kirsher dev_kfree_skb_any(skb); 5830c235113SMartin Habets 5840c235113SMartin Habets /* If we're not expecting another transmit and we had something to push 5850c235113SMartin Habets * on this queue or a partner queue then we need to push here to get the 5860c235113SMartin Habets * previous packets out. 5870c235113SMartin Habets */ 5880c235113SMartin Habets if (!xmit_more) { 5890c235113SMartin Habets struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue); 5900c235113SMartin Habets 5910c235113SMartin Habets if (txq2->xmit_more_available) 5920c235113SMartin Habets efx_nic_push_buffers(txq2); 5930c235113SMartin Habets 5940c235113SMartin Habets efx_nic_push_buffers(tx_queue); 5950c235113SMartin Habets } 5960c235113SMartin Habets 59714bf718fSBen Hutchings return NETDEV_TX_OK; 598874aeea5SJeff Kirsher } 599874aeea5SJeff Kirsher 600874aeea5SJeff Kirsher /* Remove packets from the TX queue 601874aeea5SJeff Kirsher * 602874aeea5SJeff Kirsher * This removes packets from the TX queue, up to and including the 603874aeea5SJeff Kirsher * specified index. 604874aeea5SJeff Kirsher */ 605874aeea5SJeff Kirsher static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, 606c3940999STom Herbert unsigned int index, 607c3940999STom Herbert unsigned int *pkts_compl, 608c3940999STom Herbert unsigned int *bytes_compl) 609874aeea5SJeff Kirsher { 610874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 611874aeea5SJeff Kirsher unsigned int stop_index, read_ptr; 612874aeea5SJeff Kirsher 613874aeea5SJeff Kirsher stop_index = (index + 1) & tx_queue->ptr_mask; 614874aeea5SJeff Kirsher read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 615874aeea5SJeff Kirsher 616874aeea5SJeff Kirsher while (read_ptr != stop_index) { 617874aeea5SJeff Kirsher struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; 618ba8977bdSBen Hutchings 619ba8977bdSBen Hutchings if (!(buffer->flags & EFX_TX_BUF_OPTION) && 620ba8977bdSBen Hutchings unlikely(buffer->len == 0)) { 621874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 622874aeea5SJeff Kirsher "TX queue %d spurious TX completion id %x\n", 623874aeea5SJeff Kirsher tx_queue->queue, read_ptr); 624874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_SKIP); 625874aeea5SJeff Kirsher return; 626874aeea5SJeff Kirsher } 627874aeea5SJeff Kirsher 628c3940999STom Herbert efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); 629874aeea5SJeff Kirsher 630874aeea5SJeff Kirsher ++tx_queue->read_count; 631874aeea5SJeff Kirsher read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 632874aeea5SJeff Kirsher } 633874aeea5SJeff Kirsher } 634874aeea5SJeff Kirsher 635874aeea5SJeff Kirsher /* Initiate a packet transmission. We use one channel per CPU 636874aeea5SJeff Kirsher * (sharing when we have more CPUs than channels). On Falcon, the TX 637874aeea5SJeff Kirsher * completion events will be directed back to the CPU that transmitted 638874aeea5SJeff Kirsher * the packet, which should be cache-efficient. 639874aeea5SJeff Kirsher * 640874aeea5SJeff Kirsher * Context: non-blocking. 641874aeea5SJeff Kirsher * Note that returning anything other than NETDEV_TX_OK will cause the 642874aeea5SJeff Kirsher * OS to free the skb. 643874aeea5SJeff Kirsher */ 644874aeea5SJeff Kirsher netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, 645874aeea5SJeff Kirsher struct net_device *net_dev) 646874aeea5SJeff Kirsher { 647874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 648874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 649874aeea5SJeff Kirsher unsigned index, type; 650874aeea5SJeff Kirsher 651874aeea5SJeff Kirsher EFX_WARN_ON_PARANOID(!netif_device_present(net_dev)); 652874aeea5SJeff Kirsher 6537c236c43SStuart Hodgson /* PTP "event" packet */ 6547c236c43SStuart Hodgson if (unlikely(efx_xmit_with_hwtstamp(skb)) && 6557c236c43SStuart Hodgson unlikely(efx_ptp_is_ptp_tx(efx, skb))) { 6567c236c43SStuart Hodgson return efx_ptp_tx(efx, skb); 6577c236c43SStuart Hodgson } 6587c236c43SStuart Hodgson 659874aeea5SJeff Kirsher index = skb_get_queue_mapping(skb); 660874aeea5SJeff Kirsher type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0; 661874aeea5SJeff Kirsher if (index >= efx->n_tx_channels) { 662874aeea5SJeff Kirsher index -= efx->n_tx_channels; 663874aeea5SJeff Kirsher type |= EFX_TXQ_TYPE_HIGHPRI; 664874aeea5SJeff Kirsher } 665874aeea5SJeff Kirsher tx_queue = efx_get_tx_queue(efx, index, type); 666874aeea5SJeff Kirsher 667874aeea5SJeff Kirsher return efx_enqueue_skb(tx_queue, skb); 668874aeea5SJeff Kirsher } 669874aeea5SJeff Kirsher 670874aeea5SJeff Kirsher void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue) 671874aeea5SJeff Kirsher { 672874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 673874aeea5SJeff Kirsher 674874aeea5SJeff Kirsher /* Must be inverse of queue lookup in efx_hard_start_xmit() */ 675874aeea5SJeff Kirsher tx_queue->core_txq = 676874aeea5SJeff Kirsher netdev_get_tx_queue(efx->net_dev, 677874aeea5SJeff Kirsher tx_queue->queue / EFX_TXQ_TYPES + 678874aeea5SJeff Kirsher ((tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ? 679874aeea5SJeff Kirsher efx->n_tx_channels : 0)); 680874aeea5SJeff Kirsher } 681874aeea5SJeff Kirsher 6822572ac53SJiri Pirko int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type, 683de4784caSJiri Pirko void *type_data) 684874aeea5SJeff Kirsher { 685874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 686de4784caSJiri Pirko struct tc_mqprio_qopt *mqprio = type_data; 687874aeea5SJeff Kirsher struct efx_channel *channel; 688874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 68916e5cc64SJohn Fastabend unsigned tc, num_tc; 690874aeea5SJeff Kirsher int rc; 691874aeea5SJeff Kirsher 692575ed7d3SNogah Frankel if (type != TC_SETUP_QDISC_MQPRIO) 69338cf0426SJiri Pirko return -EOPNOTSUPP; 694e4c6734eSJohn Fastabend 695de4784caSJiri Pirko num_tc = mqprio->num_tc; 69616e5cc64SJohn Fastabend 6975a6681e2SEdward Cree if (num_tc > EFX_MAX_TX_TC) 698874aeea5SJeff Kirsher return -EINVAL; 699874aeea5SJeff Kirsher 700de4784caSJiri Pirko mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 70156f36acdSAmritha Nambiar 702874aeea5SJeff Kirsher if (num_tc == net_dev->num_tc) 703874aeea5SJeff Kirsher return 0; 704874aeea5SJeff Kirsher 705874aeea5SJeff Kirsher for (tc = 0; tc < num_tc; tc++) { 706874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; 707874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].count = efx->n_tx_channels; 708874aeea5SJeff Kirsher } 709874aeea5SJeff Kirsher 710874aeea5SJeff Kirsher if (num_tc > net_dev->num_tc) { 711874aeea5SJeff Kirsher /* Initialise high-priority queues as necessary */ 712874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 713874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, 714874aeea5SJeff Kirsher channel) { 715874aeea5SJeff Kirsher if (!(tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI)) 716874aeea5SJeff Kirsher continue; 717874aeea5SJeff Kirsher if (!tx_queue->buffer) { 718874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 719874aeea5SJeff Kirsher if (rc) 720874aeea5SJeff Kirsher return rc; 721874aeea5SJeff Kirsher } 722874aeea5SJeff Kirsher if (!tx_queue->initialised) 723874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 724874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 725874aeea5SJeff Kirsher } 726874aeea5SJeff Kirsher } 727874aeea5SJeff Kirsher } else { 728874aeea5SJeff Kirsher /* Reduce number of classes before number of queues */ 729874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 730874aeea5SJeff Kirsher } 731874aeea5SJeff Kirsher 732874aeea5SJeff Kirsher rc = netif_set_real_num_tx_queues(net_dev, 733874aeea5SJeff Kirsher max_t(int, num_tc, 1) * 734874aeea5SJeff Kirsher efx->n_tx_channels); 735874aeea5SJeff Kirsher if (rc) 736874aeea5SJeff Kirsher return rc; 737874aeea5SJeff Kirsher 738874aeea5SJeff Kirsher /* Do not destroy high-priority queues when they become 739874aeea5SJeff Kirsher * unused. We would have to flush them first, and it is 740874aeea5SJeff Kirsher * fairly difficult to flush a subset of TX queues. Leave 741874aeea5SJeff Kirsher * it to efx_fini_channels(). 742874aeea5SJeff Kirsher */ 743874aeea5SJeff Kirsher 744874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 745874aeea5SJeff Kirsher return 0; 746874aeea5SJeff Kirsher } 747874aeea5SJeff Kirsher 748874aeea5SJeff Kirsher void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) 749874aeea5SJeff Kirsher { 750874aeea5SJeff Kirsher unsigned fill_level; 751874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 75214bf718fSBen Hutchings struct efx_tx_queue *txq2; 753c3940999STom Herbert unsigned int pkts_compl = 0, bytes_compl = 0; 754874aeea5SJeff Kirsher 755e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask); 756874aeea5SJeff Kirsher 757c3940999STom Herbert efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl); 758c936835cSPeter Dunning tx_queue->pkts_compl += pkts_compl; 759c936835cSPeter Dunning tx_queue->bytes_compl += bytes_compl; 760874aeea5SJeff Kirsher 76102e12165SBen Hutchings if (pkts_compl > 1) 76202e12165SBen Hutchings ++tx_queue->merge_events; 76302e12165SBen Hutchings 76414bf718fSBen Hutchings /* See if we need to restart the netif queue. This memory 76514bf718fSBen Hutchings * barrier ensures that we write read_count (inside 76614bf718fSBen Hutchings * efx_dequeue_buffers()) before reading the queue status. 76714bf718fSBen Hutchings */ 768874aeea5SJeff Kirsher smp_mb(); 769874aeea5SJeff Kirsher if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && 770874aeea5SJeff Kirsher likely(efx->port_enabled) && 771874aeea5SJeff Kirsher likely(netif_device_present(efx->net_dev))) { 77214bf718fSBen Hutchings txq2 = efx_tx_queue_partner(tx_queue); 77314bf718fSBen Hutchings fill_level = max(tx_queue->insert_count - tx_queue->read_count, 77414bf718fSBen Hutchings txq2->insert_count - txq2->read_count); 77514bf718fSBen Hutchings if (fill_level <= efx->txq_wake_thresh) 776874aeea5SJeff Kirsher netif_tx_wake_queue(tx_queue->core_txq); 777874aeea5SJeff Kirsher } 778874aeea5SJeff Kirsher 779874aeea5SJeff Kirsher /* Check whether the hardware queue is now empty */ 780874aeea5SJeff Kirsher if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { 7816aa7de05SMark Rutland tx_queue->old_write_count = READ_ONCE(tx_queue->write_count); 782874aeea5SJeff Kirsher if (tx_queue->read_count == tx_queue->old_write_count) { 783874aeea5SJeff Kirsher smp_mb(); 784874aeea5SJeff Kirsher tx_queue->empty_read_count = 785874aeea5SJeff Kirsher tx_queue->read_count | EFX_EMPTY_COUNT_VALID; 786874aeea5SJeff Kirsher } 787874aeea5SJeff Kirsher } 788874aeea5SJeff Kirsher } 789874aeea5SJeff Kirsher 790e9117e50SBert Kenward static unsigned int efx_tx_cb_page_count(struct efx_tx_queue *tx_queue) 791f7251a9cSBen Hutchings { 792e9117e50SBert Kenward return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EFX_TX_CB_ORDER); 793f7251a9cSBen Hutchings } 794f7251a9cSBen Hutchings 795874aeea5SJeff Kirsher int efx_probe_tx_queue(struct efx_tx_queue *tx_queue) 796874aeea5SJeff Kirsher { 797874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 798874aeea5SJeff Kirsher unsigned int entries; 7997668ff9cSBen Hutchings int rc; 800874aeea5SJeff Kirsher 801874aeea5SJeff Kirsher /* Create the smallest power-of-two aligned ring */ 802874aeea5SJeff Kirsher entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE); 803e01b16a7SEdward Cree EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); 804874aeea5SJeff Kirsher tx_queue->ptr_mask = entries - 1; 805874aeea5SJeff Kirsher 806874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 807874aeea5SJeff Kirsher "creating TX queue %d size %#x mask %#x\n", 808874aeea5SJeff Kirsher tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask); 809874aeea5SJeff Kirsher 810874aeea5SJeff Kirsher /* Allocate software ring */ 811c2e4e25aSThomas Meyer tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer), 812874aeea5SJeff Kirsher GFP_KERNEL); 813874aeea5SJeff Kirsher if (!tx_queue->buffer) 814874aeea5SJeff Kirsher return -ENOMEM; 815874aeea5SJeff Kirsher 816e9117e50SBert Kenward tx_queue->cb_page = kcalloc(efx_tx_cb_page_count(tx_queue), 817e9117e50SBert Kenward sizeof(tx_queue->cb_page[0]), GFP_KERNEL); 818e9117e50SBert Kenward if (!tx_queue->cb_page) { 819f7251a9cSBen Hutchings rc = -ENOMEM; 820f7251a9cSBen Hutchings goto fail1; 821f7251a9cSBen Hutchings } 822f7251a9cSBen Hutchings 823874aeea5SJeff Kirsher /* Allocate hardware ring */ 824874aeea5SJeff Kirsher rc = efx_nic_probe_tx(tx_queue); 825874aeea5SJeff Kirsher if (rc) 826f7251a9cSBen Hutchings goto fail2; 827874aeea5SJeff Kirsher 828874aeea5SJeff Kirsher return 0; 829874aeea5SJeff Kirsher 830f7251a9cSBen Hutchings fail2: 831e9117e50SBert Kenward kfree(tx_queue->cb_page); 832e9117e50SBert Kenward tx_queue->cb_page = NULL; 833f7251a9cSBen Hutchings fail1: 834874aeea5SJeff Kirsher kfree(tx_queue->buffer); 835874aeea5SJeff Kirsher tx_queue->buffer = NULL; 836874aeea5SJeff Kirsher return rc; 837874aeea5SJeff Kirsher } 838874aeea5SJeff Kirsher 839874aeea5SJeff Kirsher void efx_init_tx_queue(struct efx_tx_queue *tx_queue) 840874aeea5SJeff Kirsher { 841e9117e50SBert Kenward struct efx_nic *efx = tx_queue->efx; 842e9117e50SBert Kenward 843e9117e50SBert Kenward netif_dbg(efx, drv, efx->net_dev, 844874aeea5SJeff Kirsher "initialising TX queue %d\n", tx_queue->queue); 845874aeea5SJeff Kirsher 846874aeea5SJeff Kirsher tx_queue->insert_count = 0; 847874aeea5SJeff Kirsher tx_queue->write_count = 0; 848de1deff9SEdward Cree tx_queue->packet_write_count = 0; 849874aeea5SJeff Kirsher tx_queue->old_write_count = 0; 850874aeea5SJeff Kirsher tx_queue->read_count = 0; 851874aeea5SJeff Kirsher tx_queue->old_read_count = 0; 852874aeea5SJeff Kirsher tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID; 853b2663a4fSMartin Habets tx_queue->xmit_more_available = false; 8542935e3c3SEdward Cree tx_queue->timestamping = (efx_ptp_use_mac_tx_timestamps(efx) && 8552935e3c3SEdward Cree tx_queue->channel == efx_ptp_channel(efx)); 856b9b603d4SMartin Habets tx_queue->completed_desc_ptr = tx_queue->ptr_mask; 857b9b603d4SMartin Habets tx_queue->completed_timestamp_major = 0; 858b9b603d4SMartin Habets tx_queue->completed_timestamp_minor = 0; 859874aeea5SJeff Kirsher 860e9117e50SBert Kenward /* Set up default function pointers. These may get replaced by 861e9117e50SBert Kenward * efx_nic_init_tx() based off NIC/queue capabilities. 862e9117e50SBert Kenward */ 86346d1efd8SEdward Cree tx_queue->handle_tso = efx_enqueue_skb_tso; 864e9117e50SBert Kenward 865874aeea5SJeff Kirsher /* Set up TX descriptor ring */ 866874aeea5SJeff Kirsher efx_nic_init_tx(tx_queue); 867874aeea5SJeff Kirsher 868874aeea5SJeff Kirsher tx_queue->initialised = true; 869874aeea5SJeff Kirsher } 870874aeea5SJeff Kirsher 871e42c3d85SBen Hutchings void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) 872874aeea5SJeff Kirsher { 873874aeea5SJeff Kirsher struct efx_tx_buffer *buffer; 874874aeea5SJeff Kirsher 875e42c3d85SBen Hutchings netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, 876e42c3d85SBen Hutchings "shutting down TX queue %d\n", tx_queue->queue); 877e42c3d85SBen Hutchings 878874aeea5SJeff Kirsher if (!tx_queue->buffer) 879874aeea5SJeff Kirsher return; 880874aeea5SJeff Kirsher 881874aeea5SJeff Kirsher /* Free any buffers left in the ring */ 882874aeea5SJeff Kirsher while (tx_queue->read_count != tx_queue->write_count) { 883c3940999STom Herbert unsigned int pkts_compl = 0, bytes_compl = 0; 884874aeea5SJeff Kirsher buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; 885c3940999STom Herbert efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); 886874aeea5SJeff Kirsher 887874aeea5SJeff Kirsher ++tx_queue->read_count; 888874aeea5SJeff Kirsher } 889b2663a4fSMartin Habets tx_queue->xmit_more_available = false; 890c3940999STom Herbert netdev_tx_reset_queue(tx_queue->core_txq); 891874aeea5SJeff Kirsher } 892874aeea5SJeff Kirsher 893874aeea5SJeff Kirsher void efx_remove_tx_queue(struct efx_tx_queue *tx_queue) 894874aeea5SJeff Kirsher { 895f7251a9cSBen Hutchings int i; 896f7251a9cSBen Hutchings 897874aeea5SJeff Kirsher if (!tx_queue->buffer) 898874aeea5SJeff Kirsher return; 899874aeea5SJeff Kirsher 900874aeea5SJeff Kirsher netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, 901874aeea5SJeff Kirsher "destroying TX queue %d\n", tx_queue->queue); 902874aeea5SJeff Kirsher efx_nic_remove_tx(tx_queue); 903874aeea5SJeff Kirsher 904e9117e50SBert Kenward if (tx_queue->cb_page) { 905e9117e50SBert Kenward for (i = 0; i < efx_tx_cb_page_count(tx_queue); i++) 906f7251a9cSBen Hutchings efx_nic_free_buffer(tx_queue->efx, 907e9117e50SBert Kenward &tx_queue->cb_page[i]); 908e9117e50SBert Kenward kfree(tx_queue->cb_page); 909e9117e50SBert Kenward tx_queue->cb_page = NULL; 910f7251a9cSBen Hutchings } 911f7251a9cSBen Hutchings 912874aeea5SJeff Kirsher kfree(tx_queue->buffer); 913874aeea5SJeff Kirsher tx_queue->buffer = NULL; 914874aeea5SJeff Kirsher } 915