1874aeea5SJeff Kirsher /**************************************************************************** 2f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 3874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 4f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 5874aeea5SJeff Kirsher * 6874aeea5SJeff Kirsher * This program is free software; you can redistribute it and/or modify it 7874aeea5SJeff Kirsher * under the terms of the GNU General Public License version 2 as published 8874aeea5SJeff Kirsher * by the Free Software Foundation, incorporated herein by reference. 9874aeea5SJeff Kirsher */ 10874aeea5SJeff Kirsher 11874aeea5SJeff Kirsher #include <linux/pci.h> 12874aeea5SJeff Kirsher #include <linux/tcp.h> 13874aeea5SJeff Kirsher #include <linux/ip.h> 14874aeea5SJeff Kirsher #include <linux/in.h> 15874aeea5SJeff Kirsher #include <linux/ipv6.h> 16874aeea5SJeff Kirsher #include <linux/slab.h> 17874aeea5SJeff Kirsher #include <net/ipv6.h> 18874aeea5SJeff Kirsher #include <linux/if_ether.h> 19874aeea5SJeff Kirsher #include <linux/highmem.h> 20183233beSBen Hutchings #include <linux/cache.h> 21874aeea5SJeff Kirsher #include "net_driver.h" 22874aeea5SJeff Kirsher #include "efx.h" 23183233beSBen Hutchings #include "io.h" 24874aeea5SJeff Kirsher #include "nic.h" 25e9117e50SBert Kenward #include "tx.h" 26874aeea5SJeff Kirsher #include "workarounds.h" 27dfa50be9SBen Hutchings #include "ef10_regs.h" 28874aeea5SJeff Kirsher 29183233beSBen Hutchings #ifdef EFX_USE_PIO 30183233beSBen Hutchings 31183233beSBen Hutchings #define EFX_PIOBUF_SIZE_MAX ER_DZ_TX_PIOBUF_SIZE 32183233beSBen Hutchings #define EFX_PIOBUF_SIZE_DEF ALIGN(256, L1_CACHE_BYTES) 33183233beSBen Hutchings unsigned int efx_piobuf_size __read_mostly = EFX_PIOBUF_SIZE_DEF; 34183233beSBen Hutchings 35183233beSBen Hutchings #endif /* EFX_USE_PIO */ 36183233beSBen Hutchings 37e9117e50SBert Kenward static inline u8 *efx_tx_get_copy_buffer(struct efx_tx_queue *tx_queue, 38e9117e50SBert Kenward struct efx_tx_buffer *buffer) 390fe5565bSBen Hutchings { 40e9117e50SBert Kenward unsigned int index = efx_tx_queue_get_insert_index(tx_queue); 41e9117e50SBert Kenward struct efx_buffer *page_buf = 42e9117e50SBert Kenward &tx_queue->cb_page[index >> (PAGE_SHIFT - EFX_TX_CB_ORDER)]; 43e9117e50SBert Kenward unsigned int offset = 44e9117e50SBert Kenward ((index << EFX_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); 45e9117e50SBert Kenward 46e9117e50SBert Kenward if (unlikely(!page_buf->addr) && 47e9117e50SBert Kenward efx_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, 48e9117e50SBert Kenward GFP_ATOMIC)) 49e9117e50SBert Kenward return NULL; 50e9117e50SBert Kenward buffer->dma_addr = page_buf->dma_addr + offset; 51e9117e50SBert Kenward buffer->unmap_len = 0; 52e9117e50SBert Kenward return (u8 *)page_buf->addr + offset; 530fe5565bSBen Hutchings } 540fe5565bSBen Hutchings 55e9117e50SBert Kenward u8 *efx_tx_get_copy_buffer_limited(struct efx_tx_queue *tx_queue, 56e9117e50SBert Kenward struct efx_tx_buffer *buffer, size_t len) 570fe5565bSBen Hutchings { 58e9117e50SBert Kenward if (len > EFX_TX_CB_SIZE) 59e9117e50SBert Kenward return NULL; 60e9117e50SBert Kenward return efx_tx_get_copy_buffer(tx_queue, buffer); 610fe5565bSBen Hutchings } 620fe5565bSBen Hutchings 63874aeea5SJeff Kirsher static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, 64c3940999STom Herbert struct efx_tx_buffer *buffer, 65c3940999STom Herbert unsigned int *pkts_compl, 66c3940999STom Herbert unsigned int *bytes_compl) 67874aeea5SJeff Kirsher { 68874aeea5SJeff Kirsher if (buffer->unmap_len) { 690e33d870SBen Hutchings struct device *dma_dev = &tx_queue->efx->pci_dev->dev; 702acdb92eSAlexandre Rames dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset; 717668ff9cSBen Hutchings if (buffer->flags & EFX_TX_BUF_MAP_SINGLE) 720e33d870SBen Hutchings dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len, 730e33d870SBen Hutchings DMA_TO_DEVICE); 74874aeea5SJeff Kirsher else 750e33d870SBen Hutchings dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len, 760e33d870SBen Hutchings DMA_TO_DEVICE); 77874aeea5SJeff Kirsher buffer->unmap_len = 0; 78874aeea5SJeff Kirsher } 79874aeea5SJeff Kirsher 807668ff9cSBen Hutchings if (buffer->flags & EFX_TX_BUF_SKB) { 81c3940999STom Herbert (*pkts_compl)++; 82c3940999STom Herbert (*bytes_compl) += buffer->skb->len; 834ef6dae4SRick Jones dev_consume_skb_any((struct sk_buff *)buffer->skb); 84874aeea5SJeff Kirsher netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev, 85874aeea5SJeff Kirsher "TX queue %d transmission id %x complete\n", 86874aeea5SJeff Kirsher tx_queue->queue, tx_queue->read_count); 87f7251a9cSBen Hutchings } else if (buffer->flags & EFX_TX_BUF_HEAP) { 88f7251a9cSBen Hutchings kfree(buffer->heap_buf); 89874aeea5SJeff Kirsher } 907668ff9cSBen Hutchings 91f7251a9cSBen Hutchings buffer->len = 0; 92f7251a9cSBen Hutchings buffer->flags = 0; 93874aeea5SJeff Kirsher } 94874aeea5SJeff Kirsher 957e6d06f0SBen Hutchings unsigned int efx_tx_max_skb_descs(struct efx_nic *efx) 967e6d06f0SBen Hutchings { 977e6d06f0SBen Hutchings /* Header and payload descriptor for each output segment, plus 987e6d06f0SBen Hutchings * one for every input fragment boundary within a segment 997e6d06f0SBen Hutchings */ 1007e6d06f0SBen Hutchings unsigned int max_descs = EFX_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS; 1017e6d06f0SBen Hutchings 102dfa50be9SBen Hutchings /* Possibly one more per segment for the alignment workaround, 103dfa50be9SBen Hutchings * or for option descriptors 104dfa50be9SBen Hutchings */ 105dfa50be9SBen Hutchings if (EFX_WORKAROUND_5391(efx) || efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 1067e6d06f0SBen Hutchings max_descs += EFX_TSO_MAX_SEGS; 1077e6d06f0SBen Hutchings 1087e6d06f0SBen Hutchings /* Possibly more for PCIe page boundaries within input fragments */ 1097e6d06f0SBen Hutchings if (PAGE_SIZE > EFX_PAGE_SIZE) 1107e6d06f0SBen Hutchings max_descs += max_t(unsigned int, MAX_SKB_FRAGS, 1117e6d06f0SBen Hutchings DIV_ROUND_UP(GSO_MAX_SIZE, EFX_PAGE_SIZE)); 1127e6d06f0SBen Hutchings 1137e6d06f0SBen Hutchings return max_descs; 1147e6d06f0SBen Hutchings } 1157e6d06f0SBen Hutchings 11614bf718fSBen Hutchings static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1) 11714bf718fSBen Hutchings { 11814bf718fSBen Hutchings /* We need to consider both queues that the net core sees as one */ 11914bf718fSBen Hutchings struct efx_tx_queue *txq2 = efx_tx_queue_partner(txq1); 12014bf718fSBen Hutchings struct efx_nic *efx = txq1->efx; 12114bf718fSBen Hutchings unsigned int fill_level; 12214bf718fSBen Hutchings 12314bf718fSBen Hutchings fill_level = max(txq1->insert_count - txq1->old_read_count, 12414bf718fSBen Hutchings txq2->insert_count - txq2->old_read_count); 12514bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) 12614bf718fSBen Hutchings return; 12714bf718fSBen Hutchings 12814bf718fSBen Hutchings /* We used the stale old_read_count above, which gives us a 12914bf718fSBen Hutchings * pessimistic estimate of the fill level (which may even 13014bf718fSBen Hutchings * validly be >= efx->txq_entries). Now try again using 13114bf718fSBen Hutchings * read_count (more likely to be a cache miss). 13214bf718fSBen Hutchings * 13314bf718fSBen Hutchings * If we read read_count and then conditionally stop the 13414bf718fSBen Hutchings * queue, it is possible for the completion path to race with 13514bf718fSBen Hutchings * us and complete all outstanding descriptors in the middle, 13614bf718fSBen Hutchings * after which there will be no more completions to wake it. 13714bf718fSBen Hutchings * Therefore we stop the queue first, then read read_count 13814bf718fSBen Hutchings * (with a memory barrier to ensure the ordering), then 13914bf718fSBen Hutchings * restart the queue if the fill level turns out to be low 14014bf718fSBen Hutchings * enough. 14114bf718fSBen Hutchings */ 14214bf718fSBen Hutchings netif_tx_stop_queue(txq1->core_txq); 14314bf718fSBen Hutchings smp_mb(); 14414bf718fSBen Hutchings txq1->old_read_count = ACCESS_ONCE(txq1->read_count); 14514bf718fSBen Hutchings txq2->old_read_count = ACCESS_ONCE(txq2->read_count); 14614bf718fSBen Hutchings 14714bf718fSBen Hutchings fill_level = max(txq1->insert_count - txq1->old_read_count, 14814bf718fSBen Hutchings txq2->insert_count - txq2->old_read_count); 14914bf718fSBen Hutchings EFX_BUG_ON_PARANOID(fill_level >= efx->txq_entries); 15014bf718fSBen Hutchings if (likely(fill_level < efx->txq_stop_thresh)) { 15114bf718fSBen Hutchings smp_mb(); 15214bf718fSBen Hutchings if (likely(!efx->loopback_selftest)) 15314bf718fSBen Hutchings netif_tx_start_queue(txq1->core_txq); 15414bf718fSBen Hutchings } 15514bf718fSBen Hutchings } 15614bf718fSBen Hutchings 157e9117e50SBert Kenward static int efx_enqueue_skb_copy(struct efx_tx_queue *tx_queue, 158e9117e50SBert Kenward struct sk_buff *skb) 159e9117e50SBert Kenward { 160e9117e50SBert Kenward unsigned int min_len = tx_queue->tx_min_size; 161e9117e50SBert Kenward unsigned int copy_len = skb->len; 162e9117e50SBert Kenward struct efx_tx_buffer *buffer; 163e9117e50SBert Kenward u8 *copy_buffer; 164e9117e50SBert Kenward int rc; 165e9117e50SBert Kenward 166e9117e50SBert Kenward EFX_BUG_ON_PARANOID(copy_len > EFX_TX_CB_SIZE); 167e9117e50SBert Kenward 168e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue); 169e9117e50SBert Kenward 170e9117e50SBert Kenward copy_buffer = efx_tx_get_copy_buffer(tx_queue, buffer); 171e9117e50SBert Kenward if (unlikely(!copy_buffer)) 172e9117e50SBert Kenward return -ENOMEM; 173e9117e50SBert Kenward 174e9117e50SBert Kenward rc = skb_copy_bits(skb, 0, copy_buffer, copy_len); 175e9117e50SBert Kenward EFX_WARN_ON_PARANOID(rc); 176e9117e50SBert Kenward if (unlikely(copy_len < min_len)) { 177e9117e50SBert Kenward memset(copy_buffer + copy_len, 0, min_len - copy_len); 178e9117e50SBert Kenward buffer->len = min_len; 179e9117e50SBert Kenward } else { 180e9117e50SBert Kenward buffer->len = copy_len; 181e9117e50SBert Kenward } 182e9117e50SBert Kenward 183e9117e50SBert Kenward buffer->skb = skb; 184e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB; 185e9117e50SBert Kenward 186e9117e50SBert Kenward ++tx_queue->insert_count; 187e9117e50SBert Kenward return rc; 188e9117e50SBert Kenward } 189e9117e50SBert Kenward 190ee45fd92SJon Cooper #ifdef EFX_USE_PIO 191ee45fd92SJon Cooper 192ee45fd92SJon Cooper struct efx_short_copy_buffer { 193ee45fd92SJon Cooper int used; 194ee45fd92SJon Cooper u8 buf[L1_CACHE_BYTES]; 195ee45fd92SJon Cooper }; 196ee45fd92SJon Cooper 197ee45fd92SJon Cooper /* Copy to PIO, respecting that writes to PIO buffers must be dword aligned. 198ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 199ee45fd92SJon Cooper */ 200ee45fd92SJon Cooper static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf, 201ee45fd92SJon Cooper u8 *data, int len, 202ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 203ee45fd92SJon Cooper { 204ee45fd92SJon Cooper int block_len = len & ~(sizeof(copy_buf->buf) - 1); 205ee45fd92SJon Cooper 2064984c237SBen Hutchings __iowrite64_copy(*piobuf, data, block_len >> 3); 207ee45fd92SJon Cooper *piobuf += block_len; 208ee45fd92SJon Cooper len -= block_len; 209ee45fd92SJon Cooper 210ee45fd92SJon Cooper if (len) { 211ee45fd92SJon Cooper data += block_len; 212ee45fd92SJon Cooper BUG_ON(copy_buf->used); 213ee45fd92SJon Cooper BUG_ON(len > sizeof(copy_buf->buf)); 214ee45fd92SJon Cooper memcpy(copy_buf->buf, data, len); 215ee45fd92SJon Cooper copy_buf->used = len; 216ee45fd92SJon Cooper } 217ee45fd92SJon Cooper } 218ee45fd92SJon Cooper 219ee45fd92SJon Cooper /* Copy to PIO, respecting dword alignment, popping data from copy buffer first. 220ee45fd92SJon Cooper * Advances piobuf pointer. Leaves additional data in the copy buffer. 221ee45fd92SJon Cooper */ 222ee45fd92SJon Cooper static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf, 223ee45fd92SJon Cooper u8 *data, int len, 224ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 225ee45fd92SJon Cooper { 226ee45fd92SJon Cooper if (copy_buf->used) { 227ee45fd92SJon Cooper /* if the copy buffer is partially full, fill it up and write */ 228ee45fd92SJon Cooper int copy_to_buf = 229ee45fd92SJon Cooper min_t(int, sizeof(copy_buf->buf) - copy_buf->used, len); 230ee45fd92SJon Cooper 231ee45fd92SJon Cooper memcpy(copy_buf->buf + copy_buf->used, data, copy_to_buf); 232ee45fd92SJon Cooper copy_buf->used += copy_to_buf; 233ee45fd92SJon Cooper 234ee45fd92SJon Cooper /* if we didn't fill it up then we're done for now */ 235ee45fd92SJon Cooper if (copy_buf->used < sizeof(copy_buf->buf)) 236ee45fd92SJon Cooper return; 237ee45fd92SJon Cooper 2384984c237SBen Hutchings __iowrite64_copy(*piobuf, copy_buf->buf, 2394984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 240ee45fd92SJon Cooper *piobuf += sizeof(copy_buf->buf); 241ee45fd92SJon Cooper data += copy_to_buf; 242ee45fd92SJon Cooper len -= copy_to_buf; 243ee45fd92SJon Cooper copy_buf->used = 0; 244ee45fd92SJon Cooper } 245ee45fd92SJon Cooper 246ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, data, len, copy_buf); 247ee45fd92SJon Cooper } 248ee45fd92SJon Cooper 249ee45fd92SJon Cooper static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf, 250ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 251ee45fd92SJon Cooper { 252ee45fd92SJon Cooper /* if there's anything in it, write the whole buffer, including junk */ 253ee45fd92SJon Cooper if (copy_buf->used) 2544984c237SBen Hutchings __iowrite64_copy(piobuf, copy_buf->buf, 2554984c237SBen Hutchings sizeof(copy_buf->buf) >> 3); 256ee45fd92SJon Cooper } 257ee45fd92SJon Cooper 258ee45fd92SJon Cooper /* Traverse skb structure and copy fragments in to PIO buffer. 259ee45fd92SJon Cooper * Advances piobuf pointer. 260ee45fd92SJon Cooper */ 261ee45fd92SJon Cooper static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb, 262ee45fd92SJon Cooper u8 __iomem **piobuf, 263ee45fd92SJon Cooper struct efx_short_copy_buffer *copy_buf) 264ee45fd92SJon Cooper { 265ee45fd92SJon Cooper int i; 266ee45fd92SJon Cooper 267ee45fd92SJon Cooper efx_memcpy_toio_aligned(efx, piobuf, skb->data, skb_headlen(skb), 268ee45fd92SJon Cooper copy_buf); 269ee45fd92SJon Cooper 270ee45fd92SJon Cooper for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) { 271ee45fd92SJon Cooper skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 272ee45fd92SJon Cooper u8 *vaddr; 273ee45fd92SJon Cooper 274ee45fd92SJon Cooper vaddr = kmap_atomic(skb_frag_page(f)); 275ee45fd92SJon Cooper 276ee45fd92SJon Cooper efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + f->page_offset, 277ee45fd92SJon Cooper skb_frag_size(f), copy_buf); 278ee45fd92SJon Cooper kunmap_atomic(vaddr); 279ee45fd92SJon Cooper } 280ee45fd92SJon Cooper 281ee45fd92SJon Cooper EFX_BUG_ON_PARANOID(skb_shinfo(skb)->frag_list); 282ee45fd92SJon Cooper } 283ee45fd92SJon Cooper 284e9117e50SBert Kenward static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, 285e9117e50SBert Kenward struct sk_buff *skb) 286ee45fd92SJon Cooper { 287ee45fd92SJon Cooper struct efx_tx_buffer *buffer = 288ee45fd92SJon Cooper efx_tx_queue_get_insert_buffer(tx_queue); 289ee45fd92SJon Cooper u8 __iomem *piobuf = tx_queue->piobuf; 290ee45fd92SJon Cooper 291ee45fd92SJon Cooper /* Copy to PIO buffer. Ensure the writes are padded to the end 292ee45fd92SJon Cooper * of a cache line, as this is required for write-combining to be 293ee45fd92SJon Cooper * effective on at least x86. 294ee45fd92SJon Cooper */ 295ee45fd92SJon Cooper 296ee45fd92SJon Cooper if (skb_shinfo(skb)->nr_frags) { 297ee45fd92SJon Cooper /* The size of the copy buffer will ensure all writes 298ee45fd92SJon Cooper * are the size of a cache line. 299ee45fd92SJon Cooper */ 300ee45fd92SJon Cooper struct efx_short_copy_buffer copy_buf; 301ee45fd92SJon Cooper 302ee45fd92SJon Cooper copy_buf.used = 0; 303ee45fd92SJon Cooper 304ee45fd92SJon Cooper efx_skb_copy_bits_to_pio(tx_queue->efx, skb, 305ee45fd92SJon Cooper &piobuf, ©_buf); 306ee45fd92SJon Cooper efx_flush_copy_buffer(tx_queue->efx, piobuf, ©_buf); 307ee45fd92SJon Cooper } else { 308ee45fd92SJon Cooper /* Pad the write to the size of a cache line. 309e9117e50SBert Kenward * We can do this because we know the skb_shared_info struct is 310ee45fd92SJon Cooper * after the source, and the destination buffer is big enough. 311ee45fd92SJon Cooper */ 312ee45fd92SJon Cooper BUILD_BUG_ON(L1_CACHE_BYTES > 313ee45fd92SJon Cooper SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 3144984c237SBen Hutchings __iowrite64_copy(tx_queue->piobuf, skb->data, 3154984c237SBen Hutchings ALIGN(skb->len, L1_CACHE_BYTES) >> 3); 316ee45fd92SJon Cooper } 317ee45fd92SJon Cooper 318e9117e50SBert Kenward buffer->skb = skb; 319e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | EFX_TX_BUF_OPTION; 320e9117e50SBert Kenward 321ee45fd92SJon Cooper EFX_POPULATE_QWORD_5(buffer->option, 322ee45fd92SJon Cooper ESF_DZ_TX_DESC_IS_OPT, 1, 323ee45fd92SJon Cooper ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_PIO, 324ee45fd92SJon Cooper ESF_DZ_TX_PIO_CONT, 0, 325ee45fd92SJon Cooper ESF_DZ_TX_PIO_BYTE_CNT, skb->len, 326ee45fd92SJon Cooper ESF_DZ_TX_PIO_BUF_ADDR, 327ee45fd92SJon Cooper tx_queue->piobuf_offset); 328ee45fd92SJon Cooper ++tx_queue->insert_count; 329e9117e50SBert Kenward return 0; 330ee45fd92SJon Cooper } 331ee45fd92SJon Cooper #endif /* EFX_USE_PIO */ 332ee45fd92SJon Cooper 333e9117e50SBert Kenward static struct efx_tx_buffer *efx_tx_map_chunk(struct efx_tx_queue *tx_queue, 334e9117e50SBert Kenward dma_addr_t dma_addr, 335e9117e50SBert Kenward size_t len) 336e9117e50SBert Kenward { 337e9117e50SBert Kenward const struct efx_nic_type *nic_type = tx_queue->efx->type; 338e9117e50SBert Kenward struct efx_tx_buffer *buffer; 339e9117e50SBert Kenward unsigned int dma_len; 340e9117e50SBert Kenward 341e9117e50SBert Kenward /* Map the fragment taking account of NIC-dependent DMA limits. */ 342e9117e50SBert Kenward do { 343e9117e50SBert Kenward buffer = efx_tx_queue_get_insert_buffer(tx_queue); 344e9117e50SBert Kenward dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len); 345e9117e50SBert Kenward 346e9117e50SBert Kenward buffer->len = dma_len; 347e9117e50SBert Kenward buffer->dma_addr = dma_addr; 348e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_CONT; 349e9117e50SBert Kenward len -= dma_len; 350e9117e50SBert Kenward dma_addr += dma_len; 351e9117e50SBert Kenward ++tx_queue->insert_count; 352e9117e50SBert Kenward } while (len); 353e9117e50SBert Kenward 354e9117e50SBert Kenward return buffer; 355e9117e50SBert Kenward } 356e9117e50SBert Kenward 357e9117e50SBert Kenward /* Map all data from an SKB for DMA and create descriptors on the queue. 358e9117e50SBert Kenward */ 359e9117e50SBert Kenward static int efx_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb, 360e9117e50SBert Kenward unsigned int segment_count) 361e9117e50SBert Kenward { 362e9117e50SBert Kenward struct efx_nic *efx = tx_queue->efx; 363e9117e50SBert Kenward struct device *dma_dev = &efx->pci_dev->dev; 364e9117e50SBert Kenward unsigned int frag_index, nr_frags; 365e9117e50SBert Kenward dma_addr_t dma_addr, unmap_addr; 366e9117e50SBert Kenward unsigned short dma_flags; 367e9117e50SBert Kenward size_t len, unmap_len; 368e9117e50SBert Kenward 369e9117e50SBert Kenward nr_frags = skb_shinfo(skb)->nr_frags; 370e9117e50SBert Kenward frag_index = 0; 371e9117e50SBert Kenward 372e9117e50SBert Kenward /* Map header data. */ 373e9117e50SBert Kenward len = skb_headlen(skb); 374e9117e50SBert Kenward dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE); 375e9117e50SBert Kenward dma_flags = EFX_TX_BUF_MAP_SINGLE; 376e9117e50SBert Kenward unmap_len = len; 377e9117e50SBert Kenward unmap_addr = dma_addr; 378e9117e50SBert Kenward 379e9117e50SBert Kenward if (unlikely(dma_mapping_error(dma_dev, dma_addr))) 380e9117e50SBert Kenward return -EIO; 381e9117e50SBert Kenward 382e9117e50SBert Kenward if (segment_count) { 383e9117e50SBert Kenward /* For TSO we need to put the header in to a separate 384e9117e50SBert Kenward * descriptor. Map this separately if necessary. 385e9117e50SBert Kenward */ 386e9117e50SBert Kenward size_t header_len = skb_transport_header(skb) - skb->data + 387e9117e50SBert Kenward (tcp_hdr(skb)->doff << 2u); 388e9117e50SBert Kenward 389e9117e50SBert Kenward if (header_len != len) { 390e9117e50SBert Kenward tx_queue->tso_long_headers++; 391e9117e50SBert Kenward efx_tx_map_chunk(tx_queue, dma_addr, header_len); 392e9117e50SBert Kenward len -= header_len; 393e9117e50SBert Kenward dma_addr += header_len; 394e9117e50SBert Kenward } 395e9117e50SBert Kenward } 396e9117e50SBert Kenward 397e9117e50SBert Kenward /* Add descriptors for each fragment. */ 398e9117e50SBert Kenward do { 399e9117e50SBert Kenward struct efx_tx_buffer *buffer; 400e9117e50SBert Kenward skb_frag_t *fragment; 401e9117e50SBert Kenward 402e9117e50SBert Kenward buffer = efx_tx_map_chunk(tx_queue, dma_addr, len); 403e9117e50SBert Kenward 404e9117e50SBert Kenward /* The final descriptor for a fragment is responsible for 405e9117e50SBert Kenward * unmapping the whole fragment. 406e9117e50SBert Kenward */ 407e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_CONT | dma_flags; 408e9117e50SBert Kenward buffer->unmap_len = unmap_len; 409e9117e50SBert Kenward buffer->dma_offset = buffer->dma_addr - unmap_addr; 410e9117e50SBert Kenward 411e9117e50SBert Kenward if (frag_index >= nr_frags) { 412e9117e50SBert Kenward /* Store SKB details with the final buffer for 413e9117e50SBert Kenward * the completion. 414e9117e50SBert Kenward */ 415e9117e50SBert Kenward buffer->skb = skb; 416e9117e50SBert Kenward buffer->flags = EFX_TX_BUF_SKB | dma_flags; 417e9117e50SBert Kenward return 0; 418e9117e50SBert Kenward } 419e9117e50SBert Kenward 420e9117e50SBert Kenward /* Move on to the next fragment. */ 421e9117e50SBert Kenward fragment = &skb_shinfo(skb)->frags[frag_index++]; 422e9117e50SBert Kenward len = skb_frag_size(fragment); 423e9117e50SBert Kenward dma_addr = skb_frag_dma_map(dma_dev, fragment, 424e9117e50SBert Kenward 0, len, DMA_TO_DEVICE); 425e9117e50SBert Kenward dma_flags = 0; 426e9117e50SBert Kenward unmap_len = len; 427e9117e50SBert Kenward unmap_addr = dma_addr; 428e9117e50SBert Kenward 429e9117e50SBert Kenward if (unlikely(dma_mapping_error(dma_dev, dma_addr))) 430e9117e50SBert Kenward return -EIO; 431e9117e50SBert Kenward } while (1); 432e9117e50SBert Kenward } 433e9117e50SBert Kenward 434e9117e50SBert Kenward /* Remove buffers put into a tx_queue. None of the buffers must have 435e9117e50SBert Kenward * an skb attached. 436e9117e50SBert Kenward */ 437e9117e50SBert Kenward static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue) 438e9117e50SBert Kenward { 439e9117e50SBert Kenward struct efx_tx_buffer *buffer; 440e9117e50SBert Kenward 441e9117e50SBert Kenward /* Work backwards until we hit the original insert pointer value */ 442e9117e50SBert Kenward while (tx_queue->insert_count != tx_queue->write_count) { 443e9117e50SBert Kenward --tx_queue->insert_count; 444e9117e50SBert Kenward buffer = __efx_tx_queue_get_insert_buffer(tx_queue); 445e9117e50SBert Kenward efx_dequeue_buffer(tx_queue, buffer, NULL, NULL); 446e9117e50SBert Kenward } 447e9117e50SBert Kenward } 448e9117e50SBert Kenward 449e9117e50SBert Kenward static int efx_tx_tso_sw(struct efx_tx_queue *tx_queue, struct sk_buff *skb, 450e9117e50SBert Kenward bool *data_mapped) 451e9117e50SBert Kenward { 452e9117e50SBert Kenward return efx_enqueue_skb_tso(tx_queue, skb, data_mapped); 453e9117e50SBert Kenward } 454e9117e50SBert Kenward 455874aeea5SJeff Kirsher /* 456874aeea5SJeff Kirsher * Add a socket buffer to a TX queue 457874aeea5SJeff Kirsher * 458874aeea5SJeff Kirsher * This maps all fragments of a socket buffer for DMA and adds them to 459874aeea5SJeff Kirsher * the TX queue. The queue's insert pointer will be incremented by 460874aeea5SJeff Kirsher * the number of fragments in the socket buffer. 461874aeea5SJeff Kirsher * 462874aeea5SJeff Kirsher * If any DMA mapping fails, any mapped fragments will be unmapped, 463874aeea5SJeff Kirsher * the queue's insert pointer will be restored to its original value. 464874aeea5SJeff Kirsher * 465874aeea5SJeff Kirsher * This function is split out from efx_hard_start_xmit to allow the 466874aeea5SJeff Kirsher * loopback test to direct packets via specific TX queues. 467874aeea5SJeff Kirsher * 46814bf718fSBen Hutchings * Returns NETDEV_TX_OK. 469874aeea5SJeff Kirsher * You must hold netif_tx_lock() to call this function. 470874aeea5SJeff Kirsher */ 471874aeea5SJeff Kirsher netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) 472874aeea5SJeff Kirsher { 473e9117e50SBert Kenward bool data_mapped = false; 474e9117e50SBert Kenward unsigned int segments; 475e9117e50SBert Kenward unsigned int skb_len; 476874aeea5SJeff Kirsher 477e9117e50SBert Kenward skb_len = skb->len; 478e9117e50SBert Kenward segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0; 479e9117e50SBert Kenward if (segments == 1) 480e9117e50SBert Kenward segments = 0; /* Don't use TSO for a single segment. */ 481874aeea5SJeff Kirsher 482e9117e50SBert Kenward /* Handle TSO first - it's *possible* (although unlikely) that we might 483e9117e50SBert Kenward * be passed a packet to segment that's smaller than the copybreak/PIO 484e9117e50SBert Kenward * size limit. 485874aeea5SJeff Kirsher */ 486e9117e50SBert Kenward if (segments) { 487e9117e50SBert Kenward EFX_BUG_ON_PARANOID(!tx_queue->handle_tso); 488e9117e50SBert Kenward if (tx_queue->handle_tso(tx_queue, skb, &data_mapped)) 489e9117e50SBert Kenward goto err; 490e9117e50SBert Kenward #ifdef EFX_USE_PIO 491e9117e50SBert Kenward } else if (skb_len <= efx_piobuf_size && !skb->xmit_more && 492e9117e50SBert Kenward efx_nic_may_tx_pio(tx_queue)) { 493e9117e50SBert Kenward /* Use PIO for short packets with an empty queue. */ 494e9117e50SBert Kenward if (efx_enqueue_skb_pio(tx_queue, skb)) 495e9117e50SBert Kenward goto err; 496e9117e50SBert Kenward tx_queue->pio_packets++; 497e9117e50SBert Kenward data_mapped = true; 498e9117e50SBert Kenward #endif 499e9117e50SBert Kenward } else if (skb_len < tx_queue->tx_min_size || 500e9117e50SBert Kenward (skb->data_len && skb_len <= EFX_TX_CB_SIZE)) { 501e9117e50SBert Kenward /* Pad short packets or coalesce short fragmented packets. */ 502e9117e50SBert Kenward if (efx_enqueue_skb_copy(tx_queue, skb)) 503e9117e50SBert Kenward goto err; 504e9117e50SBert Kenward tx_queue->cb_packets++; 505e9117e50SBert Kenward data_mapped = true; 506874aeea5SJeff Kirsher } 507874aeea5SJeff Kirsher 508e9117e50SBert Kenward /* Map for DMA and create descriptors if we haven't done so already. */ 509e9117e50SBert Kenward if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments))) 510e9117e50SBert Kenward goto err; 511874aeea5SJeff Kirsher 512e9117e50SBert Kenward /* Update BQL */ 513e9117e50SBert Kenward netdev_tx_sent_queue(tx_queue->core_txq, skb_len); 51470b33fb0SEdward Cree 515874aeea5SJeff Kirsher /* Pass off to hardware */ 516b2663a4fSMartin Habets if (!skb->xmit_more || netif_xmit_stopped(tx_queue->core_txq)) { 517b2663a4fSMartin Habets struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue); 518b2663a4fSMartin Habets 519b2663a4fSMartin Habets /* There could be packets left on the partner queue if those 520b2663a4fSMartin Habets * SKBs had skb->xmit_more set. If we do not push those they 521b2663a4fSMartin Habets * could be left for a long time and cause a netdev watchdog. 522b2663a4fSMartin Habets */ 523b2663a4fSMartin Habets if (txq2->xmit_more_available) 524b2663a4fSMartin Habets efx_nic_push_buffers(txq2); 525b2663a4fSMartin Habets 526874aeea5SJeff Kirsher efx_nic_push_buffers(tx_queue); 527b2663a4fSMartin Habets } else { 528b2663a4fSMartin Habets tx_queue->xmit_more_available = skb->xmit_more; 529b2663a4fSMartin Habets } 530874aeea5SJeff Kirsher 531e9117e50SBert Kenward if (segments) { 532e9117e50SBert Kenward tx_queue->tso_bursts++; 533e9117e50SBert Kenward tx_queue->tso_packets += segments; 534e9117e50SBert Kenward tx_queue->tx_packets += segments; 535e9117e50SBert Kenward } else { 5368ccf3800SAndrew Rybchenko tx_queue->tx_packets++; 537e9117e50SBert Kenward } 538e9117e50SBert Kenward 539e9117e50SBert Kenward efx_tx_maybe_stop_queue(tx_queue); 5408ccf3800SAndrew Rybchenko 541874aeea5SJeff Kirsher return NETDEV_TX_OK; 542874aeea5SJeff Kirsher 543874aeea5SJeff Kirsher 544e9117e50SBert Kenward err: 545e9117e50SBert Kenward efx_enqueue_unwind(tx_queue); 546874aeea5SJeff Kirsher dev_kfree_skb_any(skb); 54714bf718fSBen Hutchings return NETDEV_TX_OK; 548874aeea5SJeff Kirsher } 549874aeea5SJeff Kirsher 550874aeea5SJeff Kirsher /* Remove packets from the TX queue 551874aeea5SJeff Kirsher * 552874aeea5SJeff Kirsher * This removes packets from the TX queue, up to and including the 553874aeea5SJeff Kirsher * specified index. 554874aeea5SJeff Kirsher */ 555874aeea5SJeff Kirsher static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, 556c3940999STom Herbert unsigned int index, 557c3940999STom Herbert unsigned int *pkts_compl, 558c3940999STom Herbert unsigned int *bytes_compl) 559874aeea5SJeff Kirsher { 560874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 561874aeea5SJeff Kirsher unsigned int stop_index, read_ptr; 562874aeea5SJeff Kirsher 563874aeea5SJeff Kirsher stop_index = (index + 1) & tx_queue->ptr_mask; 564874aeea5SJeff Kirsher read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 565874aeea5SJeff Kirsher 566874aeea5SJeff Kirsher while (read_ptr != stop_index) { 567874aeea5SJeff Kirsher struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; 568ba8977bdSBen Hutchings 569ba8977bdSBen Hutchings if (!(buffer->flags & EFX_TX_BUF_OPTION) && 570ba8977bdSBen Hutchings unlikely(buffer->len == 0)) { 571874aeea5SJeff Kirsher netif_err(efx, tx_err, efx->net_dev, 572874aeea5SJeff Kirsher "TX queue %d spurious TX completion id %x\n", 573874aeea5SJeff Kirsher tx_queue->queue, read_ptr); 574874aeea5SJeff Kirsher efx_schedule_reset(efx, RESET_TYPE_TX_SKIP); 575874aeea5SJeff Kirsher return; 576874aeea5SJeff Kirsher } 577874aeea5SJeff Kirsher 578c3940999STom Herbert efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); 579874aeea5SJeff Kirsher 580874aeea5SJeff Kirsher ++tx_queue->read_count; 581874aeea5SJeff Kirsher read_ptr = tx_queue->read_count & tx_queue->ptr_mask; 582874aeea5SJeff Kirsher } 583874aeea5SJeff Kirsher } 584874aeea5SJeff Kirsher 585874aeea5SJeff Kirsher /* Initiate a packet transmission. We use one channel per CPU 586874aeea5SJeff Kirsher * (sharing when we have more CPUs than channels). On Falcon, the TX 587874aeea5SJeff Kirsher * completion events will be directed back to the CPU that transmitted 588874aeea5SJeff Kirsher * the packet, which should be cache-efficient. 589874aeea5SJeff Kirsher * 590874aeea5SJeff Kirsher * Context: non-blocking. 591874aeea5SJeff Kirsher * Note that returning anything other than NETDEV_TX_OK will cause the 592874aeea5SJeff Kirsher * OS to free the skb. 593874aeea5SJeff Kirsher */ 594874aeea5SJeff Kirsher netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, 595874aeea5SJeff Kirsher struct net_device *net_dev) 596874aeea5SJeff Kirsher { 597874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 598874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 599874aeea5SJeff Kirsher unsigned index, type; 600874aeea5SJeff Kirsher 601874aeea5SJeff Kirsher EFX_WARN_ON_PARANOID(!netif_device_present(net_dev)); 602874aeea5SJeff Kirsher 6037c236c43SStuart Hodgson /* PTP "event" packet */ 6047c236c43SStuart Hodgson if (unlikely(efx_xmit_with_hwtstamp(skb)) && 6057c236c43SStuart Hodgson unlikely(efx_ptp_is_ptp_tx(efx, skb))) { 6067c236c43SStuart Hodgson return efx_ptp_tx(efx, skb); 6077c236c43SStuart Hodgson } 6087c236c43SStuart Hodgson 609874aeea5SJeff Kirsher index = skb_get_queue_mapping(skb); 610874aeea5SJeff Kirsher type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0; 611874aeea5SJeff Kirsher if (index >= efx->n_tx_channels) { 612874aeea5SJeff Kirsher index -= efx->n_tx_channels; 613874aeea5SJeff Kirsher type |= EFX_TXQ_TYPE_HIGHPRI; 614874aeea5SJeff Kirsher } 615874aeea5SJeff Kirsher tx_queue = efx_get_tx_queue(efx, index, type); 616874aeea5SJeff Kirsher 617874aeea5SJeff Kirsher return efx_enqueue_skb(tx_queue, skb); 618874aeea5SJeff Kirsher } 619874aeea5SJeff Kirsher 620874aeea5SJeff Kirsher void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue) 621874aeea5SJeff Kirsher { 622874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 623874aeea5SJeff Kirsher 624874aeea5SJeff Kirsher /* Must be inverse of queue lookup in efx_hard_start_xmit() */ 625874aeea5SJeff Kirsher tx_queue->core_txq = 626874aeea5SJeff Kirsher netdev_get_tx_queue(efx->net_dev, 627874aeea5SJeff Kirsher tx_queue->queue / EFX_TXQ_TYPES + 628874aeea5SJeff Kirsher ((tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ? 629874aeea5SJeff Kirsher efx->n_tx_channels : 0)); 630874aeea5SJeff Kirsher } 631874aeea5SJeff Kirsher 63216e5cc64SJohn Fastabend int efx_setup_tc(struct net_device *net_dev, u32 handle, __be16 proto, 63316e5cc64SJohn Fastabend struct tc_to_netdev *ntc) 634874aeea5SJeff Kirsher { 635874aeea5SJeff Kirsher struct efx_nic *efx = netdev_priv(net_dev); 636874aeea5SJeff Kirsher struct efx_channel *channel; 637874aeea5SJeff Kirsher struct efx_tx_queue *tx_queue; 63816e5cc64SJohn Fastabend unsigned tc, num_tc; 639874aeea5SJeff Kirsher int rc; 640874aeea5SJeff Kirsher 6415eb4dce3SJohn Fastabend if (ntc->type != TC_SETUP_MQPRIO) 642e4c6734eSJohn Fastabend return -EINVAL; 643e4c6734eSJohn Fastabend 64416e5cc64SJohn Fastabend num_tc = ntc->tc; 64516e5cc64SJohn Fastabend 646874aeea5SJeff Kirsher if (efx_nic_rev(efx) < EFX_REV_FALCON_B0 || num_tc > EFX_MAX_TX_TC) 647874aeea5SJeff Kirsher return -EINVAL; 648874aeea5SJeff Kirsher 649874aeea5SJeff Kirsher if (num_tc == net_dev->num_tc) 650874aeea5SJeff Kirsher return 0; 651874aeea5SJeff Kirsher 652874aeea5SJeff Kirsher for (tc = 0; tc < num_tc; tc++) { 653874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; 654874aeea5SJeff Kirsher net_dev->tc_to_txq[tc].count = efx->n_tx_channels; 655874aeea5SJeff Kirsher } 656874aeea5SJeff Kirsher 657874aeea5SJeff Kirsher if (num_tc > net_dev->num_tc) { 658874aeea5SJeff Kirsher /* Initialise high-priority queues as necessary */ 659874aeea5SJeff Kirsher efx_for_each_channel(channel, efx) { 660874aeea5SJeff Kirsher efx_for_each_possible_channel_tx_queue(tx_queue, 661874aeea5SJeff Kirsher channel) { 662874aeea5SJeff Kirsher if (!(tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI)) 663874aeea5SJeff Kirsher continue; 664874aeea5SJeff Kirsher if (!tx_queue->buffer) { 665874aeea5SJeff Kirsher rc = efx_probe_tx_queue(tx_queue); 666874aeea5SJeff Kirsher if (rc) 667874aeea5SJeff Kirsher return rc; 668874aeea5SJeff Kirsher } 669874aeea5SJeff Kirsher if (!tx_queue->initialised) 670874aeea5SJeff Kirsher efx_init_tx_queue(tx_queue); 671874aeea5SJeff Kirsher efx_init_tx_queue_core_txq(tx_queue); 672874aeea5SJeff Kirsher } 673874aeea5SJeff Kirsher } 674874aeea5SJeff Kirsher } else { 675874aeea5SJeff Kirsher /* Reduce number of classes before number of queues */ 676874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 677874aeea5SJeff Kirsher } 678874aeea5SJeff Kirsher 679874aeea5SJeff Kirsher rc = netif_set_real_num_tx_queues(net_dev, 680874aeea5SJeff Kirsher max_t(int, num_tc, 1) * 681874aeea5SJeff Kirsher efx->n_tx_channels); 682874aeea5SJeff Kirsher if (rc) 683874aeea5SJeff Kirsher return rc; 684874aeea5SJeff Kirsher 685874aeea5SJeff Kirsher /* Do not destroy high-priority queues when they become 686874aeea5SJeff Kirsher * unused. We would have to flush them first, and it is 687874aeea5SJeff Kirsher * fairly difficult to flush a subset of TX queues. Leave 688874aeea5SJeff Kirsher * it to efx_fini_channels(). 689874aeea5SJeff Kirsher */ 690874aeea5SJeff Kirsher 691874aeea5SJeff Kirsher net_dev->num_tc = num_tc; 692874aeea5SJeff Kirsher return 0; 693874aeea5SJeff Kirsher } 694874aeea5SJeff Kirsher 695874aeea5SJeff Kirsher void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) 696874aeea5SJeff Kirsher { 697874aeea5SJeff Kirsher unsigned fill_level; 698874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 69914bf718fSBen Hutchings struct efx_tx_queue *txq2; 700c3940999STom Herbert unsigned int pkts_compl = 0, bytes_compl = 0; 701874aeea5SJeff Kirsher 702874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(index > tx_queue->ptr_mask); 703874aeea5SJeff Kirsher 704c3940999STom Herbert efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl); 705c936835cSPeter Dunning tx_queue->pkts_compl += pkts_compl; 706c936835cSPeter Dunning tx_queue->bytes_compl += bytes_compl; 707874aeea5SJeff Kirsher 70802e12165SBen Hutchings if (pkts_compl > 1) 70902e12165SBen Hutchings ++tx_queue->merge_events; 71002e12165SBen Hutchings 71114bf718fSBen Hutchings /* See if we need to restart the netif queue. This memory 71214bf718fSBen Hutchings * barrier ensures that we write read_count (inside 71314bf718fSBen Hutchings * efx_dequeue_buffers()) before reading the queue status. 71414bf718fSBen Hutchings */ 715874aeea5SJeff Kirsher smp_mb(); 716874aeea5SJeff Kirsher if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && 717874aeea5SJeff Kirsher likely(efx->port_enabled) && 718874aeea5SJeff Kirsher likely(netif_device_present(efx->net_dev))) { 71914bf718fSBen Hutchings txq2 = efx_tx_queue_partner(tx_queue); 72014bf718fSBen Hutchings fill_level = max(tx_queue->insert_count - tx_queue->read_count, 72114bf718fSBen Hutchings txq2->insert_count - txq2->read_count); 72214bf718fSBen Hutchings if (fill_level <= efx->txq_wake_thresh) 723874aeea5SJeff Kirsher netif_tx_wake_queue(tx_queue->core_txq); 724874aeea5SJeff Kirsher } 725874aeea5SJeff Kirsher 726874aeea5SJeff Kirsher /* Check whether the hardware queue is now empty */ 727874aeea5SJeff Kirsher if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { 728874aeea5SJeff Kirsher tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count); 729874aeea5SJeff Kirsher if (tx_queue->read_count == tx_queue->old_write_count) { 730874aeea5SJeff Kirsher smp_mb(); 731874aeea5SJeff Kirsher tx_queue->empty_read_count = 732874aeea5SJeff Kirsher tx_queue->read_count | EFX_EMPTY_COUNT_VALID; 733874aeea5SJeff Kirsher } 734874aeea5SJeff Kirsher } 735874aeea5SJeff Kirsher } 736874aeea5SJeff Kirsher 737e9117e50SBert Kenward static unsigned int efx_tx_cb_page_count(struct efx_tx_queue *tx_queue) 738f7251a9cSBen Hutchings { 739e9117e50SBert Kenward return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EFX_TX_CB_ORDER); 740f7251a9cSBen Hutchings } 741f7251a9cSBen Hutchings 742874aeea5SJeff Kirsher int efx_probe_tx_queue(struct efx_tx_queue *tx_queue) 743874aeea5SJeff Kirsher { 744874aeea5SJeff Kirsher struct efx_nic *efx = tx_queue->efx; 745874aeea5SJeff Kirsher unsigned int entries; 7467668ff9cSBen Hutchings int rc; 747874aeea5SJeff Kirsher 748874aeea5SJeff Kirsher /* Create the smallest power-of-two aligned ring */ 749874aeea5SJeff Kirsher entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE); 750874aeea5SJeff Kirsher EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); 751874aeea5SJeff Kirsher tx_queue->ptr_mask = entries - 1; 752874aeea5SJeff Kirsher 753874aeea5SJeff Kirsher netif_dbg(efx, probe, efx->net_dev, 754874aeea5SJeff Kirsher "creating TX queue %d size %#x mask %#x\n", 755874aeea5SJeff Kirsher tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask); 756874aeea5SJeff Kirsher 757874aeea5SJeff Kirsher /* Allocate software ring */ 758c2e4e25aSThomas Meyer tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer), 759874aeea5SJeff Kirsher GFP_KERNEL); 760874aeea5SJeff Kirsher if (!tx_queue->buffer) 761874aeea5SJeff Kirsher return -ENOMEM; 762874aeea5SJeff Kirsher 763e9117e50SBert Kenward tx_queue->cb_page = kcalloc(efx_tx_cb_page_count(tx_queue), 764e9117e50SBert Kenward sizeof(tx_queue->cb_page[0]), GFP_KERNEL); 765e9117e50SBert Kenward if (!tx_queue->cb_page) { 766f7251a9cSBen Hutchings rc = -ENOMEM; 767f7251a9cSBen Hutchings goto fail1; 768f7251a9cSBen Hutchings } 769f7251a9cSBen Hutchings 770874aeea5SJeff Kirsher /* Allocate hardware ring */ 771874aeea5SJeff Kirsher rc = efx_nic_probe_tx(tx_queue); 772874aeea5SJeff Kirsher if (rc) 773f7251a9cSBen Hutchings goto fail2; 774874aeea5SJeff Kirsher 775874aeea5SJeff Kirsher return 0; 776874aeea5SJeff Kirsher 777f7251a9cSBen Hutchings fail2: 778e9117e50SBert Kenward kfree(tx_queue->cb_page); 779e9117e50SBert Kenward tx_queue->cb_page = NULL; 780f7251a9cSBen Hutchings fail1: 781874aeea5SJeff Kirsher kfree(tx_queue->buffer); 782874aeea5SJeff Kirsher tx_queue->buffer = NULL; 783874aeea5SJeff Kirsher return rc; 784874aeea5SJeff Kirsher } 785874aeea5SJeff Kirsher 786874aeea5SJeff Kirsher void efx_init_tx_queue(struct efx_tx_queue *tx_queue) 787874aeea5SJeff Kirsher { 788e9117e50SBert Kenward struct efx_nic *efx = tx_queue->efx; 789e9117e50SBert Kenward 790e9117e50SBert Kenward netif_dbg(efx, drv, efx->net_dev, 791874aeea5SJeff Kirsher "initialising TX queue %d\n", tx_queue->queue); 792874aeea5SJeff Kirsher 793874aeea5SJeff Kirsher tx_queue->insert_count = 0; 794874aeea5SJeff Kirsher tx_queue->write_count = 0; 795874aeea5SJeff Kirsher tx_queue->old_write_count = 0; 796874aeea5SJeff Kirsher tx_queue->read_count = 0; 797874aeea5SJeff Kirsher tx_queue->old_read_count = 0; 798874aeea5SJeff Kirsher tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID; 799b2663a4fSMartin Habets tx_queue->xmit_more_available = false; 800874aeea5SJeff Kirsher 801e9117e50SBert Kenward /* Set up default function pointers. These may get replaced by 802e9117e50SBert Kenward * efx_nic_init_tx() based off NIC/queue capabilities. 803e9117e50SBert Kenward */ 804e9117e50SBert Kenward tx_queue->handle_tso = efx_tx_tso_sw; 805e9117e50SBert Kenward 806e9117e50SBert Kenward /* Some older hardware requires Tx writes larger than 32. */ 807e9117e50SBert Kenward tx_queue->tx_min_size = EFX_WORKAROUND_15592(efx) ? 33 : 0; 808e9117e50SBert Kenward 809874aeea5SJeff Kirsher /* Set up TX descriptor ring */ 810874aeea5SJeff Kirsher efx_nic_init_tx(tx_queue); 811874aeea5SJeff Kirsher 812874aeea5SJeff Kirsher tx_queue->initialised = true; 813874aeea5SJeff Kirsher } 814874aeea5SJeff Kirsher 815e42c3d85SBen Hutchings void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) 816874aeea5SJeff Kirsher { 817874aeea5SJeff Kirsher struct efx_tx_buffer *buffer; 818874aeea5SJeff Kirsher 819e42c3d85SBen Hutchings netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, 820e42c3d85SBen Hutchings "shutting down TX queue %d\n", tx_queue->queue); 821e42c3d85SBen Hutchings 822874aeea5SJeff Kirsher if (!tx_queue->buffer) 823874aeea5SJeff Kirsher return; 824874aeea5SJeff Kirsher 825874aeea5SJeff Kirsher /* Free any buffers left in the ring */ 826874aeea5SJeff Kirsher while (tx_queue->read_count != tx_queue->write_count) { 827c3940999STom Herbert unsigned int pkts_compl = 0, bytes_compl = 0; 828874aeea5SJeff Kirsher buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; 829c3940999STom Herbert efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); 830874aeea5SJeff Kirsher 831874aeea5SJeff Kirsher ++tx_queue->read_count; 832874aeea5SJeff Kirsher } 833b2663a4fSMartin Habets tx_queue->xmit_more_available = false; 834c3940999STom Herbert netdev_tx_reset_queue(tx_queue->core_txq); 835874aeea5SJeff Kirsher } 836874aeea5SJeff Kirsher 837874aeea5SJeff Kirsher void efx_remove_tx_queue(struct efx_tx_queue *tx_queue) 838874aeea5SJeff Kirsher { 839f7251a9cSBen Hutchings int i; 840f7251a9cSBen Hutchings 841874aeea5SJeff Kirsher if (!tx_queue->buffer) 842874aeea5SJeff Kirsher return; 843874aeea5SJeff Kirsher 844874aeea5SJeff Kirsher netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, 845874aeea5SJeff Kirsher "destroying TX queue %d\n", tx_queue->queue); 846874aeea5SJeff Kirsher efx_nic_remove_tx(tx_queue); 847874aeea5SJeff Kirsher 848e9117e50SBert Kenward if (tx_queue->cb_page) { 849e9117e50SBert Kenward for (i = 0; i < efx_tx_cb_page_count(tx_queue); i++) 850f7251a9cSBen Hutchings efx_nic_free_buffer(tx_queue->efx, 851e9117e50SBert Kenward &tx_queue->cb_page[i]); 852e9117e50SBert Kenward kfree(tx_queue->cb_page); 853e9117e50SBert Kenward tx_queue->cb_page = NULL; 854f7251a9cSBen Hutchings } 855f7251a9cSBen Hutchings 856874aeea5SJeff Kirsher kfree(tx_queue->buffer); 857874aeea5SJeff Kirsher tx_queue->buffer = NULL; 858874aeea5SJeff Kirsher } 859