1cdedef59SAnirudh Venkataramanan // SPDX-License-Identifier: GPL-2.0 2cdedef59SAnirudh Venkataramanan /* Copyright (c) 2018, Intel Corporation. */ 3cdedef59SAnirudh Venkataramanan 4cdedef59SAnirudh Venkataramanan /* The driver transmit and receive code */ 5cdedef59SAnirudh Venkataramanan 6cdedef59SAnirudh Venkataramanan #include <linux/prefetch.h> 7cdedef59SAnirudh Venkataramanan #include <linux/mm.h> 8efc2214bSMaciej Fijalkowski #include <linux/bpf_trace.h> 9efc2214bSMaciej Fijalkowski #include <net/xdp.h> 100891d6d4SKrzysztof Kazimierczak #include "ice_txrx_lib.h" 11efc2214bSMaciej Fijalkowski #include "ice_lib.h" 12cdedef59SAnirudh Venkataramanan #include "ice.h" 135f6aa50eSAnirudh Venkataramanan #include "ice_dcb_lib.h" 142d4238f5SKrzysztof Kazimierczak #include "ice_xsk.h" 15cdedef59SAnirudh Venkataramanan 162b245cb2SAnirudh Venkataramanan #define ICE_RX_HDR_SIZE 256 172b245cb2SAnirudh Venkataramanan 18148beb61SHenry Tieman #define FDIR_DESC_RXDID 0x40 19148beb61SHenry Tieman 20cdedef59SAnirudh Venkataramanan /** 21cdedef59SAnirudh Venkataramanan * ice_unmap_and_free_tx_buf - Release a Tx buffer 22cdedef59SAnirudh Venkataramanan * @ring: the ring that owns the buffer 23cdedef59SAnirudh Venkataramanan * @tx_buf: the buffer to free 24cdedef59SAnirudh Venkataramanan */ 25cdedef59SAnirudh Venkataramanan static void 26cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(struct ice_ring *ring, struct ice_tx_buf *tx_buf) 27cdedef59SAnirudh Venkataramanan { 28cdedef59SAnirudh Venkataramanan if (tx_buf->skb) { 29148beb61SHenry Tieman if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) 30148beb61SHenry Tieman devm_kfree(ring->dev, tx_buf->raw_buf); 31148beb61SHenry Tieman else if (ice_ring_is_xdp(ring)) 32efc2214bSMaciej Fijalkowski page_frag_free(tx_buf->raw_buf); 33efc2214bSMaciej Fijalkowski else 34cdedef59SAnirudh Venkataramanan dev_kfree_skb_any(tx_buf->skb); 35cdedef59SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) 36cdedef59SAnirudh Venkataramanan dma_unmap_single(ring->dev, 37cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 38cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 39cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 40cdedef59SAnirudh Venkataramanan } else if (dma_unmap_len(tx_buf, len)) { 41cdedef59SAnirudh Venkataramanan dma_unmap_page(ring->dev, 42cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 43cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 44cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 45cdedef59SAnirudh Venkataramanan } 46cdedef59SAnirudh Venkataramanan 47cdedef59SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 48cdedef59SAnirudh Venkataramanan tx_buf->skb = NULL; 49cdedef59SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 50cdedef59SAnirudh Venkataramanan /* tx_buf must be completely set up in the transmit path */ 51cdedef59SAnirudh Venkataramanan } 52cdedef59SAnirudh Venkataramanan 53cdedef59SAnirudh Venkataramanan static struct netdev_queue *txring_txq(const struct ice_ring *ring) 54cdedef59SAnirudh Venkataramanan { 55cdedef59SAnirudh Venkataramanan return netdev_get_tx_queue(ring->netdev, ring->q_index); 56cdedef59SAnirudh Venkataramanan } 57cdedef59SAnirudh Venkataramanan 58cdedef59SAnirudh Venkataramanan /** 59cdedef59SAnirudh Venkataramanan * ice_clean_tx_ring - Free any empty Tx buffers 60cdedef59SAnirudh Venkataramanan * @tx_ring: ring to be cleaned 61cdedef59SAnirudh Venkataramanan */ 62cdedef59SAnirudh Venkataramanan void ice_clean_tx_ring(struct ice_ring *tx_ring) 63cdedef59SAnirudh Venkataramanan { 64cdedef59SAnirudh Venkataramanan u16 i; 65cdedef59SAnirudh Venkataramanan 662d4238f5SKrzysztof Kazimierczak if (ice_ring_is_xdp(tx_ring) && tx_ring->xsk_umem) { 672d4238f5SKrzysztof Kazimierczak ice_xsk_clean_xdp_ring(tx_ring); 682d4238f5SKrzysztof Kazimierczak goto tx_skip_free; 692d4238f5SKrzysztof Kazimierczak } 702d4238f5SKrzysztof Kazimierczak 71cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 72cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 73cdedef59SAnirudh Venkataramanan return; 74cdedef59SAnirudh Venkataramanan 752f2da36eSAnirudh Venkataramanan /* Free all the Tx ring sk_buffs */ 76cdedef59SAnirudh Venkataramanan for (i = 0; i < tx_ring->count; i++) 77cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]); 78cdedef59SAnirudh Venkataramanan 792d4238f5SKrzysztof Kazimierczak tx_skip_free: 80c6dfd690SBruce Allan memset(tx_ring->tx_buf, 0, sizeof(*tx_ring->tx_buf) * tx_ring->count); 81cdedef59SAnirudh Venkataramanan 82cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 83cdedef59SAnirudh Venkataramanan memset(tx_ring->desc, 0, tx_ring->size); 84cdedef59SAnirudh Venkataramanan 85cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 86cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 87cdedef59SAnirudh Venkataramanan 88cdedef59SAnirudh Venkataramanan if (!tx_ring->netdev) 89cdedef59SAnirudh Venkataramanan return; 90cdedef59SAnirudh Venkataramanan 91cdedef59SAnirudh Venkataramanan /* cleanup Tx queue statistics */ 92cdedef59SAnirudh Venkataramanan netdev_tx_reset_queue(txring_txq(tx_ring)); 93cdedef59SAnirudh Venkataramanan } 94cdedef59SAnirudh Venkataramanan 95cdedef59SAnirudh Venkataramanan /** 96cdedef59SAnirudh Venkataramanan * ice_free_tx_ring - Free Tx resources per queue 97cdedef59SAnirudh Venkataramanan * @tx_ring: Tx descriptor ring for a specific queue 98cdedef59SAnirudh Venkataramanan * 99cdedef59SAnirudh Venkataramanan * Free all transmit software resources 100cdedef59SAnirudh Venkataramanan */ 101cdedef59SAnirudh Venkataramanan void ice_free_tx_ring(struct ice_ring *tx_ring) 102cdedef59SAnirudh Venkataramanan { 103cdedef59SAnirudh Venkataramanan ice_clean_tx_ring(tx_ring); 104cdedef59SAnirudh Venkataramanan devm_kfree(tx_ring->dev, tx_ring->tx_buf); 105cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 106cdedef59SAnirudh Venkataramanan 107cdedef59SAnirudh Venkataramanan if (tx_ring->desc) { 108cdedef59SAnirudh Venkataramanan dmam_free_coherent(tx_ring->dev, tx_ring->size, 109cdedef59SAnirudh Venkataramanan tx_ring->desc, tx_ring->dma); 110cdedef59SAnirudh Venkataramanan tx_ring->desc = NULL; 111cdedef59SAnirudh Venkataramanan } 112cdedef59SAnirudh Venkataramanan } 113cdedef59SAnirudh Venkataramanan 114cdedef59SAnirudh Venkataramanan /** 1152b245cb2SAnirudh Venkataramanan * ice_clean_tx_irq - Reclaim resources after transmit completes 1162b245cb2SAnirudh Venkataramanan * @tx_ring: Tx ring to clean 1172b245cb2SAnirudh Venkataramanan * @napi_budget: Used to determine if we are in netpoll 1182b245cb2SAnirudh Venkataramanan * 1192b245cb2SAnirudh Venkataramanan * Returns true if there's any budget left (e.g. the clean is finished) 1202b245cb2SAnirudh Venkataramanan */ 1212fb0821fSJesse Brandeburg static bool ice_clean_tx_irq(struct ice_ring *tx_ring, int napi_budget) 1222b245cb2SAnirudh Venkataramanan { 1232b245cb2SAnirudh Venkataramanan unsigned int total_bytes = 0, total_pkts = 0; 1242fb0821fSJesse Brandeburg unsigned int budget = ICE_DFLT_IRQ_WORK; 1252fb0821fSJesse Brandeburg struct ice_vsi *vsi = tx_ring->vsi; 1262b245cb2SAnirudh Venkataramanan s16 i = tx_ring->next_to_clean; 1272b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 1282b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 1292b245cb2SAnirudh Venkataramanan 1302b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 1312b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 1322b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1332b245cb2SAnirudh Venkataramanan 1342fb0821fSJesse Brandeburg prefetch(&vsi->state); 1352fb0821fSJesse Brandeburg 1362b245cb2SAnirudh Venkataramanan do { 1372b245cb2SAnirudh Venkataramanan struct ice_tx_desc *eop_desc = tx_buf->next_to_watch; 1382b245cb2SAnirudh Venkataramanan 1392b245cb2SAnirudh Venkataramanan /* if next_to_watch is not set then there is no work pending */ 1402b245cb2SAnirudh Venkataramanan if (!eop_desc) 1412b245cb2SAnirudh Venkataramanan break; 1422b245cb2SAnirudh Venkataramanan 1432b245cb2SAnirudh Venkataramanan smp_rmb(); /* prevent any other reads prior to eop_desc */ 1442b245cb2SAnirudh Venkataramanan 1452b245cb2SAnirudh Venkataramanan /* if the descriptor isn't done, no work yet to do */ 1462b245cb2SAnirudh Venkataramanan if (!(eop_desc->cmd_type_offset_bsz & 1472b245cb2SAnirudh Venkataramanan cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) 1482b245cb2SAnirudh Venkataramanan break; 1492b245cb2SAnirudh Venkataramanan 1502b245cb2SAnirudh Venkataramanan /* clear next_to_watch to prevent false hangs */ 1512b245cb2SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 1522b245cb2SAnirudh Venkataramanan 1532b245cb2SAnirudh Venkataramanan /* update the statistics for this packet */ 1542b245cb2SAnirudh Venkataramanan total_bytes += tx_buf->bytecount; 1552b245cb2SAnirudh Venkataramanan total_pkts += tx_buf->gso_segs; 1562b245cb2SAnirudh Venkataramanan 157efc2214bSMaciej Fijalkowski if (ice_ring_is_xdp(tx_ring)) 158efc2214bSMaciej Fijalkowski page_frag_free(tx_buf->raw_buf); 159efc2214bSMaciej Fijalkowski else 1602b245cb2SAnirudh Venkataramanan /* free the skb */ 1612b245cb2SAnirudh Venkataramanan napi_consume_skb(tx_buf->skb, napi_budget); 1622b245cb2SAnirudh Venkataramanan 1632b245cb2SAnirudh Venkataramanan /* unmap skb header data */ 1642b245cb2SAnirudh Venkataramanan dma_unmap_single(tx_ring->dev, 1652b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 1662b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 1672b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1682b245cb2SAnirudh Venkataramanan 1692b245cb2SAnirudh Venkataramanan /* clear tx_buf data */ 1702b245cb2SAnirudh Venkataramanan tx_buf->skb = NULL; 1712b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 1722b245cb2SAnirudh Venkataramanan 1732b245cb2SAnirudh Venkataramanan /* unmap remaining buffers */ 1742b245cb2SAnirudh Venkataramanan while (tx_desc != eop_desc) { 1752b245cb2SAnirudh Venkataramanan tx_buf++; 1762b245cb2SAnirudh Venkataramanan tx_desc++; 1772b245cb2SAnirudh Venkataramanan i++; 1782b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 1792b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1802b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 1812b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1822b245cb2SAnirudh Venkataramanan } 1832b245cb2SAnirudh Venkataramanan 1842b245cb2SAnirudh Venkataramanan /* unmap any remaining paged data */ 1852b245cb2SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) { 1862b245cb2SAnirudh Venkataramanan dma_unmap_page(tx_ring->dev, 1872b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 1882b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 1892b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1902b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 1912b245cb2SAnirudh Venkataramanan } 1922b245cb2SAnirudh Venkataramanan } 1932b245cb2SAnirudh Venkataramanan 1942b245cb2SAnirudh Venkataramanan /* move us one more past the eop_desc for start of next pkt */ 1952b245cb2SAnirudh Venkataramanan tx_buf++; 1962b245cb2SAnirudh Venkataramanan tx_desc++; 1972b245cb2SAnirudh Venkataramanan i++; 1982b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 1992b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 2002b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 2012b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 2022b245cb2SAnirudh Venkataramanan } 2032b245cb2SAnirudh Venkataramanan 2042b245cb2SAnirudh Venkataramanan prefetch(tx_desc); 2052b245cb2SAnirudh Venkataramanan 2062b245cb2SAnirudh Venkataramanan /* update budget accounting */ 2072b245cb2SAnirudh Venkataramanan budget--; 2082b245cb2SAnirudh Venkataramanan } while (likely(budget)); 2092b245cb2SAnirudh Venkataramanan 2102b245cb2SAnirudh Venkataramanan i += tx_ring->count; 2112b245cb2SAnirudh Venkataramanan tx_ring->next_to_clean = i; 2122d4238f5SKrzysztof Kazimierczak 2132d4238f5SKrzysztof Kazimierczak ice_update_tx_ring_stats(tx_ring, total_pkts, total_bytes); 2142b245cb2SAnirudh Venkataramanan 215efc2214bSMaciej Fijalkowski if (ice_ring_is_xdp(tx_ring)) 216efc2214bSMaciej Fijalkowski return !!budget; 217efc2214bSMaciej Fijalkowski 2182b245cb2SAnirudh Venkataramanan netdev_tx_completed_queue(txring_txq(tx_ring), total_pkts, 2192b245cb2SAnirudh Venkataramanan total_bytes); 2202b245cb2SAnirudh Venkataramanan 2212b245cb2SAnirudh Venkataramanan #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2)) 2222b245cb2SAnirudh Venkataramanan if (unlikely(total_pkts && netif_carrier_ok(tx_ring->netdev) && 2232b245cb2SAnirudh Venkataramanan (ICE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 2242b245cb2SAnirudh Venkataramanan /* Make sure that anybody stopping the queue after this 2252b245cb2SAnirudh Venkataramanan * sees the new next_to_clean. 2262b245cb2SAnirudh Venkataramanan */ 2272b245cb2SAnirudh Venkataramanan smp_mb(); 2282b245cb2SAnirudh Venkataramanan if (__netif_subqueue_stopped(tx_ring->netdev, 2292b245cb2SAnirudh Venkataramanan tx_ring->q_index) && 2302b245cb2SAnirudh Venkataramanan !test_bit(__ICE_DOWN, vsi->state)) { 2312b245cb2SAnirudh Venkataramanan netif_wake_subqueue(tx_ring->netdev, 2322b245cb2SAnirudh Venkataramanan tx_ring->q_index); 2332b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 2342b245cb2SAnirudh Venkataramanan } 2352b245cb2SAnirudh Venkataramanan } 2362b245cb2SAnirudh Venkataramanan 2372b245cb2SAnirudh Venkataramanan return !!budget; 2382b245cb2SAnirudh Venkataramanan } 2392b245cb2SAnirudh Venkataramanan 2402b245cb2SAnirudh Venkataramanan /** 241cdedef59SAnirudh Venkataramanan * ice_setup_tx_ring - Allocate the Tx descriptors 242d337f2afSAnirudh Venkataramanan * @tx_ring: the Tx ring to set up 243cdedef59SAnirudh Venkataramanan * 244cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 245cdedef59SAnirudh Venkataramanan */ 246cdedef59SAnirudh Venkataramanan int ice_setup_tx_ring(struct ice_ring *tx_ring) 247cdedef59SAnirudh Venkataramanan { 248cdedef59SAnirudh Venkataramanan struct device *dev = tx_ring->dev; 249cdedef59SAnirudh Venkataramanan 250cdedef59SAnirudh Venkataramanan if (!dev) 251cdedef59SAnirudh Venkataramanan return -ENOMEM; 252cdedef59SAnirudh Venkataramanan 253cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 254cdedef59SAnirudh Venkataramanan WARN_ON(tx_ring->tx_buf); 255c6dfd690SBruce Allan tx_ring->tx_buf = 256c6dfd690SBruce Allan devm_kzalloc(dev, sizeof(*tx_ring->tx_buf) * tx_ring->count, 257c6dfd690SBruce Allan GFP_KERNEL); 258cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 259cdedef59SAnirudh Venkataramanan return -ENOMEM; 260cdedef59SAnirudh Venkataramanan 261ad71b256SBrett Creeley /* round up to nearest page */ 262c6dfd690SBruce Allan tx_ring->size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc), 263ad71b256SBrett Creeley PAGE_SIZE); 264cdedef59SAnirudh Venkataramanan tx_ring->desc = dmam_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, 265cdedef59SAnirudh Venkataramanan GFP_KERNEL); 266cdedef59SAnirudh Venkataramanan if (!tx_ring->desc) { 267cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 268cdedef59SAnirudh Venkataramanan tx_ring->size); 269cdedef59SAnirudh Venkataramanan goto err; 270cdedef59SAnirudh Venkataramanan } 271cdedef59SAnirudh Venkataramanan 272cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 273cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 274b3969fd7SSudheer Mogilappagari tx_ring->tx_stats.prev_pkt = -1; 275cdedef59SAnirudh Venkataramanan return 0; 276cdedef59SAnirudh Venkataramanan 277cdedef59SAnirudh Venkataramanan err: 278cdedef59SAnirudh Venkataramanan devm_kfree(dev, tx_ring->tx_buf); 279cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 280cdedef59SAnirudh Venkataramanan return -ENOMEM; 281cdedef59SAnirudh Venkataramanan } 282cdedef59SAnirudh Venkataramanan 283cdedef59SAnirudh Venkataramanan /** 284cdedef59SAnirudh Venkataramanan * ice_clean_rx_ring - Free Rx buffers 285cdedef59SAnirudh Venkataramanan * @rx_ring: ring to be cleaned 286cdedef59SAnirudh Venkataramanan */ 287cdedef59SAnirudh Venkataramanan void ice_clean_rx_ring(struct ice_ring *rx_ring) 288cdedef59SAnirudh Venkataramanan { 289cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 290cdedef59SAnirudh Venkataramanan u16 i; 291cdedef59SAnirudh Venkataramanan 292cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 293cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 294cdedef59SAnirudh Venkataramanan return; 295cdedef59SAnirudh Venkataramanan 2962d4238f5SKrzysztof Kazimierczak if (rx_ring->xsk_umem) { 2972d4238f5SKrzysztof Kazimierczak ice_xsk_clean_rx_ring(rx_ring); 2982d4238f5SKrzysztof Kazimierczak goto rx_skip_free; 2992d4238f5SKrzysztof Kazimierczak } 3002d4238f5SKrzysztof Kazimierczak 301cdedef59SAnirudh Venkataramanan /* Free all the Rx ring sk_buffs */ 302cdedef59SAnirudh Venkataramanan for (i = 0; i < rx_ring->count; i++) { 303cdedef59SAnirudh Venkataramanan struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i]; 304cdedef59SAnirudh Venkataramanan 305cdedef59SAnirudh Venkataramanan if (rx_buf->skb) { 306cdedef59SAnirudh Venkataramanan dev_kfree_skb(rx_buf->skb); 307cdedef59SAnirudh Venkataramanan rx_buf->skb = NULL; 308cdedef59SAnirudh Venkataramanan } 309cdedef59SAnirudh Venkataramanan if (!rx_buf->page) 310cdedef59SAnirudh Venkataramanan continue; 311cdedef59SAnirudh Venkataramanan 312a65f71feSMaciej Fijalkowski /* Invalidate cache lines that may have been written to by 313a65f71feSMaciej Fijalkowski * device so that we avoid corrupting memory. 314a65f71feSMaciej Fijalkowski */ 315a65f71feSMaciej Fijalkowski dma_sync_single_range_for_cpu(dev, rx_buf->dma, 316a65f71feSMaciej Fijalkowski rx_buf->page_offset, 3177237f5b0SMaciej Fijalkowski rx_ring->rx_buf_len, 3187237f5b0SMaciej Fijalkowski DMA_FROM_DEVICE); 319a65f71feSMaciej Fijalkowski 320a65f71feSMaciej Fijalkowski /* free resources associated with mapping */ 3217237f5b0SMaciej Fijalkowski dma_unmap_page_attrs(dev, rx_buf->dma, ice_rx_pg_size(rx_ring), 322a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 32303c66a13SMaciej Fijalkowski __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias); 324cdedef59SAnirudh Venkataramanan 325cdedef59SAnirudh Venkataramanan rx_buf->page = NULL; 326cdedef59SAnirudh Venkataramanan rx_buf->page_offset = 0; 327cdedef59SAnirudh Venkataramanan } 328cdedef59SAnirudh Venkataramanan 3292d4238f5SKrzysztof Kazimierczak rx_skip_free: 330c6dfd690SBruce Allan memset(rx_ring->rx_buf, 0, sizeof(*rx_ring->rx_buf) * rx_ring->count); 331cdedef59SAnirudh Venkataramanan 332cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 333cdedef59SAnirudh Venkataramanan memset(rx_ring->desc, 0, rx_ring->size); 334cdedef59SAnirudh Venkataramanan 335cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = 0; 336cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 337cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 338cdedef59SAnirudh Venkataramanan } 339cdedef59SAnirudh Venkataramanan 340cdedef59SAnirudh Venkataramanan /** 341cdedef59SAnirudh Venkataramanan * ice_free_rx_ring - Free Rx resources 342cdedef59SAnirudh Venkataramanan * @rx_ring: ring to clean the resources from 343cdedef59SAnirudh Venkataramanan * 344cdedef59SAnirudh Venkataramanan * Free all receive software resources 345cdedef59SAnirudh Venkataramanan */ 346cdedef59SAnirudh Venkataramanan void ice_free_rx_ring(struct ice_ring *rx_ring) 347cdedef59SAnirudh Venkataramanan { 348cdedef59SAnirudh Venkataramanan ice_clean_rx_ring(rx_ring); 349efc2214bSMaciej Fijalkowski if (rx_ring->vsi->type == ICE_VSI_PF) 350efc2214bSMaciej Fijalkowski if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) 351efc2214bSMaciej Fijalkowski xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 352efc2214bSMaciej Fijalkowski rx_ring->xdp_prog = NULL; 353cdedef59SAnirudh Venkataramanan devm_kfree(rx_ring->dev, rx_ring->rx_buf); 354cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 355cdedef59SAnirudh Venkataramanan 356cdedef59SAnirudh Venkataramanan if (rx_ring->desc) { 357cdedef59SAnirudh Venkataramanan dmam_free_coherent(rx_ring->dev, rx_ring->size, 358cdedef59SAnirudh Venkataramanan rx_ring->desc, rx_ring->dma); 359cdedef59SAnirudh Venkataramanan rx_ring->desc = NULL; 360cdedef59SAnirudh Venkataramanan } 361cdedef59SAnirudh Venkataramanan } 362cdedef59SAnirudh Venkataramanan 363cdedef59SAnirudh Venkataramanan /** 364cdedef59SAnirudh Venkataramanan * ice_setup_rx_ring - Allocate the Rx descriptors 365d337f2afSAnirudh Venkataramanan * @rx_ring: the Rx ring to set up 366cdedef59SAnirudh Venkataramanan * 367cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 368cdedef59SAnirudh Venkataramanan */ 369cdedef59SAnirudh Venkataramanan int ice_setup_rx_ring(struct ice_ring *rx_ring) 370cdedef59SAnirudh Venkataramanan { 371cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 372cdedef59SAnirudh Venkataramanan 373cdedef59SAnirudh Venkataramanan if (!dev) 374cdedef59SAnirudh Venkataramanan return -ENOMEM; 375cdedef59SAnirudh Venkataramanan 376cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 377cdedef59SAnirudh Venkataramanan WARN_ON(rx_ring->rx_buf); 378c6dfd690SBruce Allan rx_ring->rx_buf = 379c6dfd690SBruce Allan devm_kzalloc(dev, sizeof(*rx_ring->rx_buf) * rx_ring->count, 380c6dfd690SBruce Allan GFP_KERNEL); 381cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 382cdedef59SAnirudh Venkataramanan return -ENOMEM; 383cdedef59SAnirudh Venkataramanan 384ad71b256SBrett Creeley /* round up to nearest page */ 385ad71b256SBrett Creeley rx_ring->size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc), 386ad71b256SBrett Creeley PAGE_SIZE); 387cdedef59SAnirudh Venkataramanan rx_ring->desc = dmam_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, 388cdedef59SAnirudh Venkataramanan GFP_KERNEL); 389cdedef59SAnirudh Venkataramanan if (!rx_ring->desc) { 390cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 391cdedef59SAnirudh Venkataramanan rx_ring->size); 392cdedef59SAnirudh Venkataramanan goto err; 393cdedef59SAnirudh Venkataramanan } 394cdedef59SAnirudh Venkataramanan 395cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 396cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 397efc2214bSMaciej Fijalkowski 398efc2214bSMaciej Fijalkowski if (ice_is_xdp_ena_vsi(rx_ring->vsi)) 399efc2214bSMaciej Fijalkowski WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog); 400efc2214bSMaciej Fijalkowski 401efc2214bSMaciej Fijalkowski if (rx_ring->vsi->type == ICE_VSI_PF && 402efc2214bSMaciej Fijalkowski !xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) 403efc2214bSMaciej Fijalkowski if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, 404efc2214bSMaciej Fijalkowski rx_ring->q_index)) 405efc2214bSMaciej Fijalkowski goto err; 406cdedef59SAnirudh Venkataramanan return 0; 407cdedef59SAnirudh Venkataramanan 408cdedef59SAnirudh Venkataramanan err: 409cdedef59SAnirudh Venkataramanan devm_kfree(dev, rx_ring->rx_buf); 410cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 411cdedef59SAnirudh Venkataramanan return -ENOMEM; 412cdedef59SAnirudh Venkataramanan } 413cdedef59SAnirudh Venkataramanan 414cdedef59SAnirudh Venkataramanan /** 415efc2214bSMaciej Fijalkowski * ice_rx_offset - Return expected offset into page to access data 416efc2214bSMaciej Fijalkowski * @rx_ring: Ring we are requesting offset of 417efc2214bSMaciej Fijalkowski * 418efc2214bSMaciej Fijalkowski * Returns the offset value for ring into the data buffer. 419efc2214bSMaciej Fijalkowski */ 420efc2214bSMaciej Fijalkowski static unsigned int ice_rx_offset(struct ice_ring *rx_ring) 421efc2214bSMaciej Fijalkowski { 42259bb0808SMaciej Fijalkowski if (ice_ring_uses_build_skb(rx_ring)) 42359bb0808SMaciej Fijalkowski return ICE_SKB_PAD; 42459bb0808SMaciej Fijalkowski else if (ice_is_xdp_ena_vsi(rx_ring->vsi)) 42559bb0808SMaciej Fijalkowski return XDP_PACKET_HEADROOM; 42659bb0808SMaciej Fijalkowski 42759bb0808SMaciej Fijalkowski return 0; 428efc2214bSMaciej Fijalkowski } 429efc2214bSMaciej Fijalkowski 430d4ecdbf7SJesper Dangaard Brouer static unsigned int ice_rx_frame_truesize(struct ice_ring *rx_ring, 431d4ecdbf7SJesper Dangaard Brouer unsigned int size) 432d4ecdbf7SJesper Dangaard Brouer { 433d4ecdbf7SJesper Dangaard Brouer unsigned int truesize; 434d4ecdbf7SJesper Dangaard Brouer 435d4ecdbf7SJesper Dangaard Brouer #if (PAGE_SIZE < 8192) 436d4ecdbf7SJesper Dangaard Brouer truesize = ice_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ 437d4ecdbf7SJesper Dangaard Brouer #else 438d4ecdbf7SJesper Dangaard Brouer truesize = ice_rx_offset(rx_ring) ? 439d4ecdbf7SJesper Dangaard Brouer SKB_DATA_ALIGN(ice_rx_offset(rx_ring) + size) + 440d4ecdbf7SJesper Dangaard Brouer SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 441d4ecdbf7SJesper Dangaard Brouer SKB_DATA_ALIGN(size); 442d4ecdbf7SJesper Dangaard Brouer #endif 443d4ecdbf7SJesper Dangaard Brouer return truesize; 444d4ecdbf7SJesper Dangaard Brouer } 445d4ecdbf7SJesper Dangaard Brouer 446efc2214bSMaciej Fijalkowski /** 447efc2214bSMaciej Fijalkowski * ice_run_xdp - Executes an XDP program on initialized xdp_buff 448efc2214bSMaciej Fijalkowski * @rx_ring: Rx ring 449efc2214bSMaciej Fijalkowski * @xdp: xdp_buff used as input to the XDP program 450efc2214bSMaciej Fijalkowski * @xdp_prog: XDP program to run 451efc2214bSMaciej Fijalkowski * 452efc2214bSMaciej Fijalkowski * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} 453efc2214bSMaciej Fijalkowski */ 454efc2214bSMaciej Fijalkowski static int 455efc2214bSMaciej Fijalkowski ice_run_xdp(struct ice_ring *rx_ring, struct xdp_buff *xdp, 456efc2214bSMaciej Fijalkowski struct bpf_prog *xdp_prog) 457efc2214bSMaciej Fijalkowski { 458efc2214bSMaciej Fijalkowski int err, result = ICE_XDP_PASS; 459efc2214bSMaciej Fijalkowski struct ice_ring *xdp_ring; 460efc2214bSMaciej Fijalkowski u32 act; 461efc2214bSMaciej Fijalkowski 462efc2214bSMaciej Fijalkowski act = bpf_prog_run_xdp(xdp_prog, xdp); 463efc2214bSMaciej Fijalkowski switch (act) { 464efc2214bSMaciej Fijalkowski case XDP_PASS: 465efc2214bSMaciej Fijalkowski break; 466efc2214bSMaciej Fijalkowski case XDP_TX: 467efc2214bSMaciej Fijalkowski xdp_ring = rx_ring->vsi->xdp_rings[smp_processor_id()]; 468efc2214bSMaciej Fijalkowski result = ice_xmit_xdp_buff(xdp, xdp_ring); 469efc2214bSMaciej Fijalkowski break; 470efc2214bSMaciej Fijalkowski case XDP_REDIRECT: 471efc2214bSMaciej Fijalkowski err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); 472efc2214bSMaciej Fijalkowski result = !err ? ICE_XDP_REDIR : ICE_XDP_CONSUMED; 473efc2214bSMaciej Fijalkowski break; 474efc2214bSMaciej Fijalkowski default: 475efc2214bSMaciej Fijalkowski bpf_warn_invalid_xdp_action(act); 4764e83fc93SBruce Allan fallthrough; 477efc2214bSMaciej Fijalkowski case XDP_ABORTED: 478efc2214bSMaciej Fijalkowski trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 4794e83fc93SBruce Allan fallthrough; 480efc2214bSMaciej Fijalkowski case XDP_DROP: 481efc2214bSMaciej Fijalkowski result = ICE_XDP_CONSUMED; 482efc2214bSMaciej Fijalkowski break; 483efc2214bSMaciej Fijalkowski } 484efc2214bSMaciej Fijalkowski 485efc2214bSMaciej Fijalkowski return result; 486efc2214bSMaciej Fijalkowski } 487efc2214bSMaciej Fijalkowski 488efc2214bSMaciej Fijalkowski /** 489efc2214bSMaciej Fijalkowski * ice_xdp_xmit - submit packets to XDP ring for transmission 490efc2214bSMaciej Fijalkowski * @dev: netdev 491efc2214bSMaciej Fijalkowski * @n: number of XDP frames to be transmitted 492efc2214bSMaciej Fijalkowski * @frames: XDP frames to be transmitted 493efc2214bSMaciej Fijalkowski * @flags: transmit flags 494efc2214bSMaciej Fijalkowski * 495efc2214bSMaciej Fijalkowski * Returns number of frames successfully sent. Frames that fail are 496efc2214bSMaciej Fijalkowski * free'ed via XDP return API. 497efc2214bSMaciej Fijalkowski * For error cases, a negative errno code is returned and no-frames 498efc2214bSMaciej Fijalkowski * are transmitted (caller must handle freeing frames). 499efc2214bSMaciej Fijalkowski */ 500efc2214bSMaciej Fijalkowski int 501efc2214bSMaciej Fijalkowski ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 502efc2214bSMaciej Fijalkowski u32 flags) 503efc2214bSMaciej Fijalkowski { 504efc2214bSMaciej Fijalkowski struct ice_netdev_priv *np = netdev_priv(dev); 505efc2214bSMaciej Fijalkowski unsigned int queue_index = smp_processor_id(); 506efc2214bSMaciej Fijalkowski struct ice_vsi *vsi = np->vsi; 507efc2214bSMaciej Fijalkowski struct ice_ring *xdp_ring; 508efc2214bSMaciej Fijalkowski int drops = 0, i; 509efc2214bSMaciej Fijalkowski 510efc2214bSMaciej Fijalkowski if (test_bit(__ICE_DOWN, vsi->state)) 511efc2214bSMaciej Fijalkowski return -ENETDOWN; 512efc2214bSMaciej Fijalkowski 513efc2214bSMaciej Fijalkowski if (!ice_is_xdp_ena_vsi(vsi) || queue_index >= vsi->num_xdp_txq) 514efc2214bSMaciej Fijalkowski return -ENXIO; 515efc2214bSMaciej Fijalkowski 516efc2214bSMaciej Fijalkowski if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 517efc2214bSMaciej Fijalkowski return -EINVAL; 518efc2214bSMaciej Fijalkowski 519efc2214bSMaciej Fijalkowski xdp_ring = vsi->xdp_rings[queue_index]; 520efc2214bSMaciej Fijalkowski for (i = 0; i < n; i++) { 521efc2214bSMaciej Fijalkowski struct xdp_frame *xdpf = frames[i]; 522efc2214bSMaciej Fijalkowski int err; 523efc2214bSMaciej Fijalkowski 524efc2214bSMaciej Fijalkowski err = ice_xmit_xdp_ring(xdpf->data, xdpf->len, xdp_ring); 525efc2214bSMaciej Fijalkowski if (err != ICE_XDP_TX) { 526efc2214bSMaciej Fijalkowski xdp_return_frame_rx_napi(xdpf); 527efc2214bSMaciej Fijalkowski drops++; 528efc2214bSMaciej Fijalkowski } 529efc2214bSMaciej Fijalkowski } 530efc2214bSMaciej Fijalkowski 531efc2214bSMaciej Fijalkowski if (unlikely(flags & XDP_XMIT_FLUSH)) 532efc2214bSMaciej Fijalkowski ice_xdp_ring_update_tail(xdp_ring); 533efc2214bSMaciej Fijalkowski 534efc2214bSMaciej Fijalkowski return n - drops; 535efc2214bSMaciej Fijalkowski } 536efc2214bSMaciej Fijalkowski 537efc2214bSMaciej Fijalkowski /** 538cdedef59SAnirudh Venkataramanan * ice_alloc_mapped_page - recycle or make a new page 539cdedef59SAnirudh Venkataramanan * @rx_ring: ring to use 540cdedef59SAnirudh Venkataramanan * @bi: rx_buf struct to modify 541cdedef59SAnirudh Venkataramanan * 542cdedef59SAnirudh Venkataramanan * Returns true if the page was successfully allocated or 543cdedef59SAnirudh Venkataramanan * reused. 544cdedef59SAnirudh Venkataramanan */ 545c8b7abddSBruce Allan static bool 546c8b7abddSBruce Allan ice_alloc_mapped_page(struct ice_ring *rx_ring, struct ice_rx_buf *bi) 547cdedef59SAnirudh Venkataramanan { 548cdedef59SAnirudh Venkataramanan struct page *page = bi->page; 549cdedef59SAnirudh Venkataramanan dma_addr_t dma; 550cdedef59SAnirudh Venkataramanan 551cdedef59SAnirudh Venkataramanan /* since we are recycling buffers we should seldom need to alloc */ 5522b245cb2SAnirudh Venkataramanan if (likely(page)) { 5532b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 554cdedef59SAnirudh Venkataramanan return true; 5552b245cb2SAnirudh Venkataramanan } 556cdedef59SAnirudh Venkataramanan 557cdedef59SAnirudh Venkataramanan /* alloc new page for storage */ 5587237f5b0SMaciej Fijalkowski page = dev_alloc_pages(ice_rx_pg_order(rx_ring)); 5592b245cb2SAnirudh Venkataramanan if (unlikely(!page)) { 5602b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 561cdedef59SAnirudh Venkataramanan return false; 5622b245cb2SAnirudh Venkataramanan } 563cdedef59SAnirudh Venkataramanan 564cdedef59SAnirudh Venkataramanan /* map page for use */ 5657237f5b0SMaciej Fijalkowski dma = dma_map_page_attrs(rx_ring->dev, page, 0, ice_rx_pg_size(rx_ring), 566a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 567cdedef59SAnirudh Venkataramanan 568cdedef59SAnirudh Venkataramanan /* if mapping failed free memory back to system since 569cdedef59SAnirudh Venkataramanan * there isn't much point in holding memory we can't use 570cdedef59SAnirudh Venkataramanan */ 571cdedef59SAnirudh Venkataramanan if (dma_mapping_error(rx_ring->dev, dma)) { 5727237f5b0SMaciej Fijalkowski __free_pages(page, ice_rx_pg_order(rx_ring)); 5732b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 574cdedef59SAnirudh Venkataramanan return false; 575cdedef59SAnirudh Venkataramanan } 576cdedef59SAnirudh Venkataramanan 577cdedef59SAnirudh Venkataramanan bi->dma = dma; 578cdedef59SAnirudh Venkataramanan bi->page = page; 579efc2214bSMaciej Fijalkowski bi->page_offset = ice_rx_offset(rx_ring); 58003c66a13SMaciej Fijalkowski page_ref_add(page, USHRT_MAX - 1); 58103c66a13SMaciej Fijalkowski bi->pagecnt_bias = USHRT_MAX; 582cdedef59SAnirudh Venkataramanan 583cdedef59SAnirudh Venkataramanan return true; 584cdedef59SAnirudh Venkataramanan } 585cdedef59SAnirudh Venkataramanan 586cdedef59SAnirudh Venkataramanan /** 587cdedef59SAnirudh Venkataramanan * ice_alloc_rx_bufs - Replace used receive buffers 588cdedef59SAnirudh Venkataramanan * @rx_ring: ring to place buffers on 589cdedef59SAnirudh Venkataramanan * @cleaned_count: number of buffers to replace 590cdedef59SAnirudh Venkataramanan * 591cb7db356SBrett Creeley * Returns false if all allocations were successful, true if any fail. Returning 592cb7db356SBrett Creeley * true signals to the caller that we didn't replace cleaned_count buffers and 593cb7db356SBrett Creeley * there is more work to do. 594cb7db356SBrett Creeley * 595cb7db356SBrett Creeley * First, try to clean "cleaned_count" Rx buffers. Then refill the cleaned Rx 596cb7db356SBrett Creeley * buffers. Then bump tail at most one time. Grouping like this lets us avoid 597cb7db356SBrett Creeley * multiple tail writes per call. 598cdedef59SAnirudh Venkataramanan */ 599cdedef59SAnirudh Venkataramanan bool ice_alloc_rx_bufs(struct ice_ring *rx_ring, u16 cleaned_count) 600cdedef59SAnirudh Venkataramanan { 601cdedef59SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 602cdedef59SAnirudh Venkataramanan u16 ntu = rx_ring->next_to_use; 603cdedef59SAnirudh Venkataramanan struct ice_rx_buf *bi; 604cdedef59SAnirudh Venkataramanan 605cdedef59SAnirudh Venkataramanan /* do nothing if no valid netdev defined */ 606148beb61SHenry Tieman if ((!rx_ring->netdev && rx_ring->vsi->type != ICE_VSI_CTRL) || 607148beb61SHenry Tieman !cleaned_count) 608cdedef59SAnirudh Venkataramanan return false; 609cdedef59SAnirudh Venkataramanan 610f9867df6SAnirudh Venkataramanan /* get the Rx descriptor and buffer based on next_to_use */ 611cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, ntu); 612cdedef59SAnirudh Venkataramanan bi = &rx_ring->rx_buf[ntu]; 613cdedef59SAnirudh Venkataramanan 614cdedef59SAnirudh Venkataramanan do { 615a1e99685SBrett Creeley /* if we fail here, we have work remaining */ 616cdedef59SAnirudh Venkataramanan if (!ice_alloc_mapped_page(rx_ring, bi)) 617a1e99685SBrett Creeley break; 618cdedef59SAnirudh Venkataramanan 619a65f71feSMaciej Fijalkowski /* sync the buffer for use by the device */ 620a65f71feSMaciej Fijalkowski dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 621a65f71feSMaciej Fijalkowski bi->page_offset, 6227237f5b0SMaciej Fijalkowski rx_ring->rx_buf_len, 623a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE); 624a65f71feSMaciej Fijalkowski 625cdedef59SAnirudh Venkataramanan /* Refresh the desc even if buffer_addrs didn't change 626cdedef59SAnirudh Venkataramanan * because each write-back erases this info. 627cdedef59SAnirudh Venkataramanan */ 628cdedef59SAnirudh Venkataramanan rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 629cdedef59SAnirudh Venkataramanan 630cdedef59SAnirudh Venkataramanan rx_desc++; 631cdedef59SAnirudh Venkataramanan bi++; 632cdedef59SAnirudh Venkataramanan ntu++; 633cdedef59SAnirudh Venkataramanan if (unlikely(ntu == rx_ring->count)) { 634cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, 0); 635cdedef59SAnirudh Venkataramanan bi = rx_ring->rx_buf; 636cdedef59SAnirudh Venkataramanan ntu = 0; 637cdedef59SAnirudh Venkataramanan } 638cdedef59SAnirudh Venkataramanan 639cdedef59SAnirudh Venkataramanan /* clear the status bits for the next_to_use descriptor */ 640cdedef59SAnirudh Venkataramanan rx_desc->wb.status_error0 = 0; 641cdedef59SAnirudh Venkataramanan 642cdedef59SAnirudh Venkataramanan cleaned_count--; 643cdedef59SAnirudh Venkataramanan } while (cleaned_count); 644cdedef59SAnirudh Venkataramanan 645cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 646cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 647cdedef59SAnirudh Venkataramanan 648a1e99685SBrett Creeley return !!cleaned_count; 649cdedef59SAnirudh Venkataramanan } 6502b245cb2SAnirudh Venkataramanan 6512b245cb2SAnirudh Venkataramanan /** 6522b245cb2SAnirudh Venkataramanan * ice_page_is_reserved - check if reuse is possible 6532b245cb2SAnirudh Venkataramanan * @page: page struct to check 6542b245cb2SAnirudh Venkataramanan */ 6552b245cb2SAnirudh Venkataramanan static bool ice_page_is_reserved(struct page *page) 6562b245cb2SAnirudh Venkataramanan { 6572b245cb2SAnirudh Venkataramanan return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 6582b245cb2SAnirudh Venkataramanan } 6592b245cb2SAnirudh Venkataramanan 6602b245cb2SAnirudh Venkataramanan /** 6611d032bc7SMaciej Fijalkowski * ice_rx_buf_adjust_pg_offset - Prepare Rx buffer for reuse 6621d032bc7SMaciej Fijalkowski * @rx_buf: Rx buffer to adjust 6631d032bc7SMaciej Fijalkowski * @size: Size of adjustment 6642b245cb2SAnirudh Venkataramanan * 6651d032bc7SMaciej Fijalkowski * Update the offset within page so that Rx buf will be ready to be reused. 6661d032bc7SMaciej Fijalkowski * For systems with PAGE_SIZE < 8192 this function will flip the page offset 6671d032bc7SMaciej Fijalkowski * so the second half of page assigned to Rx buffer will be used, otherwise 6684ee656bbSTony Nguyen * the offset is moved by "size" bytes 6692b245cb2SAnirudh Venkataramanan */ 6701d032bc7SMaciej Fijalkowski static void 6711d032bc7SMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size) 6722b245cb2SAnirudh Venkataramanan { 6732b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE < 8192) 6741d032bc7SMaciej Fijalkowski /* flip page offset to other buffer */ 6751d032bc7SMaciej Fijalkowski rx_buf->page_offset ^= size; 6762b245cb2SAnirudh Venkataramanan #else 6771d032bc7SMaciej Fijalkowski /* move offset up to the next cache line */ 6781d032bc7SMaciej Fijalkowski rx_buf->page_offset += size; 6791d032bc7SMaciej Fijalkowski #endif 6802b245cb2SAnirudh Venkataramanan } 6812b245cb2SAnirudh Venkataramanan 6821d032bc7SMaciej Fijalkowski /** 683bbb97808SMaciej Fijalkowski * ice_can_reuse_rx_page - Determine if page can be reused for another Rx 684bbb97808SMaciej Fijalkowski * @rx_buf: buffer containing the page 685bbb97808SMaciej Fijalkowski * 686bbb97808SMaciej Fijalkowski * If page is reusable, we have a green light for calling ice_reuse_rx_page, 687bbb97808SMaciej Fijalkowski * which will assign the current buffer to the buffer that next_to_alloc is 688bbb97808SMaciej Fijalkowski * pointing to; otherwise, the DMA mapping needs to be destroyed and 689bbb97808SMaciej Fijalkowski * page freed 690bbb97808SMaciej Fijalkowski */ 6911d032bc7SMaciej Fijalkowski static bool ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf) 692bbb97808SMaciej Fijalkowski { 69303c66a13SMaciej Fijalkowski unsigned int pagecnt_bias = rx_buf->pagecnt_bias; 694bbb97808SMaciej Fijalkowski struct page *page = rx_buf->page; 6952b245cb2SAnirudh Venkataramanan 6962b245cb2SAnirudh Venkataramanan /* avoid re-using remote pages */ 6972b245cb2SAnirudh Venkataramanan if (unlikely(ice_page_is_reserved(page))) 6982b245cb2SAnirudh Venkataramanan return false; 6992b245cb2SAnirudh Venkataramanan 7002b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE < 8192) 7012b245cb2SAnirudh Venkataramanan /* if we are only owner of page we can reuse it */ 70203c66a13SMaciej Fijalkowski if (unlikely((page_count(page) - pagecnt_bias) > 1)) 7032b245cb2SAnirudh Venkataramanan return false; 7042b245cb2SAnirudh Venkataramanan #else 7057237f5b0SMaciej Fijalkowski #define ICE_LAST_OFFSET \ 7067237f5b0SMaciej Fijalkowski (SKB_WITH_OVERHEAD(PAGE_SIZE) - ICE_RXBUF_2048) 7077237f5b0SMaciej Fijalkowski if (rx_buf->page_offset > ICE_LAST_OFFSET) 7082b245cb2SAnirudh Venkataramanan return false; 7092b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE < 8192) */ 7102b245cb2SAnirudh Venkataramanan 71103c66a13SMaciej Fijalkowski /* If we have drained the page fragment pool we need to update 71203c66a13SMaciej Fijalkowski * the pagecnt_bias and page count so that we fully restock the 71303c66a13SMaciej Fijalkowski * number of references the driver holds. 7142b245cb2SAnirudh Venkataramanan */ 71503c66a13SMaciej Fijalkowski if (unlikely(pagecnt_bias == 1)) { 71603c66a13SMaciej Fijalkowski page_ref_add(page, USHRT_MAX - 1); 71703c66a13SMaciej Fijalkowski rx_buf->pagecnt_bias = USHRT_MAX; 71803c66a13SMaciej Fijalkowski } 7192b245cb2SAnirudh Venkataramanan 7202b245cb2SAnirudh Venkataramanan return true; 7212b245cb2SAnirudh Venkataramanan } 7222b245cb2SAnirudh Venkataramanan 7232b245cb2SAnirudh Venkataramanan /** 724712edbbbSMaciej Fijalkowski * ice_add_rx_frag - Add contents of Rx buffer to sk_buff as a frag 7257237f5b0SMaciej Fijalkowski * @rx_ring: Rx descriptor ring to transact packets on 7262b245cb2SAnirudh Venkataramanan * @rx_buf: buffer containing page to add 727712edbbbSMaciej Fijalkowski * @skb: sk_buff to place the data into 728712edbbbSMaciej Fijalkowski * @size: packet length from rx_desc 7292b245cb2SAnirudh Venkataramanan * 7302b245cb2SAnirudh Venkataramanan * This function will add the data contained in rx_buf->page to the skb. 731712edbbbSMaciej Fijalkowski * It will just attach the page as a frag to the skb. 732712edbbbSMaciej Fijalkowski * The function will then update the page offset. 7332b245cb2SAnirudh Venkataramanan */ 7341d032bc7SMaciej Fijalkowski static void 7357237f5b0SMaciej Fijalkowski ice_add_rx_frag(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, 7367237f5b0SMaciej Fijalkowski struct sk_buff *skb, unsigned int size) 7372b245cb2SAnirudh Venkataramanan { 738712edbbbSMaciej Fijalkowski #if (PAGE_SIZE >= 8192) 73959bb0808SMaciej Fijalkowski unsigned int truesize = SKB_DATA_ALIGN(size + ice_rx_offset(rx_ring)); 7402b245cb2SAnirudh Venkataramanan #else 7417237f5b0SMaciej Fijalkowski unsigned int truesize = ice_rx_pg_size(rx_ring) / 2; 742712edbbbSMaciej Fijalkowski #endif 7431857ca42SMaciej Fijalkowski 744ac6f733aSMitch Williams if (!size) 745ac6f733aSMitch Williams return; 746712edbbbSMaciej Fijalkowski skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page, 747712edbbbSMaciej Fijalkowski rx_buf->page_offset, size, truesize); 7482b245cb2SAnirudh Venkataramanan 749712edbbbSMaciej Fijalkowski /* page is being used so we must update the page offset */ 7501d032bc7SMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(rx_buf, truesize); 7512b245cb2SAnirudh Venkataramanan } 7522b245cb2SAnirudh Venkataramanan 7532b245cb2SAnirudh Venkataramanan /** 7542b245cb2SAnirudh Venkataramanan * ice_reuse_rx_page - page flip buffer and store it back on the ring 755d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to store buffers on 7562b245cb2SAnirudh Venkataramanan * @old_buf: donor buffer to have page reused 7572b245cb2SAnirudh Venkataramanan * 7582b245cb2SAnirudh Venkataramanan * Synchronizes page for reuse by the adapter 7592b245cb2SAnirudh Venkataramanan */ 760c8b7abddSBruce Allan static void 761c8b7abddSBruce Allan ice_reuse_rx_page(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf) 7622b245cb2SAnirudh Venkataramanan { 7632b245cb2SAnirudh Venkataramanan u16 nta = rx_ring->next_to_alloc; 7642b245cb2SAnirudh Venkataramanan struct ice_rx_buf *new_buf; 7652b245cb2SAnirudh Venkataramanan 7662b245cb2SAnirudh Venkataramanan new_buf = &rx_ring->rx_buf[nta]; 7672b245cb2SAnirudh Venkataramanan 7682b245cb2SAnirudh Venkataramanan /* update, and store next to alloc */ 7692b245cb2SAnirudh Venkataramanan nta++; 7702b245cb2SAnirudh Venkataramanan rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 7712b245cb2SAnirudh Venkataramanan 772712edbbbSMaciej Fijalkowski /* Transfer page from old buffer to new buffer. 773712edbbbSMaciej Fijalkowski * Move each member individually to avoid possible store 774712edbbbSMaciej Fijalkowski * forwarding stalls and unnecessary copy of skb. 775712edbbbSMaciej Fijalkowski */ 776712edbbbSMaciej Fijalkowski new_buf->dma = old_buf->dma; 777712edbbbSMaciej Fijalkowski new_buf->page = old_buf->page; 778712edbbbSMaciej Fijalkowski new_buf->page_offset = old_buf->page_offset; 779712edbbbSMaciej Fijalkowski new_buf->pagecnt_bias = old_buf->pagecnt_bias; 7802b245cb2SAnirudh Venkataramanan } 7812b245cb2SAnirudh Venkataramanan 7822b245cb2SAnirudh Venkataramanan /** 7836c869cb7SMaciej Fijalkowski * ice_get_rx_buf - Fetch Rx buffer and synchronize data for use 784d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to transact packets on 785712edbbbSMaciej Fijalkowski * @skb: skb to be used 7866c869cb7SMaciej Fijalkowski * @size: size of buffer to add to skb 7872b245cb2SAnirudh Venkataramanan * 7886c869cb7SMaciej Fijalkowski * This function will pull an Rx buffer from the ring and synchronize it 7896c869cb7SMaciej Fijalkowski * for use by the CPU. 7902b245cb2SAnirudh Venkataramanan */ 7916c869cb7SMaciej Fijalkowski static struct ice_rx_buf * 792712edbbbSMaciej Fijalkowski ice_get_rx_buf(struct ice_ring *rx_ring, struct sk_buff **skb, 793712edbbbSMaciej Fijalkowski const unsigned int size) 7942b245cb2SAnirudh Venkataramanan { 7952b245cb2SAnirudh Venkataramanan struct ice_rx_buf *rx_buf; 7962b245cb2SAnirudh Venkataramanan 7972b245cb2SAnirudh Venkataramanan rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean]; 7986c869cb7SMaciej Fijalkowski prefetchw(rx_buf->page); 799712edbbbSMaciej Fijalkowski *skb = rx_buf->skb; 8002b245cb2SAnirudh Venkataramanan 801ac6f733aSMitch Williams if (!size) 802ac6f733aSMitch Williams return rx_buf; 8036c869cb7SMaciej Fijalkowski /* we are reusing so sync this buffer for CPU use */ 8046c869cb7SMaciej Fijalkowski dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 8056c869cb7SMaciej Fijalkowski rx_buf->page_offset, size, 8066c869cb7SMaciej Fijalkowski DMA_FROM_DEVICE); 8072b245cb2SAnirudh Venkataramanan 80803c66a13SMaciej Fijalkowski /* We have pulled a buffer for use, so decrement pagecnt_bias */ 80903c66a13SMaciej Fijalkowski rx_buf->pagecnt_bias--; 81003c66a13SMaciej Fijalkowski 8116c869cb7SMaciej Fijalkowski return rx_buf; 8126c869cb7SMaciej Fijalkowski } 8136c869cb7SMaciej Fijalkowski 8146c869cb7SMaciej Fijalkowski /** 815aaf27254SMaciej Fijalkowski * ice_build_skb - Build skb around an existing buffer 816aaf27254SMaciej Fijalkowski * @rx_ring: Rx descriptor ring to transact packets on 817aaf27254SMaciej Fijalkowski * @rx_buf: Rx buffer to pull data from 818aaf27254SMaciej Fijalkowski * @xdp: xdp_buff pointing to the data 819aaf27254SMaciej Fijalkowski * 820aaf27254SMaciej Fijalkowski * This function builds an skb around an existing Rx buffer, taking care 821aaf27254SMaciej Fijalkowski * to set up the skb correctly and avoid any memcpy overhead. 822aaf27254SMaciej Fijalkowski */ 823aaf27254SMaciej Fijalkowski static struct sk_buff * 824aaf27254SMaciej Fijalkowski ice_build_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, 825aaf27254SMaciej Fijalkowski struct xdp_buff *xdp) 826aaf27254SMaciej Fijalkowski { 82788865fc4SKarol Kolacinski u8 metasize = xdp->data - xdp->data_meta; 828aaf27254SMaciej Fijalkowski #if (PAGE_SIZE < 8192) 829aaf27254SMaciej Fijalkowski unsigned int truesize = ice_rx_pg_size(rx_ring) / 2; 830aaf27254SMaciej Fijalkowski #else 831aaf27254SMaciej Fijalkowski unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 832aaf27254SMaciej Fijalkowski SKB_DATA_ALIGN(xdp->data_end - 833aaf27254SMaciej Fijalkowski xdp->data_hard_start); 834aaf27254SMaciej Fijalkowski #endif 835aaf27254SMaciej Fijalkowski struct sk_buff *skb; 836aaf27254SMaciej Fijalkowski 837aaf27254SMaciej Fijalkowski /* Prefetch first cache line of first page. If xdp->data_meta 838aaf27254SMaciej Fijalkowski * is unused, this points exactly as xdp->data, otherwise we 839aaf27254SMaciej Fijalkowski * likely have a consumer accessing first few bytes of meta 840aaf27254SMaciej Fijalkowski * data, and then actual data. 841aaf27254SMaciej Fijalkowski */ 842aaf27254SMaciej Fijalkowski prefetch(xdp->data_meta); 843aaf27254SMaciej Fijalkowski #if L1_CACHE_BYTES < 128 844aaf27254SMaciej Fijalkowski prefetch((void *)(xdp->data + L1_CACHE_BYTES)); 845aaf27254SMaciej Fijalkowski #endif 846aaf27254SMaciej Fijalkowski /* build an skb around the page buffer */ 847aaf27254SMaciej Fijalkowski skb = build_skb(xdp->data_hard_start, truesize); 848aaf27254SMaciej Fijalkowski if (unlikely(!skb)) 849aaf27254SMaciej Fijalkowski return NULL; 850aaf27254SMaciej Fijalkowski 851aaf27254SMaciej Fijalkowski /* must to record Rx queue, otherwise OS features such as 852aaf27254SMaciej Fijalkowski * symmetric queue won't work 853aaf27254SMaciej Fijalkowski */ 854aaf27254SMaciej Fijalkowski skb_record_rx_queue(skb, rx_ring->q_index); 855aaf27254SMaciej Fijalkowski 856aaf27254SMaciej Fijalkowski /* update pointers within the skb to store the data */ 857aaf27254SMaciej Fijalkowski skb_reserve(skb, xdp->data - xdp->data_hard_start); 858aaf27254SMaciej Fijalkowski __skb_put(skb, xdp->data_end - xdp->data); 859aaf27254SMaciej Fijalkowski if (metasize) 860aaf27254SMaciej Fijalkowski skb_metadata_set(skb, metasize); 861aaf27254SMaciej Fijalkowski 862aaf27254SMaciej Fijalkowski /* buffer is used by skb, update page_offset */ 863aaf27254SMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(rx_buf, truesize); 864aaf27254SMaciej Fijalkowski 865aaf27254SMaciej Fijalkowski return skb; 866aaf27254SMaciej Fijalkowski } 867aaf27254SMaciej Fijalkowski 868aaf27254SMaciej Fijalkowski /** 869712edbbbSMaciej Fijalkowski * ice_construct_skb - Allocate skb and populate it 8702b245cb2SAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to transact packets on 8716c869cb7SMaciej Fijalkowski * @rx_buf: Rx buffer to pull data from 872efc2214bSMaciej Fijalkowski * @xdp: xdp_buff pointing to the data 8732b245cb2SAnirudh Venkataramanan * 874712edbbbSMaciej Fijalkowski * This function allocates an skb. It then populates it with the page 875712edbbbSMaciej Fijalkowski * data from the current receive descriptor, taking care to set up the 876712edbbbSMaciej Fijalkowski * skb correctly. 8772b245cb2SAnirudh Venkataramanan */ 878c8b7abddSBruce Allan static struct sk_buff * 879712edbbbSMaciej Fijalkowski ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, 880efc2214bSMaciej Fijalkowski struct xdp_buff *xdp) 8812b245cb2SAnirudh Venkataramanan { 882efc2214bSMaciej Fijalkowski unsigned int size = xdp->data_end - xdp->data; 883712edbbbSMaciej Fijalkowski unsigned int headlen; 884712edbbbSMaciej Fijalkowski struct sk_buff *skb; 8852b245cb2SAnirudh Venkataramanan 8862b245cb2SAnirudh Venkataramanan /* prefetch first cache line of first page */ 887efc2214bSMaciej Fijalkowski prefetch(xdp->data); 8882b245cb2SAnirudh Venkataramanan #if L1_CACHE_BYTES < 128 889efc2214bSMaciej Fijalkowski prefetch((void *)(xdp->data + L1_CACHE_BYTES)); 8902b245cb2SAnirudh Venkataramanan #endif /* L1_CACHE_BYTES */ 8912b245cb2SAnirudh Venkataramanan 8922b245cb2SAnirudh Venkataramanan /* allocate a skb to store the frags */ 893712edbbbSMaciej Fijalkowski skb = __napi_alloc_skb(&rx_ring->q_vector->napi, ICE_RX_HDR_SIZE, 8942b245cb2SAnirudh Venkataramanan GFP_ATOMIC | __GFP_NOWARN); 895712edbbbSMaciej Fijalkowski if (unlikely(!skb)) 8962b245cb2SAnirudh Venkataramanan return NULL; 8972b245cb2SAnirudh Venkataramanan 8982b245cb2SAnirudh Venkataramanan skb_record_rx_queue(skb, rx_ring->q_index); 899712edbbbSMaciej Fijalkowski /* Determine available headroom for copy */ 900712edbbbSMaciej Fijalkowski headlen = size; 901712edbbbSMaciej Fijalkowski if (headlen > ICE_RX_HDR_SIZE) 902efc2214bSMaciej Fijalkowski headlen = eth_get_headlen(skb->dev, xdp->data, ICE_RX_HDR_SIZE); 9032b245cb2SAnirudh Venkataramanan 904712edbbbSMaciej Fijalkowski /* align pull length to size of long to optimize memcpy performance */ 905efc2214bSMaciej Fijalkowski memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, 906efc2214bSMaciej Fijalkowski sizeof(long))); 907712edbbbSMaciej Fijalkowski 908712edbbbSMaciej Fijalkowski /* if we exhaust the linear part then add what is left as a frag */ 909712edbbbSMaciej Fijalkowski size -= headlen; 910712edbbbSMaciej Fijalkowski if (size) { 911712edbbbSMaciej Fijalkowski #if (PAGE_SIZE >= 8192) 912712edbbbSMaciej Fijalkowski unsigned int truesize = SKB_DATA_ALIGN(size); 913712edbbbSMaciej Fijalkowski #else 9147237f5b0SMaciej Fijalkowski unsigned int truesize = ice_rx_pg_size(rx_ring) / 2; 915712edbbbSMaciej Fijalkowski #endif 916712edbbbSMaciej Fijalkowski skb_add_rx_frag(skb, 0, rx_buf->page, 917712edbbbSMaciej Fijalkowski rx_buf->page_offset + headlen, size, truesize); 918712edbbbSMaciej Fijalkowski /* buffer is used by skb, update page_offset */ 919712edbbbSMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(rx_buf, truesize); 9202b245cb2SAnirudh Venkataramanan } else { 921712edbbbSMaciej Fijalkowski /* buffer is unused, reset bias back to rx_buf; data was copied 922712edbbbSMaciej Fijalkowski * onto skb's linear part so there's no need for adjusting 923712edbbbSMaciej Fijalkowski * page offset and we can reuse this buffer as-is 924712edbbbSMaciej Fijalkowski */ 925712edbbbSMaciej Fijalkowski rx_buf->pagecnt_bias++; 9262b245cb2SAnirudh Venkataramanan } 9272b245cb2SAnirudh Venkataramanan 9282b245cb2SAnirudh Venkataramanan return skb; 9292b245cb2SAnirudh Venkataramanan } 9302b245cb2SAnirudh Venkataramanan 9312b245cb2SAnirudh Venkataramanan /** 9321d032bc7SMaciej Fijalkowski * ice_put_rx_buf - Clean up used buffer and either recycle or free 9331d032bc7SMaciej Fijalkowski * @rx_ring: Rx descriptor ring to transact packets on 9341d032bc7SMaciej Fijalkowski * @rx_buf: Rx buffer to pull data from 9352b245cb2SAnirudh Venkataramanan * 936efc2214bSMaciej Fijalkowski * This function will update next_to_clean and then clean up the contents 937efc2214bSMaciej Fijalkowski * of the rx_buf. It will either recycle the buffer or unmap it and free 938efc2214bSMaciej Fijalkowski * the associated resources. 9392b245cb2SAnirudh Venkataramanan */ 9401d032bc7SMaciej Fijalkowski static void ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf) 9412b245cb2SAnirudh Venkataramanan { 94288865fc4SKarol Kolacinski u16 ntc = rx_ring->next_to_clean + 1; 943efc2214bSMaciej Fijalkowski 944efc2214bSMaciej Fijalkowski /* fetch, update, and store next to clean */ 945efc2214bSMaciej Fijalkowski ntc = (ntc < rx_ring->count) ? ntc : 0; 946efc2214bSMaciej Fijalkowski rx_ring->next_to_clean = ntc; 947efc2214bSMaciej Fijalkowski 948ac6f733aSMitch Williams if (!rx_buf) 949ac6f733aSMitch Williams return; 950ac6f733aSMitch Williams 9511d032bc7SMaciej Fijalkowski if (ice_can_reuse_rx_page(rx_buf)) { 952ac6f733aSMitch Williams /* hand second half of page back to the ring */ 9532b245cb2SAnirudh Venkataramanan ice_reuse_rx_page(rx_ring, rx_buf); 9542b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 9552b245cb2SAnirudh Venkataramanan } else { 9562b245cb2SAnirudh Venkataramanan /* we are not reusing the buffer so unmap it */ 9577237f5b0SMaciej Fijalkowski dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma, 9587237f5b0SMaciej Fijalkowski ice_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 9597237f5b0SMaciej Fijalkowski ICE_RX_DMA_ATTR); 96003c66a13SMaciej Fijalkowski __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias); 9612b245cb2SAnirudh Venkataramanan } 9622b245cb2SAnirudh Venkataramanan 9632b245cb2SAnirudh Venkataramanan /* clear contents of buffer_info */ 9642b245cb2SAnirudh Venkataramanan rx_buf->page = NULL; 965712edbbbSMaciej Fijalkowski rx_buf->skb = NULL; 9662b245cb2SAnirudh Venkataramanan } 9672b245cb2SAnirudh Venkataramanan 9682b245cb2SAnirudh Venkataramanan /** 9692b245cb2SAnirudh Venkataramanan * ice_is_non_eop - process handling of non-EOP buffers 9702b245cb2SAnirudh Venkataramanan * @rx_ring: Rx ring being processed 9712b245cb2SAnirudh Venkataramanan * @rx_desc: Rx descriptor for current buffer 9722b245cb2SAnirudh Venkataramanan * @skb: Current socket buffer containing buffer in progress 9732b245cb2SAnirudh Venkataramanan * 974efc2214bSMaciej Fijalkowski * If the buffer is an EOP buffer, this function exits returning false, 975efc2214bSMaciej Fijalkowski * otherwise return true indicating that this is in fact a non-EOP buffer. 9762b245cb2SAnirudh Venkataramanan */ 977c8b7abddSBruce Allan static bool 978c8b7abddSBruce Allan ice_is_non_eop(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, 9792b245cb2SAnirudh Venkataramanan struct sk_buff *skb) 9802b245cb2SAnirudh Venkataramanan { 9812b245cb2SAnirudh Venkataramanan /* if we are the last buffer then there is nothing else to do */ 9822b245cb2SAnirudh Venkataramanan #define ICE_RXD_EOF BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S) 9832b245cb2SAnirudh Venkataramanan if (likely(ice_test_staterr(rx_desc, ICE_RXD_EOF))) 9842b245cb2SAnirudh Venkataramanan return false; 9852b245cb2SAnirudh Venkataramanan 9862b245cb2SAnirudh Venkataramanan /* place skb in next buffer to be received */ 987efc2214bSMaciej Fijalkowski rx_ring->rx_buf[rx_ring->next_to_clean].skb = skb; 9882b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.non_eop_descs++; 9892b245cb2SAnirudh Venkataramanan 9902b245cb2SAnirudh Venkataramanan return true; 9912b245cb2SAnirudh Venkataramanan } 9922b245cb2SAnirudh Venkataramanan 9932b245cb2SAnirudh Venkataramanan /** 9942b245cb2SAnirudh Venkataramanan * ice_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 995d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to transact packets on 9962b245cb2SAnirudh Venkataramanan * @budget: Total limit on number of packets to process 9972b245cb2SAnirudh Venkataramanan * 9982b245cb2SAnirudh Venkataramanan * This function provides a "bounce buffer" approach to Rx interrupt 9992b245cb2SAnirudh Venkataramanan * processing. The advantage to this is that on systems that have 10002b245cb2SAnirudh Venkataramanan * expensive overhead for IOMMU access this provides a means of avoiding 10012b245cb2SAnirudh Venkataramanan * it by maintaining the mapping of the page to the system. 10022b245cb2SAnirudh Venkataramanan * 10032b245cb2SAnirudh Venkataramanan * Returns amount of work completed 10042b245cb2SAnirudh Venkataramanan */ 1005148beb61SHenry Tieman int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) 10062b245cb2SAnirudh Venkataramanan { 10072b245cb2SAnirudh Venkataramanan unsigned int total_rx_bytes = 0, total_rx_pkts = 0; 10082b245cb2SAnirudh Venkataramanan u16 cleaned_count = ICE_DESC_UNUSED(rx_ring); 1009efc2214bSMaciej Fijalkowski unsigned int xdp_res, xdp_xmit = 0; 1010efc2214bSMaciej Fijalkowski struct bpf_prog *xdp_prog = NULL; 1011efc2214bSMaciej Fijalkowski struct xdp_buff xdp; 1012cb7db356SBrett Creeley bool failure; 10132b245cb2SAnirudh Venkataramanan 1014efc2214bSMaciej Fijalkowski xdp.rxq = &rx_ring->xdp_rxq; 1015d4ecdbf7SJesper Dangaard Brouer /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ 1016d4ecdbf7SJesper Dangaard Brouer #if (PAGE_SIZE < 8192) 1017d4ecdbf7SJesper Dangaard Brouer xdp.frame_sz = ice_rx_frame_truesize(rx_ring, 0); 1018d4ecdbf7SJesper Dangaard Brouer #endif 1019efc2214bSMaciej Fijalkowski 1020f9867df6SAnirudh Venkataramanan /* start the loop to process Rx packets bounded by 'budget' */ 10212b245cb2SAnirudh Venkataramanan while (likely(total_rx_pkts < (unsigned int)budget)) { 10222b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 10236c869cb7SMaciej Fijalkowski struct ice_rx_buf *rx_buf; 10242b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 10256c869cb7SMaciej Fijalkowski unsigned int size; 10262b245cb2SAnirudh Venkataramanan u16 stat_err_bits; 10272b245cb2SAnirudh Venkataramanan u16 vlan_tag = 0; 1028d76a60baSAnirudh Venkataramanan u8 rx_ptype; 10292b245cb2SAnirudh Venkataramanan 1030f9867df6SAnirudh Venkataramanan /* get the Rx desc from Rx ring based on 'next_to_clean' */ 10312b245cb2SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); 10322b245cb2SAnirudh Venkataramanan 10332b245cb2SAnirudh Venkataramanan /* status_error_len will always be zero for unused descriptors 10342b245cb2SAnirudh Venkataramanan * because it's cleared in cleanup, and overlaps with hdr_addr 10352b245cb2SAnirudh Venkataramanan * which is always zero because packet split isn't used, if the 10362b245cb2SAnirudh Venkataramanan * hardware wrote DD then it will be non-zero 10372b245cb2SAnirudh Venkataramanan */ 10382b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 10392b245cb2SAnirudh Venkataramanan if (!ice_test_staterr(rx_desc, stat_err_bits)) 10402b245cb2SAnirudh Venkataramanan break; 10412b245cb2SAnirudh Venkataramanan 10422b245cb2SAnirudh Venkataramanan /* This memory barrier is needed to keep us from reading 10432b245cb2SAnirudh Venkataramanan * any other fields out of the rx_desc until we know the 10442b245cb2SAnirudh Venkataramanan * DD bit is set. 10452b245cb2SAnirudh Venkataramanan */ 10462b245cb2SAnirudh Venkataramanan dma_rmb(); 10472b245cb2SAnirudh Venkataramanan 1048148beb61SHenry Tieman if (rx_desc->wb.rxdid == FDIR_DESC_RXDID || !rx_ring->netdev) { 1049148beb61SHenry Tieman ice_put_rx_buf(rx_ring, NULL); 1050148beb61SHenry Tieman cleaned_count++; 1051148beb61SHenry Tieman continue; 1052148beb61SHenry Tieman } 1053148beb61SHenry Tieman 10546c869cb7SMaciej Fijalkowski size = le16_to_cpu(rx_desc->wb.pkt_len) & 10556c869cb7SMaciej Fijalkowski ICE_RX_FLX_DESC_PKT_LEN_M; 10562b245cb2SAnirudh Venkataramanan 1057ac6f733aSMitch Williams /* retrieve a buffer from the ring */ 1058712edbbbSMaciej Fijalkowski rx_buf = ice_get_rx_buf(rx_ring, &skb, size); 1059ac6f733aSMitch Williams 1060efc2214bSMaciej Fijalkowski if (!size) { 1061efc2214bSMaciej Fijalkowski xdp.data = NULL; 1062efc2214bSMaciej Fijalkowski xdp.data_end = NULL; 1063aaf27254SMaciej Fijalkowski xdp.data_hard_start = NULL; 1064aaf27254SMaciej Fijalkowski xdp.data_meta = NULL; 1065efc2214bSMaciej Fijalkowski goto construct_skb; 1066efc2214bSMaciej Fijalkowski } 1067efc2214bSMaciej Fijalkowski 1068efc2214bSMaciej Fijalkowski xdp.data = page_address(rx_buf->page) + rx_buf->page_offset; 1069efc2214bSMaciej Fijalkowski xdp.data_hard_start = xdp.data - ice_rx_offset(rx_ring); 1070aaf27254SMaciej Fijalkowski xdp.data_meta = xdp.data; 1071efc2214bSMaciej Fijalkowski xdp.data_end = xdp.data + size; 1072d4ecdbf7SJesper Dangaard Brouer #if (PAGE_SIZE > 4096) 1073d4ecdbf7SJesper Dangaard Brouer /* At larger PAGE_SIZE, frame_sz depend on len size */ 1074d4ecdbf7SJesper Dangaard Brouer xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size); 1075d4ecdbf7SJesper Dangaard Brouer #endif 1076efc2214bSMaciej Fijalkowski 1077efc2214bSMaciej Fijalkowski rcu_read_lock(); 1078efc2214bSMaciej Fijalkowski xdp_prog = READ_ONCE(rx_ring->xdp_prog); 1079efc2214bSMaciej Fijalkowski if (!xdp_prog) { 1080efc2214bSMaciej Fijalkowski rcu_read_unlock(); 1081efc2214bSMaciej Fijalkowski goto construct_skb; 1082efc2214bSMaciej Fijalkowski } 1083efc2214bSMaciej Fijalkowski 1084efc2214bSMaciej Fijalkowski xdp_res = ice_run_xdp(rx_ring, &xdp, xdp_prog); 1085efc2214bSMaciej Fijalkowski rcu_read_unlock(); 108659bb0808SMaciej Fijalkowski if (!xdp_res) 108759bb0808SMaciej Fijalkowski goto construct_skb; 1088efc2214bSMaciej Fijalkowski if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) { 1089efc2214bSMaciej Fijalkowski xdp_xmit |= xdp_res; 1090d4ecdbf7SJesper Dangaard Brouer ice_rx_buf_adjust_pg_offset(rx_buf, xdp.frame_sz); 1091efc2214bSMaciej Fijalkowski } else { 1092efc2214bSMaciej Fijalkowski rx_buf->pagecnt_bias++; 1093efc2214bSMaciej Fijalkowski } 1094efc2214bSMaciej Fijalkowski total_rx_bytes += size; 1095efc2214bSMaciej Fijalkowski total_rx_pkts++; 1096efc2214bSMaciej Fijalkowski 1097efc2214bSMaciej Fijalkowski cleaned_count++; 1098efc2214bSMaciej Fijalkowski ice_put_rx_buf(rx_ring, rx_buf); 1099efc2214bSMaciej Fijalkowski continue; 1100efc2214bSMaciej Fijalkowski construct_skb: 11011f45ebe0SMitch Williams if (skb) { 11027237f5b0SMaciej Fijalkowski ice_add_rx_frag(rx_ring, rx_buf, skb, size); 11031f45ebe0SMitch Williams } else if (likely(xdp.data)) { 11041f45ebe0SMitch Williams if (ice_ring_uses_build_skb(rx_ring)) 1105aaf27254SMaciej Fijalkowski skb = ice_build_skb(rx_ring, rx_buf, &xdp); 1106712edbbbSMaciej Fijalkowski else 1107efc2214bSMaciej Fijalkowski skb = ice_construct_skb(rx_ring, rx_buf, &xdp); 11081f45ebe0SMitch Williams } 1109712edbbbSMaciej Fijalkowski /* exit if we failed to retrieve a buffer */ 1110712edbbbSMaciej Fijalkowski if (!skb) { 1111712edbbbSMaciej Fijalkowski rx_ring->rx_stats.alloc_buf_failed++; 1112ac6f733aSMitch Williams if (rx_buf) 1113712edbbbSMaciej Fijalkowski rx_buf->pagecnt_bias++; 11142b245cb2SAnirudh Venkataramanan break; 1115712edbbbSMaciej Fijalkowski } 11162b245cb2SAnirudh Venkataramanan 11171d032bc7SMaciej Fijalkowski ice_put_rx_buf(rx_ring, rx_buf); 11182b245cb2SAnirudh Venkataramanan cleaned_count++; 11192b245cb2SAnirudh Venkataramanan 11202b245cb2SAnirudh Venkataramanan /* skip if it is NOP desc */ 11212b245cb2SAnirudh Venkataramanan if (ice_is_non_eop(rx_ring, rx_desc, skb)) 11222b245cb2SAnirudh Venkataramanan continue; 11232b245cb2SAnirudh Venkataramanan 11242b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S); 11252b245cb2SAnirudh Venkataramanan if (unlikely(ice_test_staterr(rx_desc, stat_err_bits))) { 11262b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 11272b245cb2SAnirudh Venkataramanan continue; 11282b245cb2SAnirudh Venkataramanan } 11292b245cb2SAnirudh Venkataramanan 11302b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S); 11312b245cb2SAnirudh Venkataramanan if (ice_test_staterr(rx_desc, stat_err_bits)) 11322b245cb2SAnirudh Venkataramanan vlan_tag = le16_to_cpu(rx_desc->wb.l2tag1); 11332b245cb2SAnirudh Venkataramanan 1134133f4883SKrzysztof Kazimierczak /* pad the skb if needed, to make a valid ethernet frame */ 1135133f4883SKrzysztof Kazimierczak if (eth_skb_pad(skb)) { 11362b245cb2SAnirudh Venkataramanan skb = NULL; 11372b245cb2SAnirudh Venkataramanan continue; 11382b245cb2SAnirudh Venkataramanan } 11392b245cb2SAnirudh Venkataramanan 11402b245cb2SAnirudh Venkataramanan /* probably a little skewed due to removing CRC */ 11412b245cb2SAnirudh Venkataramanan total_rx_bytes += skb->len; 11422b245cb2SAnirudh Venkataramanan 1143d76a60baSAnirudh Venkataramanan /* populate checksum, VLAN, and protocol */ 11446503b659SJesse Brandeburg rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) & 11456503b659SJesse Brandeburg ICE_RX_FLEX_DESC_PTYPE_M; 11466503b659SJesse Brandeburg 1147d76a60baSAnirudh Venkataramanan ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); 1148d76a60baSAnirudh Venkataramanan 11492b245cb2SAnirudh Venkataramanan /* send completed skb up the stack */ 11502b245cb2SAnirudh Venkataramanan ice_receive_skb(rx_ring, skb, vlan_tag); 11512b245cb2SAnirudh Venkataramanan 11522b245cb2SAnirudh Venkataramanan /* update budget accounting */ 11532b245cb2SAnirudh Venkataramanan total_rx_pkts++; 11542b245cb2SAnirudh Venkataramanan } 11552b245cb2SAnirudh Venkataramanan 1156cb7db356SBrett Creeley /* return up to cleaned_count buffers to hardware */ 1157cb7db356SBrett Creeley failure = ice_alloc_rx_bufs(rx_ring, cleaned_count); 1158cb7db356SBrett Creeley 1159efc2214bSMaciej Fijalkowski if (xdp_prog) 1160efc2214bSMaciej Fijalkowski ice_finalize_xdp_rx(rx_ring, xdp_xmit); 1161efc2214bSMaciej Fijalkowski 11622d4238f5SKrzysztof Kazimierczak ice_update_rx_ring_stats(rx_ring, total_rx_pkts, total_rx_bytes); 11632b245cb2SAnirudh Venkataramanan 11642b245cb2SAnirudh Venkataramanan /* guarantee a trip back through this routine if there was a failure */ 11652b245cb2SAnirudh Venkataramanan return failure ? budget : (int)total_rx_pkts; 11662b245cb2SAnirudh Venkataramanan } 11672b245cb2SAnirudh Venkataramanan 11682b245cb2SAnirudh Venkataramanan /** 1169711987bbSBrett Creeley * ice_adjust_itr_by_size_and_speed - Adjust ITR based on current traffic 1170711987bbSBrett Creeley * @port_info: port_info structure containing the current link speed 1171711987bbSBrett Creeley * @avg_pkt_size: average size of Tx or Rx packets based on clean routine 11722f2da36eSAnirudh Venkataramanan * @itr: ITR value to update 1173711987bbSBrett Creeley * 1174711987bbSBrett Creeley * Calculate how big of an increment should be applied to the ITR value passed 1175711987bbSBrett Creeley * in based on wmem_default, SKB overhead, Ethernet overhead, and the current 1176711987bbSBrett Creeley * link speed. 1177711987bbSBrett Creeley * 1178711987bbSBrett Creeley * The following is a calculation derived from: 1179711987bbSBrett Creeley * wmem_default / (size + overhead) = desired_pkts_per_int 1180711987bbSBrett Creeley * rate / bits_per_byte / (size + Ethernet overhead) = pkt_rate 1181711987bbSBrett Creeley * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value 1182711987bbSBrett Creeley * 1183711987bbSBrett Creeley * Assuming wmem_default is 212992 and overhead is 640 bytes per 1184711987bbSBrett Creeley * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the 1185711987bbSBrett Creeley * formula down to: 1186711987bbSBrett Creeley * 1187711987bbSBrett Creeley * wmem_default * bits_per_byte * usecs_per_sec pkt_size + 24 1188711987bbSBrett Creeley * ITR = -------------------------------------------- * -------------- 1189711987bbSBrett Creeley * rate pkt_size + 640 1190711987bbSBrett Creeley */ 1191711987bbSBrett Creeley static unsigned int 1192711987bbSBrett Creeley ice_adjust_itr_by_size_and_speed(struct ice_port_info *port_info, 1193711987bbSBrett Creeley unsigned int avg_pkt_size, 1194711987bbSBrett Creeley unsigned int itr) 119564a59d05SAnirudh Venkataramanan { 1196711987bbSBrett Creeley switch (port_info->phy.link_info.link_speed) { 1197711987bbSBrett Creeley case ICE_AQ_LINK_SPEED_100GB: 1198711987bbSBrett Creeley itr += DIV_ROUND_UP(17 * (avg_pkt_size + 24), 1199711987bbSBrett Creeley avg_pkt_size + 640); 1200711987bbSBrett Creeley break; 1201711987bbSBrett Creeley case ICE_AQ_LINK_SPEED_50GB: 1202711987bbSBrett Creeley itr += DIV_ROUND_UP(34 * (avg_pkt_size + 24), 1203711987bbSBrett Creeley avg_pkt_size + 640); 1204711987bbSBrett Creeley break; 120564a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_40GB: 1206711987bbSBrett Creeley itr += DIV_ROUND_UP(43 * (avg_pkt_size + 24), 1207711987bbSBrett Creeley avg_pkt_size + 640); 1208711987bbSBrett Creeley break; 120964a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_25GB: 1210711987bbSBrett Creeley itr += DIV_ROUND_UP(68 * (avg_pkt_size + 24), 1211711987bbSBrett Creeley avg_pkt_size + 640); 1212711987bbSBrett Creeley break; 121364a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_20GB: 1214711987bbSBrett Creeley itr += DIV_ROUND_UP(85 * (avg_pkt_size + 24), 1215711987bbSBrett Creeley avg_pkt_size + 640); 1216711987bbSBrett Creeley break; 1217711987bbSBrett Creeley case ICE_AQ_LINK_SPEED_10GB: 121864a59d05SAnirudh Venkataramanan default: 1219711987bbSBrett Creeley itr += DIV_ROUND_UP(170 * (avg_pkt_size + 24), 1220711987bbSBrett Creeley avg_pkt_size + 640); 1221711987bbSBrett Creeley break; 122264a59d05SAnirudh Venkataramanan } 1223711987bbSBrett Creeley 1224711987bbSBrett Creeley if ((itr & ICE_ITR_MASK) > ICE_ITR_ADAPTIVE_MAX_USECS) { 1225711987bbSBrett Creeley itr &= ICE_ITR_ADAPTIVE_LATENCY; 1226711987bbSBrett Creeley itr += ICE_ITR_ADAPTIVE_MAX_USECS; 1227711987bbSBrett Creeley } 1228711987bbSBrett Creeley 1229711987bbSBrett Creeley return itr; 123064a59d05SAnirudh Venkataramanan } 123164a59d05SAnirudh Venkataramanan 123264a59d05SAnirudh Venkataramanan /** 123364a59d05SAnirudh Venkataramanan * ice_update_itr - update the adaptive ITR value based on statistics 123464a59d05SAnirudh Venkataramanan * @q_vector: structure containing interrupt and ring information 123564a59d05SAnirudh Venkataramanan * @rc: structure containing ring performance data 123664a59d05SAnirudh Venkataramanan * 123764a59d05SAnirudh Venkataramanan * Stores a new ITR value based on packets and byte 123864a59d05SAnirudh Venkataramanan * counts during the last interrupt. The advantage of per interrupt 123964a59d05SAnirudh Venkataramanan * computation is faster updates and more accurate ITR for the current 124064a59d05SAnirudh Venkataramanan * traffic pattern. Constants in this function were computed 124164a59d05SAnirudh Venkataramanan * based on theoretical maximum wire speed and thresholds were set based 124264a59d05SAnirudh Venkataramanan * on testing data as well as attempting to minimize response time 124364a59d05SAnirudh Venkataramanan * while increasing bulk throughput. 124464a59d05SAnirudh Venkataramanan */ 124564a59d05SAnirudh Venkataramanan static void 124664a59d05SAnirudh Venkataramanan ice_update_itr(struct ice_q_vector *q_vector, struct ice_ring_container *rc) 124764a59d05SAnirudh Venkataramanan { 124864a59d05SAnirudh Venkataramanan unsigned long next_update = jiffies; 1249711987bbSBrett Creeley unsigned int packets, bytes, itr; 125064a59d05SAnirudh Venkataramanan bool container_is_rx; 125164a59d05SAnirudh Venkataramanan 125264a59d05SAnirudh Venkataramanan if (!rc->ring || !ITR_IS_DYNAMIC(rc->itr_setting)) 125364a59d05SAnirudh Venkataramanan return; 125464a59d05SAnirudh Venkataramanan 125564a59d05SAnirudh Venkataramanan /* If itr_countdown is set it means we programmed an ITR within 125664a59d05SAnirudh Venkataramanan * the last 4 interrupt cycles. This has a side effect of us 125764a59d05SAnirudh Venkataramanan * potentially firing an early interrupt. In order to work around 125864a59d05SAnirudh Venkataramanan * this we need to throw out any data received for a few 125964a59d05SAnirudh Venkataramanan * interrupts following the update. 126064a59d05SAnirudh Venkataramanan */ 126164a59d05SAnirudh Venkataramanan if (q_vector->itr_countdown) { 126264a59d05SAnirudh Venkataramanan itr = rc->target_itr; 126364a59d05SAnirudh Venkataramanan goto clear_counts; 126464a59d05SAnirudh Venkataramanan } 126564a59d05SAnirudh Venkataramanan 126664a59d05SAnirudh Venkataramanan container_is_rx = (&q_vector->rx == rc); 126764a59d05SAnirudh Venkataramanan /* For Rx we want to push the delay up and default to low latency. 126864a59d05SAnirudh Venkataramanan * for Tx we want to pull the delay down and default to high latency. 126964a59d05SAnirudh Venkataramanan */ 127064a59d05SAnirudh Venkataramanan itr = container_is_rx ? 127164a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MIN_USECS | ICE_ITR_ADAPTIVE_LATENCY : 127264a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MAX_USECS | ICE_ITR_ADAPTIVE_LATENCY; 127364a59d05SAnirudh Venkataramanan 127464a59d05SAnirudh Venkataramanan /* If we didn't update within up to 1 - 2 jiffies we can assume 127564a59d05SAnirudh Venkataramanan * that either packets are coming in so slow there hasn't been 127664a59d05SAnirudh Venkataramanan * any work, or that there is so much work that NAPI is dealing 127764a59d05SAnirudh Venkataramanan * with interrupt moderation and we don't need to do anything. 127864a59d05SAnirudh Venkataramanan */ 127964a59d05SAnirudh Venkataramanan if (time_after(next_update, rc->next_update)) 128064a59d05SAnirudh Venkataramanan goto clear_counts; 128164a59d05SAnirudh Venkataramanan 1282d27525ecSJesse Brandeburg prefetch(q_vector->vsi->port_info); 1283d27525ecSJesse Brandeburg 128464a59d05SAnirudh Venkataramanan packets = rc->total_pkts; 128564a59d05SAnirudh Venkataramanan bytes = rc->total_bytes; 128664a59d05SAnirudh Venkataramanan 128764a59d05SAnirudh Venkataramanan if (container_is_rx) { 128864a59d05SAnirudh Venkataramanan /* If Rx there are 1 to 4 packets and bytes are less than 128964a59d05SAnirudh Venkataramanan * 9000 assume insufficient data to use bulk rate limiting 129064a59d05SAnirudh Venkataramanan * approach unless Tx is already in bulk rate limiting. We 129164a59d05SAnirudh Venkataramanan * are likely latency driven. 129264a59d05SAnirudh Venkataramanan */ 129364a59d05SAnirudh Venkataramanan if (packets && packets < 4 && bytes < 9000 && 129464a59d05SAnirudh Venkataramanan (q_vector->tx.target_itr & ICE_ITR_ADAPTIVE_LATENCY)) { 129564a59d05SAnirudh Venkataramanan itr = ICE_ITR_ADAPTIVE_LATENCY; 1296711987bbSBrett Creeley goto adjust_by_size_and_speed; 129764a59d05SAnirudh Venkataramanan } 129864a59d05SAnirudh Venkataramanan } else if (packets < 4) { 129964a59d05SAnirudh Venkataramanan /* If we have Tx and Rx ITR maxed and Tx ITR is running in 130064a59d05SAnirudh Venkataramanan * bulk mode and we are receiving 4 or fewer packets just 130164a59d05SAnirudh Venkataramanan * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so 130264a59d05SAnirudh Venkataramanan * that the Rx can relax. 130364a59d05SAnirudh Venkataramanan */ 130464a59d05SAnirudh Venkataramanan if (rc->target_itr == ICE_ITR_ADAPTIVE_MAX_USECS && 130564a59d05SAnirudh Venkataramanan (q_vector->rx.target_itr & ICE_ITR_MASK) == 130664a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MAX_USECS) 130764a59d05SAnirudh Venkataramanan goto clear_counts; 130864a59d05SAnirudh Venkataramanan } else if (packets > 32) { 130964a59d05SAnirudh Venkataramanan /* If we have processed over 32 packets in a single interrupt 131064a59d05SAnirudh Venkataramanan * for Tx assume we need to switch over to "bulk" mode. 131164a59d05SAnirudh Venkataramanan */ 131264a59d05SAnirudh Venkataramanan rc->target_itr &= ~ICE_ITR_ADAPTIVE_LATENCY; 131364a59d05SAnirudh Venkataramanan } 131464a59d05SAnirudh Venkataramanan 131564a59d05SAnirudh Venkataramanan /* We have no packets to actually measure against. This means 131664a59d05SAnirudh Venkataramanan * either one of the other queues on this vector is active or 131764a59d05SAnirudh Venkataramanan * we are a Tx queue doing TSO with too high of an interrupt rate. 131864a59d05SAnirudh Venkataramanan * 131964a59d05SAnirudh Venkataramanan * Between 4 and 56 we can assume that our current interrupt delay 132064a59d05SAnirudh Venkataramanan * is only slightly too low. As such we should increase it by a small 132164a59d05SAnirudh Venkataramanan * fixed amount. 132264a59d05SAnirudh Venkataramanan */ 132364a59d05SAnirudh Venkataramanan if (packets < 56) { 132464a59d05SAnirudh Venkataramanan itr = rc->target_itr + ICE_ITR_ADAPTIVE_MIN_INC; 132564a59d05SAnirudh Venkataramanan if ((itr & ICE_ITR_MASK) > ICE_ITR_ADAPTIVE_MAX_USECS) { 132664a59d05SAnirudh Venkataramanan itr &= ICE_ITR_ADAPTIVE_LATENCY; 132764a59d05SAnirudh Venkataramanan itr += ICE_ITR_ADAPTIVE_MAX_USECS; 132864a59d05SAnirudh Venkataramanan } 132964a59d05SAnirudh Venkataramanan goto clear_counts; 133064a59d05SAnirudh Venkataramanan } 133164a59d05SAnirudh Venkataramanan 133264a59d05SAnirudh Venkataramanan if (packets <= 256) { 133364a59d05SAnirudh Venkataramanan itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr); 133464a59d05SAnirudh Venkataramanan itr &= ICE_ITR_MASK; 133564a59d05SAnirudh Venkataramanan 133664a59d05SAnirudh Venkataramanan /* Between 56 and 112 is our "goldilocks" zone where we are 133764a59d05SAnirudh Venkataramanan * working out "just right". Just report that our current 133864a59d05SAnirudh Venkataramanan * ITR is good for us. 133964a59d05SAnirudh Venkataramanan */ 134064a59d05SAnirudh Venkataramanan if (packets <= 112) 134164a59d05SAnirudh Venkataramanan goto clear_counts; 134264a59d05SAnirudh Venkataramanan 134364a59d05SAnirudh Venkataramanan /* If packet count is 128 or greater we are likely looking 134464a59d05SAnirudh Venkataramanan * at a slight overrun of the delay we want. Try halving 134564a59d05SAnirudh Venkataramanan * our delay to see if that will cut the number of packets 134664a59d05SAnirudh Venkataramanan * in half per interrupt. 134764a59d05SAnirudh Venkataramanan */ 134864a59d05SAnirudh Venkataramanan itr >>= 1; 134964a59d05SAnirudh Venkataramanan itr &= ICE_ITR_MASK; 135064a59d05SAnirudh Venkataramanan if (itr < ICE_ITR_ADAPTIVE_MIN_USECS) 135164a59d05SAnirudh Venkataramanan itr = ICE_ITR_ADAPTIVE_MIN_USECS; 135264a59d05SAnirudh Venkataramanan 135364a59d05SAnirudh Venkataramanan goto clear_counts; 135464a59d05SAnirudh Venkataramanan } 135564a59d05SAnirudh Venkataramanan 135664a59d05SAnirudh Venkataramanan /* The paths below assume we are dealing with a bulk ITR since 135764a59d05SAnirudh Venkataramanan * number of packets is greater than 256. We are just going to have 135864a59d05SAnirudh Venkataramanan * to compute a value and try to bring the count under control, 135964a59d05SAnirudh Venkataramanan * though for smaller packet sizes there isn't much we can do as 136064a59d05SAnirudh Venkataramanan * NAPI polling will likely be kicking in sooner rather than later. 136164a59d05SAnirudh Venkataramanan */ 136264a59d05SAnirudh Venkataramanan itr = ICE_ITR_ADAPTIVE_BULK; 136364a59d05SAnirudh Venkataramanan 1364711987bbSBrett Creeley adjust_by_size_and_speed: 136564a59d05SAnirudh Venkataramanan 1366711987bbSBrett Creeley /* based on checks above packets cannot be 0 so division is safe */ 1367711987bbSBrett Creeley itr = ice_adjust_itr_by_size_and_speed(q_vector->vsi->port_info, 1368711987bbSBrett Creeley bytes / packets, itr); 136964a59d05SAnirudh Venkataramanan 137064a59d05SAnirudh Venkataramanan clear_counts: 137164a59d05SAnirudh Venkataramanan /* write back value */ 137264a59d05SAnirudh Venkataramanan rc->target_itr = itr; 137364a59d05SAnirudh Venkataramanan 137464a59d05SAnirudh Venkataramanan /* next update should occur within next jiffy */ 137564a59d05SAnirudh Venkataramanan rc->next_update = next_update + 1; 137664a59d05SAnirudh Venkataramanan 137764a59d05SAnirudh Venkataramanan rc->total_bytes = 0; 137864a59d05SAnirudh Venkataramanan rc->total_pkts = 0; 137964a59d05SAnirudh Venkataramanan } 138064a59d05SAnirudh Venkataramanan 13812b245cb2SAnirudh Venkataramanan /** 138263f545edSBrett Creeley * ice_buildreg_itr - build value for writing to the GLINT_DYN_CTL register 138363f545edSBrett Creeley * @itr_idx: interrupt throttling index 138464a59d05SAnirudh Venkataramanan * @itr: interrupt throttling value in usecs 138563f545edSBrett Creeley */ 13868244dd2dSBrett Creeley static u32 ice_buildreg_itr(u16 itr_idx, u16 itr) 138763f545edSBrett Creeley { 13882f2da36eSAnirudh Venkataramanan /* The ITR value is reported in microseconds, and the register value is 138964a59d05SAnirudh Venkataramanan * recorded in 2 microsecond units. For this reason we only need to 139064a59d05SAnirudh Venkataramanan * shift by the GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S to apply this 139164a59d05SAnirudh Venkataramanan * granularity as a shift instead of division. The mask makes sure the 139264a59d05SAnirudh Venkataramanan * ITR value is never odd so we don't accidentally write into the field 139364a59d05SAnirudh Venkataramanan * prior to the ITR field. 139464a59d05SAnirudh Venkataramanan */ 139564a59d05SAnirudh Venkataramanan itr &= ICE_ITR_MASK; 139664a59d05SAnirudh Venkataramanan 139763f545edSBrett Creeley return GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M | 139863f545edSBrett Creeley (itr_idx << GLINT_DYN_CTL_ITR_INDX_S) | 139964a59d05SAnirudh Venkataramanan (itr << (GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S)); 140063f545edSBrett Creeley } 140163f545edSBrett Creeley 140264a59d05SAnirudh Venkataramanan /* The act of updating the ITR will cause it to immediately trigger. In order 140364a59d05SAnirudh Venkataramanan * to prevent this from throwing off adaptive update statistics we defer the 140464a59d05SAnirudh Venkataramanan * update so that it can only happen so often. So after either Tx or Rx are 140564a59d05SAnirudh Venkataramanan * updated we make the adaptive scheme wait until either the ITR completely 140664a59d05SAnirudh Venkataramanan * expires via the next_update expiration or we have been through at least 140764a59d05SAnirudh Venkataramanan * 3 interrupts. 140864a59d05SAnirudh Venkataramanan */ 140964a59d05SAnirudh Venkataramanan #define ITR_COUNTDOWN_START 3 141064a59d05SAnirudh Venkataramanan 141163f545edSBrett Creeley /** 141263f545edSBrett Creeley * ice_update_ena_itr - Update ITR and re-enable MSIX interrupt 141363f545edSBrett Creeley * @q_vector: q_vector for which ITR is being updated and interrupt enabled 141463f545edSBrett Creeley */ 14152fb0821fSJesse Brandeburg static void ice_update_ena_itr(struct ice_q_vector *q_vector) 141663f545edSBrett Creeley { 141764a59d05SAnirudh Venkataramanan struct ice_ring_container *tx = &q_vector->tx; 141864a59d05SAnirudh Venkataramanan struct ice_ring_container *rx = &q_vector->rx; 14192fb0821fSJesse Brandeburg struct ice_vsi *vsi = q_vector->vsi; 142063f545edSBrett Creeley u32 itr_val; 142163f545edSBrett Creeley 14222ab28bb0SBrett Creeley /* when exiting WB_ON_ITR lets set a low ITR value and trigger 14232ab28bb0SBrett Creeley * interrupts to expire right away in case we have more work ready to go 14242ab28bb0SBrett Creeley * already 14252ab28bb0SBrett Creeley */ 14262ab28bb0SBrett Creeley if (q_vector->itr_countdown == ICE_IN_WB_ON_ITR_MODE) { 14272ab28bb0SBrett Creeley itr_val = ice_buildreg_itr(rx->itr_idx, ICE_WB_ON_ITR_USECS); 14282ab28bb0SBrett Creeley wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx), itr_val); 14292ab28bb0SBrett Creeley /* set target back to last user set value */ 14302ab28bb0SBrett Creeley rx->target_itr = rx->itr_setting; 14312ab28bb0SBrett Creeley /* set current to what we just wrote and dynamic if needed */ 14322ab28bb0SBrett Creeley rx->current_itr = ICE_WB_ON_ITR_USECS | 14332ab28bb0SBrett Creeley (rx->itr_setting & ICE_ITR_DYNAMIC); 14342ab28bb0SBrett Creeley /* allow normal interrupt flow to start */ 14352ab28bb0SBrett Creeley q_vector->itr_countdown = 0; 14362ab28bb0SBrett Creeley return; 14372ab28bb0SBrett Creeley } 14382ab28bb0SBrett Creeley 143964a59d05SAnirudh Venkataramanan /* This will do nothing if dynamic updates are not enabled */ 144064a59d05SAnirudh Venkataramanan ice_update_itr(q_vector, tx); 144164a59d05SAnirudh Venkataramanan ice_update_itr(q_vector, rx); 144264a59d05SAnirudh Venkataramanan 144363f545edSBrett Creeley /* This block of logic allows us to get away with only updating 144463f545edSBrett Creeley * one ITR value with each interrupt. The idea is to perform a 144563f545edSBrett Creeley * pseudo-lazy update with the following criteria. 144663f545edSBrett Creeley * 144763f545edSBrett Creeley * 1. Rx is given higher priority than Tx if both are in same state 144863f545edSBrett Creeley * 2. If we must reduce an ITR that is given highest priority. 144963f545edSBrett Creeley * 3. We then give priority to increasing ITR based on amount. 145063f545edSBrett Creeley */ 145164a59d05SAnirudh Venkataramanan if (rx->target_itr < rx->current_itr) { 145263f545edSBrett Creeley /* Rx ITR needs to be reduced, this is highest priority */ 145364a59d05SAnirudh Venkataramanan itr_val = ice_buildreg_itr(rx->itr_idx, rx->target_itr); 145464a59d05SAnirudh Venkataramanan rx->current_itr = rx->target_itr; 145564a59d05SAnirudh Venkataramanan q_vector->itr_countdown = ITR_COUNTDOWN_START; 145664a59d05SAnirudh Venkataramanan } else if ((tx->target_itr < tx->current_itr) || 145764a59d05SAnirudh Venkataramanan ((rx->target_itr - rx->current_itr) < 145864a59d05SAnirudh Venkataramanan (tx->target_itr - tx->current_itr))) { 145963f545edSBrett Creeley /* Tx ITR needs to be reduced, this is second priority 146063f545edSBrett Creeley * Tx ITR needs to be increased more than Rx, fourth priority 146163f545edSBrett Creeley */ 146264a59d05SAnirudh Venkataramanan itr_val = ice_buildreg_itr(tx->itr_idx, tx->target_itr); 146364a59d05SAnirudh Venkataramanan tx->current_itr = tx->target_itr; 146464a59d05SAnirudh Venkataramanan q_vector->itr_countdown = ITR_COUNTDOWN_START; 146564a59d05SAnirudh Venkataramanan } else if (rx->current_itr != rx->target_itr) { 146663f545edSBrett Creeley /* Rx ITR needs to be increased, third priority */ 146764a59d05SAnirudh Venkataramanan itr_val = ice_buildreg_itr(rx->itr_idx, rx->target_itr); 146864a59d05SAnirudh Venkataramanan rx->current_itr = rx->target_itr; 146964a59d05SAnirudh Venkataramanan q_vector->itr_countdown = ITR_COUNTDOWN_START; 147063f545edSBrett Creeley } else { 147163f545edSBrett Creeley /* Still have to re-enable the interrupts */ 147263f545edSBrett Creeley itr_val = ice_buildreg_itr(ICE_ITR_NONE, 0); 147364a59d05SAnirudh Venkataramanan if (q_vector->itr_countdown) 147464a59d05SAnirudh Venkataramanan q_vector->itr_countdown--; 147563f545edSBrett Creeley } 147663f545edSBrett Creeley 14772fb0821fSJesse Brandeburg if (!test_bit(__ICE_DOWN, q_vector->vsi->state)) 14782fb0821fSJesse Brandeburg wr32(&q_vector->vsi->back->hw, 1479b07833a0SBrett Creeley GLINT_DYN_CTL(q_vector->reg_idx), 148064a59d05SAnirudh Venkataramanan itr_val); 148163f545edSBrett Creeley } 148263f545edSBrett Creeley 148363f545edSBrett Creeley /** 14842ab28bb0SBrett Creeley * ice_set_wb_on_itr - set WB_ON_ITR for this q_vector 14852ab28bb0SBrett Creeley * @q_vector: q_vector to set WB_ON_ITR on 14862ab28bb0SBrett Creeley * 14872ab28bb0SBrett Creeley * We need to tell hardware to write-back completed descriptors even when 14882ab28bb0SBrett Creeley * interrupts are disabled. Descriptors will be written back on cache line 14892ab28bb0SBrett Creeley * boundaries without WB_ON_ITR enabled, but if we don't enable WB_ON_ITR 14902ab28bb0SBrett Creeley * descriptors may not be written back if they don't fill a cache line until the 14912ab28bb0SBrett Creeley * next interrupt. 14922ab28bb0SBrett Creeley * 14932ab28bb0SBrett Creeley * This sets the write-back frequency to 2 microseconds as that is the minimum 14942ab28bb0SBrett Creeley * value that's not 0 due to ITR granularity. Also, set the INTENA_MSK bit to 14952ab28bb0SBrett Creeley * make sure hardware knows we aren't meddling with the INTENA_M bit. 14962ab28bb0SBrett Creeley */ 14972fb0821fSJesse Brandeburg static void ice_set_wb_on_itr(struct ice_q_vector *q_vector) 14982ab28bb0SBrett Creeley { 14992fb0821fSJesse Brandeburg struct ice_vsi *vsi = q_vector->vsi; 15002fb0821fSJesse Brandeburg 15012ab28bb0SBrett Creeley /* already in WB_ON_ITR mode no need to change it */ 15022ab28bb0SBrett Creeley if (q_vector->itr_countdown == ICE_IN_WB_ON_ITR_MODE) 15032ab28bb0SBrett Creeley return; 15042ab28bb0SBrett Creeley 15052ab28bb0SBrett Creeley if (q_vector->num_ring_rx) 15062ab28bb0SBrett Creeley wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx), 15072ab28bb0SBrett Creeley ICE_GLINT_DYN_CTL_WB_ON_ITR(ICE_WB_ON_ITR_USECS, 15082ab28bb0SBrett Creeley ICE_RX_ITR)); 15092ab28bb0SBrett Creeley 15102ab28bb0SBrett Creeley if (q_vector->num_ring_tx) 15112ab28bb0SBrett Creeley wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx), 15122ab28bb0SBrett Creeley ICE_GLINT_DYN_CTL_WB_ON_ITR(ICE_WB_ON_ITR_USECS, 15132ab28bb0SBrett Creeley ICE_TX_ITR)); 15142ab28bb0SBrett Creeley 15152ab28bb0SBrett Creeley q_vector->itr_countdown = ICE_IN_WB_ON_ITR_MODE; 15162ab28bb0SBrett Creeley } 15172ab28bb0SBrett Creeley 15182ab28bb0SBrett Creeley /** 15192b245cb2SAnirudh Venkataramanan * ice_napi_poll - NAPI polling Rx/Tx cleanup routine 15202b245cb2SAnirudh Venkataramanan * @napi: napi struct with our devices info in it 15212b245cb2SAnirudh Venkataramanan * @budget: amount of work driver is allowed to do this pass, in packets 15222b245cb2SAnirudh Venkataramanan * 15232b245cb2SAnirudh Venkataramanan * This function will clean all queues associated with a q_vector. 15242b245cb2SAnirudh Venkataramanan * 15252b245cb2SAnirudh Venkataramanan * Returns the amount of work done 15262b245cb2SAnirudh Venkataramanan */ 15272b245cb2SAnirudh Venkataramanan int ice_napi_poll(struct napi_struct *napi, int budget) 15282b245cb2SAnirudh Venkataramanan { 15292b245cb2SAnirudh Venkataramanan struct ice_q_vector *q_vector = 15302b245cb2SAnirudh Venkataramanan container_of(napi, struct ice_q_vector, napi); 15312b245cb2SAnirudh Venkataramanan bool clean_complete = true; 15322b245cb2SAnirudh Venkataramanan struct ice_ring *ring; 15339118fcd5SBrett Creeley int budget_per_ring; 15342b245cb2SAnirudh Venkataramanan int work_done = 0; 15352b245cb2SAnirudh Venkataramanan 15362b245cb2SAnirudh Venkataramanan /* Since the actual Tx work is minimal, we can give the Tx a larger 15372b245cb2SAnirudh Venkataramanan * budget and be more aggressive about cleaning up the Tx descriptors. 15382b245cb2SAnirudh Venkataramanan */ 15392d4238f5SKrzysztof Kazimierczak ice_for_each_ring(ring, q_vector->tx) { 15402d4238f5SKrzysztof Kazimierczak bool wd = ring->xsk_umem ? 15412d4238f5SKrzysztof Kazimierczak ice_clean_tx_irq_zc(ring, budget) : 15422d4238f5SKrzysztof Kazimierczak ice_clean_tx_irq(ring, budget); 15432d4238f5SKrzysztof Kazimierczak 15442d4238f5SKrzysztof Kazimierczak if (!wd) 15452b245cb2SAnirudh Venkataramanan clean_complete = false; 15462d4238f5SKrzysztof Kazimierczak } 15472b245cb2SAnirudh Venkataramanan 15482b245cb2SAnirudh Venkataramanan /* Handle case where we are called by netpoll with a budget of 0 */ 1549d27525ecSJesse Brandeburg if (unlikely(budget <= 0)) 15502b245cb2SAnirudh Venkataramanan return budget; 15512b245cb2SAnirudh Venkataramanan 15529118fcd5SBrett Creeley /* normally we have 1 Rx ring per q_vector */ 15539118fcd5SBrett Creeley if (unlikely(q_vector->num_ring_rx > 1)) 15549118fcd5SBrett Creeley /* We attempt to distribute budget to each Rx queue fairly, but 15559118fcd5SBrett Creeley * don't allow the budget to go below 1 because that would exit 15569118fcd5SBrett Creeley * polling early. 15572b245cb2SAnirudh Venkataramanan */ 155888865fc4SKarol Kolacinski budget_per_ring = max_t(int, budget / q_vector->num_ring_rx, 1); 15599118fcd5SBrett Creeley else 15609118fcd5SBrett Creeley /* Max of 1 Rx ring in this q_vector so give it the budget */ 15619118fcd5SBrett Creeley budget_per_ring = budget; 15622b245cb2SAnirudh Venkataramanan 15632b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->rx) { 15642b245cb2SAnirudh Venkataramanan int cleaned; 15652b245cb2SAnirudh Venkataramanan 15662d4238f5SKrzysztof Kazimierczak /* A dedicated path for zero-copy allows making a single 15672d4238f5SKrzysztof Kazimierczak * comparison in the irq context instead of many inside the 15682d4238f5SKrzysztof Kazimierczak * ice_clean_rx_irq function and makes the codebase cleaner. 15692d4238f5SKrzysztof Kazimierczak */ 15702d4238f5SKrzysztof Kazimierczak cleaned = ring->xsk_umem ? 15712d4238f5SKrzysztof Kazimierczak ice_clean_rx_irq_zc(ring, budget_per_ring) : 15722d4238f5SKrzysztof Kazimierczak ice_clean_rx_irq(ring, budget_per_ring); 15732b245cb2SAnirudh Venkataramanan work_done += cleaned; 15742b245cb2SAnirudh Venkataramanan /* if we clean as many as budgeted, we must not be done */ 15752b245cb2SAnirudh Venkataramanan if (cleaned >= budget_per_ring) 15762b245cb2SAnirudh Venkataramanan clean_complete = false; 15772b245cb2SAnirudh Venkataramanan } 15782b245cb2SAnirudh Venkataramanan 15792b245cb2SAnirudh Venkataramanan /* If work not completed, return budget and polling will return */ 15802b245cb2SAnirudh Venkataramanan if (!clean_complete) 15812b245cb2SAnirudh Venkataramanan return budget; 15822b245cb2SAnirudh Venkataramanan 15830bcd952fSJesse Brandeburg /* Exit the polling mode, but don't re-enable interrupts if stack might 15840bcd952fSJesse Brandeburg * poll us due to busy-polling 15850bcd952fSJesse Brandeburg */ 15860bcd952fSJesse Brandeburg if (likely(napi_complete_done(napi, work_done))) 15872fb0821fSJesse Brandeburg ice_update_ena_itr(q_vector); 15882ab28bb0SBrett Creeley else 15892fb0821fSJesse Brandeburg ice_set_wb_on_itr(q_vector); 1590e0c9fd9bSDave Ertman 159132a64994SBruce Allan return min_t(int, work_done, budget - 1); 15922b245cb2SAnirudh Venkataramanan } 15932b245cb2SAnirudh Venkataramanan 15942b245cb2SAnirudh Venkataramanan /** 1595d337f2afSAnirudh Venkataramanan * __ice_maybe_stop_tx - 2nd level check for Tx stop conditions 15962b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 15972b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 15982b245cb2SAnirudh Venkataramanan * 15992b245cb2SAnirudh Venkataramanan * Returns -EBUSY if a stop is needed, else 0 16002b245cb2SAnirudh Venkataramanan */ 16012b245cb2SAnirudh Venkataramanan static int __ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 16022b245cb2SAnirudh Venkataramanan { 16032b245cb2SAnirudh Venkataramanan netif_stop_subqueue(tx_ring->netdev, tx_ring->q_index); 16042b245cb2SAnirudh Venkataramanan /* Memory barrier before checking head and tail */ 16052b245cb2SAnirudh Venkataramanan smp_mb(); 16062b245cb2SAnirudh Venkataramanan 16072b245cb2SAnirudh Venkataramanan /* Check again in a case another CPU has just made room available. */ 16082b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) < size)) 16092b245cb2SAnirudh Venkataramanan return -EBUSY; 16102b245cb2SAnirudh Venkataramanan 16112b245cb2SAnirudh Venkataramanan /* A reprieve! - use start_subqueue because it doesn't call schedule */ 16122b245cb2SAnirudh Venkataramanan netif_start_subqueue(tx_ring->netdev, tx_ring->q_index); 16132b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 16142b245cb2SAnirudh Venkataramanan return 0; 16152b245cb2SAnirudh Venkataramanan } 16162b245cb2SAnirudh Venkataramanan 16172b245cb2SAnirudh Venkataramanan /** 1618d337f2afSAnirudh Venkataramanan * ice_maybe_stop_tx - 1st level check for Tx stop conditions 16192b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 16202b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 16212b245cb2SAnirudh Venkataramanan * 16222b245cb2SAnirudh Venkataramanan * Returns 0 if stop is not needed 16232b245cb2SAnirudh Venkataramanan */ 16242b245cb2SAnirudh Venkataramanan static int ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 16252b245cb2SAnirudh Venkataramanan { 16262b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) >= size)) 16272b245cb2SAnirudh Venkataramanan return 0; 1628d337f2afSAnirudh Venkataramanan 16292b245cb2SAnirudh Venkataramanan return __ice_maybe_stop_tx(tx_ring, size); 16302b245cb2SAnirudh Venkataramanan } 16312b245cb2SAnirudh Venkataramanan 16322b245cb2SAnirudh Venkataramanan /** 16332b245cb2SAnirudh Venkataramanan * ice_tx_map - Build the Tx descriptor 16342b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 16352b245cb2SAnirudh Venkataramanan * @first: first buffer info buffer to use 1636d76a60baSAnirudh Venkataramanan * @off: pointer to struct that holds offload parameters 16372b245cb2SAnirudh Venkataramanan * 16382b245cb2SAnirudh Venkataramanan * This function loops over the skb data pointed to by *first 16392b245cb2SAnirudh Venkataramanan * and gets a physical address for each memory location and programs 16402b245cb2SAnirudh Venkataramanan * it and the length into the transmit descriptor. 16412b245cb2SAnirudh Venkataramanan */ 1642d76a60baSAnirudh Venkataramanan static void 1643d76a60baSAnirudh Venkataramanan ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first, 1644d76a60baSAnirudh Venkataramanan struct ice_tx_offload_params *off) 16452b245cb2SAnirudh Venkataramanan { 1646d76a60baSAnirudh Venkataramanan u64 td_offset, td_tag, td_cmd; 16472b245cb2SAnirudh Venkataramanan u16 i = tx_ring->next_to_use; 16482b245cb2SAnirudh Venkataramanan unsigned int data_len, size; 16492b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 16502b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 16512b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 16524ee656bbSTony Nguyen skb_frag_t *frag; 16532b245cb2SAnirudh Venkataramanan dma_addr_t dma; 16542b245cb2SAnirudh Venkataramanan 1655d76a60baSAnirudh Venkataramanan td_tag = off->td_l2tag1; 1656d76a60baSAnirudh Venkataramanan td_cmd = off->td_cmd; 1657d76a60baSAnirudh Venkataramanan td_offset = off->td_offset; 16582b245cb2SAnirudh Venkataramanan skb = first->skb; 16592b245cb2SAnirudh Venkataramanan 16602b245cb2SAnirudh Venkataramanan data_len = skb->data_len; 16612b245cb2SAnirudh Venkataramanan size = skb_headlen(skb); 16622b245cb2SAnirudh Venkataramanan 16632b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 16642b245cb2SAnirudh Venkataramanan 1665d76a60baSAnirudh Venkataramanan if (first->tx_flags & ICE_TX_FLAGS_HW_VLAN) { 1666d76a60baSAnirudh Venkataramanan td_cmd |= (u64)ICE_TX_DESC_CMD_IL2TAG1; 1667d76a60baSAnirudh Venkataramanan td_tag = (first->tx_flags & ICE_TX_FLAGS_VLAN_M) >> 1668d76a60baSAnirudh Venkataramanan ICE_TX_FLAGS_VLAN_S; 1669d76a60baSAnirudh Venkataramanan } 1670d76a60baSAnirudh Venkataramanan 16712b245cb2SAnirudh Venkataramanan dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 16722b245cb2SAnirudh Venkataramanan 16732b245cb2SAnirudh Venkataramanan tx_buf = first; 16742b245cb2SAnirudh Venkataramanan 16752b245cb2SAnirudh Venkataramanan for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 16762b245cb2SAnirudh Venkataramanan unsigned int max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 16772b245cb2SAnirudh Venkataramanan 16782b245cb2SAnirudh Venkataramanan if (dma_mapping_error(tx_ring->dev, dma)) 16792b245cb2SAnirudh Venkataramanan goto dma_error; 16802b245cb2SAnirudh Venkataramanan 16812b245cb2SAnirudh Venkataramanan /* record length, and DMA address */ 16822b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, size); 16832b245cb2SAnirudh Venkataramanan dma_unmap_addr_set(tx_buf, dma, dma); 16842b245cb2SAnirudh Venkataramanan 16852b245cb2SAnirudh Venkataramanan /* align size to end of page */ 16862b245cb2SAnirudh Venkataramanan max_data += -dma & (ICE_MAX_READ_REQ_SIZE - 1); 16872b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 16882b245cb2SAnirudh Venkataramanan 16892b245cb2SAnirudh Venkataramanan /* account for data chunks larger than the hardware 16902b245cb2SAnirudh Venkataramanan * can handle 16912b245cb2SAnirudh Venkataramanan */ 16922b245cb2SAnirudh Venkataramanan while (unlikely(size > ICE_MAX_DATA_PER_TXD)) { 16932b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 16945757cc7cSTony Nguyen ice_build_ctob(td_cmd, td_offset, max_data, 16955757cc7cSTony Nguyen td_tag); 16962b245cb2SAnirudh Venkataramanan 16972b245cb2SAnirudh Venkataramanan tx_desc++; 16982b245cb2SAnirudh Venkataramanan i++; 16992b245cb2SAnirudh Venkataramanan 17002b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 17012b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 17022b245cb2SAnirudh Venkataramanan i = 0; 17032b245cb2SAnirudh Venkataramanan } 17042b245cb2SAnirudh Venkataramanan 17052b245cb2SAnirudh Venkataramanan dma += max_data; 17062b245cb2SAnirudh Venkataramanan size -= max_data; 17072b245cb2SAnirudh Venkataramanan 17082b245cb2SAnirudh Venkataramanan max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 17092b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 17102b245cb2SAnirudh Venkataramanan } 17112b245cb2SAnirudh Venkataramanan 17122b245cb2SAnirudh Venkataramanan if (likely(!data_len)) 17132b245cb2SAnirudh Venkataramanan break; 17142b245cb2SAnirudh Venkataramanan 17155757cc7cSTony Nguyen tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd, td_offset, 17162b245cb2SAnirudh Venkataramanan size, td_tag); 17172b245cb2SAnirudh Venkataramanan 17182b245cb2SAnirudh Venkataramanan tx_desc++; 17192b245cb2SAnirudh Venkataramanan i++; 17202b245cb2SAnirudh Venkataramanan 17212b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 17222b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 17232b245cb2SAnirudh Venkataramanan i = 0; 17242b245cb2SAnirudh Venkataramanan } 17252b245cb2SAnirudh Venkataramanan 17262b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag); 17272b245cb2SAnirudh Venkataramanan data_len -= size; 17282b245cb2SAnirudh Venkataramanan 17292b245cb2SAnirudh Venkataramanan dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 17302b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 17312b245cb2SAnirudh Venkataramanan 17322b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 17332b245cb2SAnirudh Venkataramanan } 17342b245cb2SAnirudh Venkataramanan 17352b245cb2SAnirudh Venkataramanan /* record bytecount for BQL */ 17362b245cb2SAnirudh Venkataramanan netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 17372b245cb2SAnirudh Venkataramanan 17382b245cb2SAnirudh Venkataramanan /* record SW timestamp if HW timestamp is not available */ 17392b245cb2SAnirudh Venkataramanan skb_tx_timestamp(first->skb); 17402b245cb2SAnirudh Venkataramanan 17412b245cb2SAnirudh Venkataramanan i++; 17422b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) 17432b245cb2SAnirudh Venkataramanan i = 0; 17442b245cb2SAnirudh Venkataramanan 17452b245cb2SAnirudh Venkataramanan /* write last descriptor with RS and EOP bits */ 1746efc2214bSMaciej Fijalkowski td_cmd |= (u64)ICE_TXD_LAST_DESC_CMD; 17475757cc7cSTony Nguyen tx_desc->cmd_type_offset_bsz = 17485757cc7cSTony Nguyen ice_build_ctob(td_cmd, td_offset, size, td_tag); 17492b245cb2SAnirudh Venkataramanan 17502b245cb2SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w know there 17512b245cb2SAnirudh Venkataramanan * are new descriptors to fetch. 17522b245cb2SAnirudh Venkataramanan * 17532b245cb2SAnirudh Venkataramanan * We also use this memory barrier to make certain all of the 17542b245cb2SAnirudh Venkataramanan * status bits have been updated before next_to_watch is written. 17552b245cb2SAnirudh Venkataramanan */ 17562b245cb2SAnirudh Venkataramanan wmb(); 17572b245cb2SAnirudh Venkataramanan 17582b245cb2SAnirudh Venkataramanan /* set next_to_watch value indicating a packet is present */ 17592b245cb2SAnirudh Venkataramanan first->next_to_watch = tx_desc; 17602b245cb2SAnirudh Venkataramanan 17612b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 17622b245cb2SAnirudh Venkataramanan 17632b245cb2SAnirudh Venkataramanan ice_maybe_stop_tx(tx_ring, DESC_NEEDED); 17642b245cb2SAnirudh Venkataramanan 17652b245cb2SAnirudh Venkataramanan /* notify HW of packet */ 17664ee656bbSTony Nguyen if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) 17672b245cb2SAnirudh Venkataramanan writel(i, tx_ring->tail); 17682b245cb2SAnirudh Venkataramanan 17692b245cb2SAnirudh Venkataramanan return; 17702b245cb2SAnirudh Venkataramanan 17712b245cb2SAnirudh Venkataramanan dma_error: 17722f2da36eSAnirudh Venkataramanan /* clear DMA mappings for failed tx_buf map */ 17732b245cb2SAnirudh Venkataramanan for (;;) { 17742b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 17752b245cb2SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, tx_buf); 17762b245cb2SAnirudh Venkataramanan if (tx_buf == first) 17772b245cb2SAnirudh Venkataramanan break; 17782b245cb2SAnirudh Venkataramanan if (i == 0) 17792b245cb2SAnirudh Venkataramanan i = tx_ring->count; 17802b245cb2SAnirudh Venkataramanan i--; 17812b245cb2SAnirudh Venkataramanan } 17822b245cb2SAnirudh Venkataramanan 17832b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 17842b245cb2SAnirudh Venkataramanan } 17852b245cb2SAnirudh Venkataramanan 17862b245cb2SAnirudh Venkataramanan /** 1787d76a60baSAnirudh Venkataramanan * ice_tx_csum - Enable Tx checksum offloads 1788d76a60baSAnirudh Venkataramanan * @first: pointer to the first descriptor 1789d76a60baSAnirudh Venkataramanan * @off: pointer to struct that holds offload parameters 1790d76a60baSAnirudh Venkataramanan * 1791d76a60baSAnirudh Venkataramanan * Returns 0 or error (negative) if checksum offload can't happen, 1 otherwise. 1792d76a60baSAnirudh Venkataramanan */ 1793d76a60baSAnirudh Venkataramanan static 1794d76a60baSAnirudh Venkataramanan int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) 1795d76a60baSAnirudh Venkataramanan { 1796d76a60baSAnirudh Venkataramanan u32 l4_len = 0, l3_len = 0, l2_len = 0; 1797d76a60baSAnirudh Venkataramanan struct sk_buff *skb = first->skb; 1798d76a60baSAnirudh Venkataramanan union { 1799d76a60baSAnirudh Venkataramanan struct iphdr *v4; 1800d76a60baSAnirudh Venkataramanan struct ipv6hdr *v6; 1801d76a60baSAnirudh Venkataramanan unsigned char *hdr; 1802d76a60baSAnirudh Venkataramanan } ip; 1803d76a60baSAnirudh Venkataramanan union { 1804d76a60baSAnirudh Venkataramanan struct tcphdr *tcp; 1805d76a60baSAnirudh Venkataramanan unsigned char *hdr; 1806d76a60baSAnirudh Venkataramanan } l4; 1807d76a60baSAnirudh Venkataramanan __be16 frag_off, protocol; 1808d76a60baSAnirudh Venkataramanan unsigned char *exthdr; 1809d76a60baSAnirudh Venkataramanan u32 offset, cmd = 0; 1810d76a60baSAnirudh Venkataramanan u8 l4_proto = 0; 1811d76a60baSAnirudh Venkataramanan 1812d76a60baSAnirudh Venkataramanan if (skb->ip_summed != CHECKSUM_PARTIAL) 1813d76a60baSAnirudh Venkataramanan return 0; 1814d76a60baSAnirudh Venkataramanan 1815d76a60baSAnirudh Venkataramanan ip.hdr = skb_network_header(skb); 1816d76a60baSAnirudh Venkataramanan l4.hdr = skb_transport_header(skb); 1817d76a60baSAnirudh Venkataramanan 1818d76a60baSAnirudh Venkataramanan /* compute outer L2 header size */ 1819d76a60baSAnirudh Venkataramanan l2_len = ip.hdr - skb->data; 1820d76a60baSAnirudh Venkataramanan offset = (l2_len / 2) << ICE_TX_DESC_LEN_MACLEN_S; 1821d76a60baSAnirudh Venkataramanan 1822a4e82a81STony Nguyen protocol = vlan_get_protocol(skb); 1823a4e82a81STony Nguyen 1824a4e82a81STony Nguyen if (protocol == htons(ETH_P_IP)) 1825a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_IPV4; 1826a4e82a81STony Nguyen else if (protocol == htons(ETH_P_IPV6)) 1827a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_IPV6; 1828a4e82a81STony Nguyen 1829a4e82a81STony Nguyen if (skb->encapsulation) { 1830a4e82a81STony Nguyen bool gso_ena = false; 1831a4e82a81STony Nguyen u32 tunnel = 0; 1832a4e82a81STony Nguyen 1833a4e82a81STony Nguyen /* define outer network header type */ 1834a4e82a81STony Nguyen if (first->tx_flags & ICE_TX_FLAGS_IPV4) { 1835a4e82a81STony Nguyen tunnel |= (first->tx_flags & ICE_TX_FLAGS_TSO) ? 1836a4e82a81STony Nguyen ICE_TX_CTX_EIPT_IPV4 : 1837a4e82a81STony Nguyen ICE_TX_CTX_EIPT_IPV4_NO_CSUM; 1838a4e82a81STony Nguyen l4_proto = ip.v4->protocol; 1839a4e82a81STony Nguyen } else if (first->tx_flags & ICE_TX_FLAGS_IPV6) { 1840a4e82a81STony Nguyen tunnel |= ICE_TX_CTX_EIPT_IPV6; 1841a4e82a81STony Nguyen exthdr = ip.hdr + sizeof(*ip.v6); 1842a4e82a81STony Nguyen l4_proto = ip.v6->nexthdr; 1843a4e82a81STony Nguyen if (l4.hdr != exthdr) 1844a4e82a81STony Nguyen ipv6_skip_exthdr(skb, exthdr - skb->data, 1845a4e82a81STony Nguyen &l4_proto, &frag_off); 1846a4e82a81STony Nguyen } 1847a4e82a81STony Nguyen 1848a4e82a81STony Nguyen /* define outer transport */ 1849a4e82a81STony Nguyen switch (l4_proto) { 1850a4e82a81STony Nguyen case IPPROTO_UDP: 1851a4e82a81STony Nguyen tunnel |= ICE_TXD_CTX_UDP_TUNNELING; 1852a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_TUNNEL; 1853a4e82a81STony Nguyen break; 1854a4e82a81STony Nguyen case IPPROTO_GRE: 1855a4e82a81STony Nguyen tunnel |= ICE_TXD_CTX_GRE_TUNNELING; 1856a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_TUNNEL; 1857a4e82a81STony Nguyen break; 1858a4e82a81STony Nguyen case IPPROTO_IPIP: 1859a4e82a81STony Nguyen case IPPROTO_IPV6: 1860a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_TUNNEL; 1861a4e82a81STony Nguyen l4.hdr = skb_inner_network_header(skb); 1862a4e82a81STony Nguyen break; 1863a4e82a81STony Nguyen default: 1864a4e82a81STony Nguyen if (first->tx_flags & ICE_TX_FLAGS_TSO) 1865d76a60baSAnirudh Venkataramanan return -1; 1866d76a60baSAnirudh Venkataramanan 1867a4e82a81STony Nguyen skb_checksum_help(skb); 1868a4e82a81STony Nguyen return 0; 1869a4e82a81STony Nguyen } 1870a4e82a81STony Nguyen 1871a4e82a81STony Nguyen /* compute outer L3 header size */ 1872a4e82a81STony Nguyen tunnel |= ((l4.hdr - ip.hdr) / 4) << 1873a4e82a81STony Nguyen ICE_TXD_CTX_QW0_EIPLEN_S; 1874a4e82a81STony Nguyen 1875a4e82a81STony Nguyen /* switch IP header pointer from outer to inner header */ 1876a4e82a81STony Nguyen ip.hdr = skb_inner_network_header(skb); 1877a4e82a81STony Nguyen 1878a4e82a81STony Nguyen /* compute tunnel header size */ 1879a4e82a81STony Nguyen tunnel |= ((ip.hdr - l4.hdr) / 2) << 1880a4e82a81STony Nguyen ICE_TXD_CTX_QW0_NATLEN_S; 1881a4e82a81STony Nguyen 1882a4e82a81STony Nguyen gso_ena = skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL; 1883a4e82a81STony Nguyen /* indicate if we need to offload outer UDP header */ 1884a4e82a81STony Nguyen if ((first->tx_flags & ICE_TX_FLAGS_TSO) && !gso_ena && 1885a4e82a81STony Nguyen (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) 1886a4e82a81STony Nguyen tunnel |= ICE_TXD_CTX_QW0_L4T_CS_M; 1887a4e82a81STony Nguyen 1888a4e82a81STony Nguyen /* record tunnel offload values */ 1889a4e82a81STony Nguyen off->cd_tunnel_params |= tunnel; 1890a4e82a81STony Nguyen 1891a4e82a81STony Nguyen /* set DTYP=1 to indicate that it's an Tx context descriptor 1892a4e82a81STony Nguyen * in IPsec tunnel mode with Tx offloads in Quad word 1 1893a4e82a81STony Nguyen */ 1894a4e82a81STony Nguyen off->cd_qw1 |= (u64)ICE_TX_DESC_DTYPE_CTX; 1895a4e82a81STony Nguyen 1896a4e82a81STony Nguyen /* switch L4 header pointer from outer to inner */ 1897a4e82a81STony Nguyen l4.hdr = skb_inner_transport_header(skb); 1898a4e82a81STony Nguyen l4_proto = 0; 1899a4e82a81STony Nguyen 1900a4e82a81STony Nguyen /* reset type as we transition from outer to inner headers */ 1901a4e82a81STony Nguyen first->tx_flags &= ~(ICE_TX_FLAGS_IPV4 | ICE_TX_FLAGS_IPV6); 1902a4e82a81STony Nguyen if (ip.v4->version == 4) 1903a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_IPV4; 1904a4e82a81STony Nguyen if (ip.v6->version == 6) 1905a4e82a81STony Nguyen first->tx_flags |= ICE_TX_FLAGS_IPV6; 1906a4e82a81STony Nguyen } 1907a4e82a81STony Nguyen 1908d76a60baSAnirudh Venkataramanan /* Enable IP checksum offloads */ 1909a4e82a81STony Nguyen if (first->tx_flags & ICE_TX_FLAGS_IPV4) { 1910d76a60baSAnirudh Venkataramanan l4_proto = ip.v4->protocol; 1911d76a60baSAnirudh Venkataramanan /* the stack computes the IP header already, the only time we 1912d76a60baSAnirudh Venkataramanan * need the hardware to recompute it is in the case of TSO. 1913d76a60baSAnirudh Venkataramanan */ 1914d76a60baSAnirudh Venkataramanan if (first->tx_flags & ICE_TX_FLAGS_TSO) 1915d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 1916d76a60baSAnirudh Venkataramanan else 1917d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 1918d76a60baSAnirudh Venkataramanan 1919a4e82a81STony Nguyen } else if (first->tx_flags & ICE_TX_FLAGS_IPV6) { 1920d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 1921d76a60baSAnirudh Venkataramanan exthdr = ip.hdr + sizeof(*ip.v6); 1922d76a60baSAnirudh Venkataramanan l4_proto = ip.v6->nexthdr; 1923d76a60baSAnirudh Venkataramanan if (l4.hdr != exthdr) 1924d76a60baSAnirudh Venkataramanan ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto, 1925d76a60baSAnirudh Venkataramanan &frag_off); 1926d76a60baSAnirudh Venkataramanan } else { 1927d76a60baSAnirudh Venkataramanan return -1; 1928d76a60baSAnirudh Venkataramanan } 1929d76a60baSAnirudh Venkataramanan 1930d76a60baSAnirudh Venkataramanan /* compute inner L3 header size */ 1931d76a60baSAnirudh Venkataramanan l3_len = l4.hdr - ip.hdr; 1932d76a60baSAnirudh Venkataramanan offset |= (l3_len / 4) << ICE_TX_DESC_LEN_IPLEN_S; 1933d76a60baSAnirudh Venkataramanan 1934d76a60baSAnirudh Venkataramanan /* Enable L4 checksum offloads */ 1935d76a60baSAnirudh Venkataramanan switch (l4_proto) { 1936d76a60baSAnirudh Venkataramanan case IPPROTO_TCP: 1937d76a60baSAnirudh Venkataramanan /* enable checksum offloads */ 1938d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 1939d76a60baSAnirudh Venkataramanan l4_len = l4.tcp->doff; 1940d76a60baSAnirudh Venkataramanan offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S; 1941d76a60baSAnirudh Venkataramanan break; 1942d76a60baSAnirudh Venkataramanan case IPPROTO_UDP: 1943d76a60baSAnirudh Venkataramanan /* enable UDP checksum offload */ 1944d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 1945d76a60baSAnirudh Venkataramanan l4_len = (sizeof(struct udphdr) >> 2); 1946d76a60baSAnirudh Venkataramanan offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S; 1947d76a60baSAnirudh Venkataramanan break; 1948d76a60baSAnirudh Venkataramanan case IPPROTO_SCTP: 1949cf909e19SAnirudh Venkataramanan /* enable SCTP checksum offload */ 1950cf909e19SAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 1951cf909e19SAnirudh Venkataramanan l4_len = sizeof(struct sctphdr) >> 2; 1952cf909e19SAnirudh Venkataramanan offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S; 1953cf909e19SAnirudh Venkataramanan break; 1954cf909e19SAnirudh Venkataramanan 1955d76a60baSAnirudh Venkataramanan default: 1956d76a60baSAnirudh Venkataramanan if (first->tx_flags & ICE_TX_FLAGS_TSO) 1957d76a60baSAnirudh Venkataramanan return -1; 1958d76a60baSAnirudh Venkataramanan skb_checksum_help(skb); 1959d76a60baSAnirudh Venkataramanan return 0; 1960d76a60baSAnirudh Venkataramanan } 1961d76a60baSAnirudh Venkataramanan 1962d76a60baSAnirudh Venkataramanan off->td_cmd |= cmd; 1963d76a60baSAnirudh Venkataramanan off->td_offset |= offset; 1964d76a60baSAnirudh Venkataramanan return 1; 1965d76a60baSAnirudh Venkataramanan } 1966d76a60baSAnirudh Venkataramanan 1967d76a60baSAnirudh Venkataramanan /** 1968f9867df6SAnirudh Venkataramanan * ice_tx_prepare_vlan_flags - prepare generic Tx VLAN tagging flags for HW 1969d76a60baSAnirudh Venkataramanan * @tx_ring: ring to send buffer on 1970d76a60baSAnirudh Venkataramanan * @first: pointer to struct ice_tx_buf 1971d76a60baSAnirudh Venkataramanan * 1972d76a60baSAnirudh Venkataramanan * Checks the skb and set up correspondingly several generic transmit flags 1973d76a60baSAnirudh Venkataramanan * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 1974d76a60baSAnirudh Venkataramanan * 1975d76a60baSAnirudh Venkataramanan * Returns error code indicate the frame should be dropped upon error and the 1976d76a60baSAnirudh Venkataramanan * otherwise returns 0 to indicate the flags has been set properly. 1977d76a60baSAnirudh Venkataramanan */ 1978d76a60baSAnirudh Venkataramanan static int 1979d76a60baSAnirudh Venkataramanan ice_tx_prepare_vlan_flags(struct ice_ring *tx_ring, struct ice_tx_buf *first) 1980d76a60baSAnirudh Venkataramanan { 1981d76a60baSAnirudh Venkataramanan struct sk_buff *skb = first->skb; 1982d76a60baSAnirudh Venkataramanan __be16 protocol = skb->protocol; 1983d76a60baSAnirudh Venkataramanan 1984d76a60baSAnirudh Venkataramanan if (protocol == htons(ETH_P_8021Q) && 1985d76a60baSAnirudh Venkataramanan !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { 1986d76a60baSAnirudh Venkataramanan /* when HW VLAN acceleration is turned off by the user the 1987d76a60baSAnirudh Venkataramanan * stack sets the protocol to 8021q so that the driver 1988d76a60baSAnirudh Venkataramanan * can take any steps required to support the SW only 1989d76a60baSAnirudh Venkataramanan * VLAN handling. In our case the driver doesn't need 1990d76a60baSAnirudh Venkataramanan * to take any further steps so just set the protocol 1991d76a60baSAnirudh Venkataramanan * to the encapsulated ethertype. 1992d76a60baSAnirudh Venkataramanan */ 1993d76a60baSAnirudh Venkataramanan skb->protocol = vlan_get_protocol(skb); 19945f6aa50eSAnirudh Venkataramanan return 0; 1995d76a60baSAnirudh Venkataramanan } 1996d76a60baSAnirudh Venkataramanan 1997d76a60baSAnirudh Venkataramanan /* if we have a HW VLAN tag being added, default to the HW one */ 1998d76a60baSAnirudh Venkataramanan if (skb_vlan_tag_present(skb)) { 1999d76a60baSAnirudh Venkataramanan first->tx_flags |= skb_vlan_tag_get(skb) << ICE_TX_FLAGS_VLAN_S; 2000d76a60baSAnirudh Venkataramanan first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; 2001d76a60baSAnirudh Venkataramanan } else if (protocol == htons(ETH_P_8021Q)) { 2002d76a60baSAnirudh Venkataramanan struct vlan_hdr *vhdr, _vhdr; 2003d76a60baSAnirudh Venkataramanan 2004d76a60baSAnirudh Venkataramanan /* for SW VLAN, check the next protocol and store the tag */ 2005d76a60baSAnirudh Venkataramanan vhdr = (struct vlan_hdr *)skb_header_pointer(skb, ETH_HLEN, 2006d76a60baSAnirudh Venkataramanan sizeof(_vhdr), 2007d76a60baSAnirudh Venkataramanan &_vhdr); 2008d76a60baSAnirudh Venkataramanan if (!vhdr) 2009d76a60baSAnirudh Venkataramanan return -EINVAL; 2010d76a60baSAnirudh Venkataramanan 2011d76a60baSAnirudh Venkataramanan first->tx_flags |= ntohs(vhdr->h_vlan_TCI) << 2012d76a60baSAnirudh Venkataramanan ICE_TX_FLAGS_VLAN_S; 2013d76a60baSAnirudh Venkataramanan first->tx_flags |= ICE_TX_FLAGS_SW_VLAN; 2014d76a60baSAnirudh Venkataramanan } 2015d76a60baSAnirudh Venkataramanan 20165f6aa50eSAnirudh Venkataramanan return ice_tx_prepare_vlan_flags_dcb(tx_ring, first); 2017d76a60baSAnirudh Venkataramanan } 2018d76a60baSAnirudh Venkataramanan 2019d76a60baSAnirudh Venkataramanan /** 2020d76a60baSAnirudh Venkataramanan * ice_tso - computes mss and TSO length to prepare for TSO 2021d76a60baSAnirudh Venkataramanan * @first: pointer to struct ice_tx_buf 2022d76a60baSAnirudh Venkataramanan * @off: pointer to struct that holds offload parameters 2023d76a60baSAnirudh Venkataramanan * 2024d76a60baSAnirudh Venkataramanan * Returns 0 or error (negative) if TSO can't happen, 1 otherwise. 2025d76a60baSAnirudh Venkataramanan */ 2026d76a60baSAnirudh Venkataramanan static 2027d76a60baSAnirudh Venkataramanan int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off) 2028d76a60baSAnirudh Venkataramanan { 2029d76a60baSAnirudh Venkataramanan struct sk_buff *skb = first->skb; 2030d76a60baSAnirudh Venkataramanan union { 2031d76a60baSAnirudh Venkataramanan struct iphdr *v4; 2032d76a60baSAnirudh Venkataramanan struct ipv6hdr *v6; 2033d76a60baSAnirudh Venkataramanan unsigned char *hdr; 2034d76a60baSAnirudh Venkataramanan } ip; 2035d76a60baSAnirudh Venkataramanan union { 2036d76a60baSAnirudh Venkataramanan struct tcphdr *tcp; 2037a54e3b8cSBrett Creeley struct udphdr *udp; 2038d76a60baSAnirudh Venkataramanan unsigned char *hdr; 2039d76a60baSAnirudh Venkataramanan } l4; 2040d76a60baSAnirudh Venkataramanan u64 cd_mss, cd_tso_len; 204188865fc4SKarol Kolacinski u32 paylen; 204288865fc4SKarol Kolacinski u8 l4_start; 2043d76a60baSAnirudh Venkataramanan int err; 2044d76a60baSAnirudh Venkataramanan 2045d76a60baSAnirudh Venkataramanan if (skb->ip_summed != CHECKSUM_PARTIAL) 2046d76a60baSAnirudh Venkataramanan return 0; 2047d76a60baSAnirudh Venkataramanan 2048d76a60baSAnirudh Venkataramanan if (!skb_is_gso(skb)) 2049d76a60baSAnirudh Venkataramanan return 0; 2050d76a60baSAnirudh Venkataramanan 2051d76a60baSAnirudh Venkataramanan err = skb_cow_head(skb, 0); 2052d76a60baSAnirudh Venkataramanan if (err < 0) 2053d76a60baSAnirudh Venkataramanan return err; 2054d76a60baSAnirudh Venkataramanan 2055c3a6825eSBruce Allan /* cppcheck-suppress unreadVariable */ 2056d76a60baSAnirudh Venkataramanan ip.hdr = skb_network_header(skb); 2057d76a60baSAnirudh Venkataramanan l4.hdr = skb_transport_header(skb); 2058d76a60baSAnirudh Venkataramanan 2059d76a60baSAnirudh Venkataramanan /* initialize outer IP header fields */ 2060d76a60baSAnirudh Venkataramanan if (ip.v4->version == 4) { 2061d76a60baSAnirudh Venkataramanan ip.v4->tot_len = 0; 2062d76a60baSAnirudh Venkataramanan ip.v4->check = 0; 2063d76a60baSAnirudh Venkataramanan } else { 2064d76a60baSAnirudh Venkataramanan ip.v6->payload_len = 0; 2065d76a60baSAnirudh Venkataramanan } 2066d76a60baSAnirudh Venkataramanan 2067a4e82a81STony Nguyen if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 2068a4e82a81STony Nguyen SKB_GSO_GRE_CSUM | 2069a4e82a81STony Nguyen SKB_GSO_IPXIP4 | 2070a4e82a81STony Nguyen SKB_GSO_IPXIP6 | 2071a4e82a81STony Nguyen SKB_GSO_UDP_TUNNEL | 2072a4e82a81STony Nguyen SKB_GSO_UDP_TUNNEL_CSUM)) { 2073a4e82a81STony Nguyen if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 2074a4e82a81STony Nguyen (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) { 2075a4e82a81STony Nguyen l4.udp->len = 0; 2076a4e82a81STony Nguyen 2077a4e82a81STony Nguyen /* determine offset of outer transport header */ 207888865fc4SKarol Kolacinski l4_start = (u8)(l4.hdr - skb->data); 2079a4e82a81STony Nguyen 2080a4e82a81STony Nguyen /* remove payload length from outer checksum */ 2081a4e82a81STony Nguyen paylen = skb->len - l4_start; 2082a4e82a81STony Nguyen csum_replace_by_diff(&l4.udp->check, 2083a4e82a81STony Nguyen (__force __wsum)htonl(paylen)); 2084a4e82a81STony Nguyen } 2085a4e82a81STony Nguyen 2086a4e82a81STony Nguyen /* reset pointers to inner headers */ 2087a4e82a81STony Nguyen 2088a4e82a81STony Nguyen /* cppcheck-suppress unreadVariable */ 2089a4e82a81STony Nguyen ip.hdr = skb_inner_network_header(skb); 2090a4e82a81STony Nguyen l4.hdr = skb_inner_transport_header(skb); 2091a4e82a81STony Nguyen 2092a4e82a81STony Nguyen /* initialize inner IP header fields */ 2093a4e82a81STony Nguyen if (ip.v4->version == 4) { 2094a4e82a81STony Nguyen ip.v4->tot_len = 0; 2095a4e82a81STony Nguyen ip.v4->check = 0; 2096a4e82a81STony Nguyen } else { 2097a4e82a81STony Nguyen ip.v6->payload_len = 0; 2098a4e82a81STony Nguyen } 2099a4e82a81STony Nguyen } 2100a4e82a81STony Nguyen 2101d76a60baSAnirudh Venkataramanan /* determine offset of transport header */ 210288865fc4SKarol Kolacinski l4_start = (u8)(l4.hdr - skb->data); 2103d76a60baSAnirudh Venkataramanan 2104d76a60baSAnirudh Venkataramanan /* remove payload length from checksum */ 2105d76a60baSAnirudh Venkataramanan paylen = skb->len - l4_start; 2106d76a60baSAnirudh Venkataramanan 2107a54e3b8cSBrett Creeley if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 2108a54e3b8cSBrett Creeley csum_replace_by_diff(&l4.udp->check, 2109a54e3b8cSBrett Creeley (__force __wsum)htonl(paylen)); 2110a54e3b8cSBrett Creeley /* compute length of UDP segmentation header */ 211188865fc4SKarol Kolacinski off->header_len = (u8)sizeof(l4.udp) + l4_start; 2112a54e3b8cSBrett Creeley } else { 2113a54e3b8cSBrett Creeley csum_replace_by_diff(&l4.tcp->check, 2114a54e3b8cSBrett Creeley (__force __wsum)htonl(paylen)); 2115a54e3b8cSBrett Creeley /* compute length of TCP segmentation header */ 211688865fc4SKarol Kolacinski off->header_len = (u8)((l4.tcp->doff * 4) + l4_start); 2117a54e3b8cSBrett Creeley } 2118d76a60baSAnirudh Venkataramanan 2119d76a60baSAnirudh Venkataramanan /* update gso_segs and bytecount */ 2120d76a60baSAnirudh Venkataramanan first->gso_segs = skb_shinfo(skb)->gso_segs; 2121d944b469SBrett Creeley first->bytecount += (first->gso_segs - 1) * off->header_len; 2122d76a60baSAnirudh Venkataramanan 2123d76a60baSAnirudh Venkataramanan cd_tso_len = skb->len - off->header_len; 2124d76a60baSAnirudh Venkataramanan cd_mss = skb_shinfo(skb)->gso_size; 2125d76a60baSAnirudh Venkataramanan 2126d76a60baSAnirudh Venkataramanan /* record cdesc_qw1 with TSO parameters */ 2127e65e9e15SBruce Allan off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX | 2128d76a60baSAnirudh Venkataramanan (ICE_TX_CTX_DESC_TSO << ICE_TXD_CTX_QW1_CMD_S) | 2129d76a60baSAnirudh Venkataramanan (cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) | 2130e65e9e15SBruce Allan (cd_mss << ICE_TXD_CTX_QW1_MSS_S)); 2131d76a60baSAnirudh Venkataramanan first->tx_flags |= ICE_TX_FLAGS_TSO; 2132d76a60baSAnirudh Venkataramanan return 1; 2133d76a60baSAnirudh Venkataramanan } 2134d76a60baSAnirudh Venkataramanan 2135d76a60baSAnirudh Venkataramanan /** 21362b245cb2SAnirudh Venkataramanan * ice_txd_use_count - estimate the number of descriptors needed for Tx 21372b245cb2SAnirudh Venkataramanan * @size: transmit request size in bytes 21382b245cb2SAnirudh Venkataramanan * 21392b245cb2SAnirudh Venkataramanan * Due to hardware alignment restrictions (4K alignment), we need to 21402b245cb2SAnirudh Venkataramanan * assume that we can have no more than 12K of data per descriptor, even 21412b245cb2SAnirudh Venkataramanan * though each descriptor can take up to 16K - 1 bytes of aligned memory. 21422b245cb2SAnirudh Venkataramanan * Thus, we need to divide by 12K. But division is slow! Instead, 21432b245cb2SAnirudh Venkataramanan * we decompose the operation into shifts and one relatively cheap 21442b245cb2SAnirudh Venkataramanan * multiply operation. 21452b245cb2SAnirudh Venkataramanan * 21462b245cb2SAnirudh Venkataramanan * To divide by 12K, we first divide by 4K, then divide by 3: 21472b245cb2SAnirudh Venkataramanan * To divide by 4K, shift right by 12 bits 21482b245cb2SAnirudh Venkataramanan * To divide by 3, multiply by 85, then divide by 256 21492b245cb2SAnirudh Venkataramanan * (Divide by 256 is done by shifting right by 8 bits) 21502b245cb2SAnirudh Venkataramanan * Finally, we add one to round up. Because 256 isn't an exact multiple of 21512b245cb2SAnirudh Venkataramanan * 3, we'll underestimate near each multiple of 12K. This is actually more 21522b245cb2SAnirudh Venkataramanan * accurate as we have 4K - 1 of wiggle room that we can fit into the last 21532b245cb2SAnirudh Venkataramanan * segment. For our purposes this is accurate out to 1M which is orders of 21542b245cb2SAnirudh Venkataramanan * magnitude greater than our largest possible GSO size. 21552b245cb2SAnirudh Venkataramanan * 21562b245cb2SAnirudh Venkataramanan * This would then be implemented as: 2157c585ea42SBrett Creeley * return (((size >> 12) * 85) >> 8) + ICE_DESCS_FOR_SKB_DATA_PTR; 21582b245cb2SAnirudh Venkataramanan * 21592b245cb2SAnirudh Venkataramanan * Since multiplication and division are commutative, we can reorder 21602b245cb2SAnirudh Venkataramanan * operations into: 2161c585ea42SBrett Creeley * return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR; 21622b245cb2SAnirudh Venkataramanan */ 21632b245cb2SAnirudh Venkataramanan static unsigned int ice_txd_use_count(unsigned int size) 21642b245cb2SAnirudh Venkataramanan { 2165c585ea42SBrett Creeley return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR; 21662b245cb2SAnirudh Venkataramanan } 21672b245cb2SAnirudh Venkataramanan 21682b245cb2SAnirudh Venkataramanan /** 2169d337f2afSAnirudh Venkataramanan * ice_xmit_desc_count - calculate number of Tx descriptors needed 21702b245cb2SAnirudh Venkataramanan * @skb: send buffer 21712b245cb2SAnirudh Venkataramanan * 21722b245cb2SAnirudh Venkataramanan * Returns number of data descriptors needed for this skb. 21732b245cb2SAnirudh Venkataramanan */ 21742b245cb2SAnirudh Venkataramanan static unsigned int ice_xmit_desc_count(struct sk_buff *skb) 21752b245cb2SAnirudh Venkataramanan { 2176d7840976SMatthew Wilcox (Oracle) const skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; 21772b245cb2SAnirudh Venkataramanan unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 21782b245cb2SAnirudh Venkataramanan unsigned int count = 0, size = skb_headlen(skb); 21792b245cb2SAnirudh Venkataramanan 21802b245cb2SAnirudh Venkataramanan for (;;) { 21812b245cb2SAnirudh Venkataramanan count += ice_txd_use_count(size); 21822b245cb2SAnirudh Venkataramanan 21832b245cb2SAnirudh Venkataramanan if (!nr_frags--) 21842b245cb2SAnirudh Venkataramanan break; 21852b245cb2SAnirudh Venkataramanan 21862b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag++); 21872b245cb2SAnirudh Venkataramanan } 21882b245cb2SAnirudh Venkataramanan 21892b245cb2SAnirudh Venkataramanan return count; 21902b245cb2SAnirudh Venkataramanan } 21912b245cb2SAnirudh Venkataramanan 21922b245cb2SAnirudh Venkataramanan /** 21932b245cb2SAnirudh Venkataramanan * __ice_chk_linearize - Check if there are more than 8 buffers per packet 21942b245cb2SAnirudh Venkataramanan * @skb: send buffer 21952b245cb2SAnirudh Venkataramanan * 21962b245cb2SAnirudh Venkataramanan * Note: This HW can't DMA more than 8 buffers to build a packet on the wire 21972b245cb2SAnirudh Venkataramanan * and so we need to figure out the cases where we need to linearize the skb. 21982b245cb2SAnirudh Venkataramanan * 21992b245cb2SAnirudh Venkataramanan * For TSO we need to count the TSO header and segment payload separately. 22002b245cb2SAnirudh Venkataramanan * As such we need to check cases where we have 7 fragments or more as we 22012b245cb2SAnirudh Venkataramanan * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 22022b245cb2SAnirudh Venkataramanan * the segment payload in the first descriptor, and another 7 for the 22032b245cb2SAnirudh Venkataramanan * fragments. 22042b245cb2SAnirudh Venkataramanan */ 22052b245cb2SAnirudh Venkataramanan static bool __ice_chk_linearize(struct sk_buff *skb) 22062b245cb2SAnirudh Venkataramanan { 2207d7840976SMatthew Wilcox (Oracle) const skb_frag_t *frag, *stale; 22082b245cb2SAnirudh Venkataramanan int nr_frags, sum; 22092b245cb2SAnirudh Venkataramanan 22102b245cb2SAnirudh Venkataramanan /* no need to check if number of frags is less than 7 */ 22112b245cb2SAnirudh Venkataramanan nr_frags = skb_shinfo(skb)->nr_frags; 22122b245cb2SAnirudh Venkataramanan if (nr_frags < (ICE_MAX_BUF_TXD - 1)) 22132b245cb2SAnirudh Venkataramanan return false; 22142b245cb2SAnirudh Venkataramanan 22152b245cb2SAnirudh Venkataramanan /* We need to walk through the list and validate that each group 22162b245cb2SAnirudh Venkataramanan * of 6 fragments totals at least gso_size. 22172b245cb2SAnirudh Venkataramanan */ 22182b245cb2SAnirudh Venkataramanan nr_frags -= ICE_MAX_BUF_TXD - 2; 22192b245cb2SAnirudh Venkataramanan frag = &skb_shinfo(skb)->frags[0]; 22202b245cb2SAnirudh Venkataramanan 22212b245cb2SAnirudh Venkataramanan /* Initialize size to the negative value of gso_size minus 1. We 22224ee656bbSTony Nguyen * use this as the worst case scenario in which the frag ahead 22232b245cb2SAnirudh Venkataramanan * of us only provides one byte which is why we are limited to 6 22242b245cb2SAnirudh Venkataramanan * descriptors for a single transmit as the header and previous 22252b245cb2SAnirudh Venkataramanan * fragment are already consuming 2 descriptors. 22262b245cb2SAnirudh Venkataramanan */ 22272b245cb2SAnirudh Venkataramanan sum = 1 - skb_shinfo(skb)->gso_size; 22282b245cb2SAnirudh Venkataramanan 22292b245cb2SAnirudh Venkataramanan /* Add size of frags 0 through 4 to create our initial sum */ 22302b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 22312b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 22322b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 22332b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 22342b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 22352b245cb2SAnirudh Venkataramanan 22362b245cb2SAnirudh Venkataramanan /* Walk through fragments adding latest fragment, testing it, and 22372b245cb2SAnirudh Venkataramanan * then removing stale fragments from the sum. 22382b245cb2SAnirudh Venkataramanan */ 22392b245cb2SAnirudh Venkataramanan stale = &skb_shinfo(skb)->frags[0]; 22402b245cb2SAnirudh Venkataramanan for (;;) { 22412b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 22422b245cb2SAnirudh Venkataramanan 22432b245cb2SAnirudh Venkataramanan /* if sum is negative we failed to make sufficient progress */ 22442b245cb2SAnirudh Venkataramanan if (sum < 0) 22452b245cb2SAnirudh Venkataramanan return true; 22462b245cb2SAnirudh Venkataramanan 22472b245cb2SAnirudh Venkataramanan if (!nr_frags--) 22482b245cb2SAnirudh Venkataramanan break; 22492b245cb2SAnirudh Venkataramanan 22502b245cb2SAnirudh Venkataramanan sum -= skb_frag_size(stale++); 22512b245cb2SAnirudh Venkataramanan } 22522b245cb2SAnirudh Venkataramanan 22532b245cb2SAnirudh Venkataramanan return false; 22542b245cb2SAnirudh Venkataramanan } 22552b245cb2SAnirudh Venkataramanan 22562b245cb2SAnirudh Venkataramanan /** 22572b245cb2SAnirudh Venkataramanan * ice_chk_linearize - Check if there are more than 8 fragments per packet 22582b245cb2SAnirudh Venkataramanan * @skb: send buffer 22592b245cb2SAnirudh Venkataramanan * @count: number of buffers used 22602b245cb2SAnirudh Venkataramanan * 22612b245cb2SAnirudh Venkataramanan * Note: Our HW can't scatter-gather more than 8 fragments to build 22622b245cb2SAnirudh Venkataramanan * a packet on the wire and so we need to figure out the cases where we 22632b245cb2SAnirudh Venkataramanan * need to linearize the skb. 22642b245cb2SAnirudh Venkataramanan */ 22652b245cb2SAnirudh Venkataramanan static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count) 22662b245cb2SAnirudh Venkataramanan { 22672b245cb2SAnirudh Venkataramanan /* Both TSO and single send will work if count is less than 8 */ 22682b245cb2SAnirudh Venkataramanan if (likely(count < ICE_MAX_BUF_TXD)) 22692b245cb2SAnirudh Venkataramanan return false; 22702b245cb2SAnirudh Venkataramanan 22712b245cb2SAnirudh Venkataramanan if (skb_is_gso(skb)) 22722b245cb2SAnirudh Venkataramanan return __ice_chk_linearize(skb); 22732b245cb2SAnirudh Venkataramanan 22742b245cb2SAnirudh Venkataramanan /* we can support up to 8 data buffers for a single send */ 22752b245cb2SAnirudh Venkataramanan return count != ICE_MAX_BUF_TXD; 22762b245cb2SAnirudh Venkataramanan } 22772b245cb2SAnirudh Venkataramanan 22782b245cb2SAnirudh Venkataramanan /** 22792b245cb2SAnirudh Venkataramanan * ice_xmit_frame_ring - Sends buffer on Tx ring 22802b245cb2SAnirudh Venkataramanan * @skb: send buffer 22812b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 22822b245cb2SAnirudh Venkataramanan * 22832b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 22842b245cb2SAnirudh Venkataramanan */ 22852b245cb2SAnirudh Venkataramanan static netdev_tx_t 22862b245cb2SAnirudh Venkataramanan ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) 22872b245cb2SAnirudh Venkataramanan { 2288d76a60baSAnirudh Venkataramanan struct ice_tx_offload_params offload = { 0 }; 22890c3a6101SDave Ertman struct ice_vsi *vsi = tx_ring->vsi; 22902b245cb2SAnirudh Venkataramanan struct ice_tx_buf *first; 22912b245cb2SAnirudh Venkataramanan unsigned int count; 2292d76a60baSAnirudh Venkataramanan int tso, csum; 22932b245cb2SAnirudh Venkataramanan 22942b245cb2SAnirudh Venkataramanan count = ice_xmit_desc_count(skb); 22952b245cb2SAnirudh Venkataramanan if (ice_chk_linearize(skb, count)) { 22962b245cb2SAnirudh Venkataramanan if (__skb_linearize(skb)) 22972b245cb2SAnirudh Venkataramanan goto out_drop; 22982b245cb2SAnirudh Venkataramanan count = ice_txd_use_count(skb->len); 22992b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_linearize++; 23002b245cb2SAnirudh Venkataramanan } 23012b245cb2SAnirudh Venkataramanan 23022b245cb2SAnirudh Venkataramanan /* need: 1 descriptor per page * PAGE_SIZE/ICE_MAX_DATA_PER_TXD, 23032b245cb2SAnirudh Venkataramanan * + 1 desc for skb_head_len/ICE_MAX_DATA_PER_TXD, 23042b245cb2SAnirudh Venkataramanan * + 4 desc gap to avoid the cache line where head is, 23052b245cb2SAnirudh Venkataramanan * + 1 desc for context descriptor, 23062b245cb2SAnirudh Venkataramanan * otherwise try next time 23072b245cb2SAnirudh Venkataramanan */ 2308c585ea42SBrett Creeley if (ice_maybe_stop_tx(tx_ring, count + ICE_DESCS_PER_CACHE_LINE + 2309c585ea42SBrett Creeley ICE_DESCS_FOR_CTX_DESC)) { 23102b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_busy++; 23112b245cb2SAnirudh Venkataramanan return NETDEV_TX_BUSY; 23122b245cb2SAnirudh Venkataramanan } 23132b245cb2SAnirudh Venkataramanan 2314d76a60baSAnirudh Venkataramanan offload.tx_ring = tx_ring; 2315d76a60baSAnirudh Venkataramanan 23162b245cb2SAnirudh Venkataramanan /* record the location of the first descriptor for this packet */ 23172b245cb2SAnirudh Venkataramanan first = &tx_ring->tx_buf[tx_ring->next_to_use]; 23182b245cb2SAnirudh Venkataramanan first->skb = skb; 23192b245cb2SAnirudh Venkataramanan first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 23202b245cb2SAnirudh Venkataramanan first->gso_segs = 1; 2321d76a60baSAnirudh Venkataramanan first->tx_flags = 0; 23222b245cb2SAnirudh Venkataramanan 2323d76a60baSAnirudh Venkataramanan /* prepare the VLAN tagging flags for Tx */ 2324d76a60baSAnirudh Venkataramanan if (ice_tx_prepare_vlan_flags(tx_ring, first)) 2325d76a60baSAnirudh Venkataramanan goto out_drop; 2326d76a60baSAnirudh Venkataramanan 2327d76a60baSAnirudh Venkataramanan /* set up TSO offload */ 2328d76a60baSAnirudh Venkataramanan tso = ice_tso(first, &offload); 2329d76a60baSAnirudh Venkataramanan if (tso < 0) 2330d76a60baSAnirudh Venkataramanan goto out_drop; 2331d76a60baSAnirudh Venkataramanan 2332d76a60baSAnirudh Venkataramanan /* always set up Tx checksum offload */ 2333d76a60baSAnirudh Venkataramanan csum = ice_tx_csum(first, &offload); 2334d76a60baSAnirudh Venkataramanan if (csum < 0) 2335d76a60baSAnirudh Venkataramanan goto out_drop; 2336d76a60baSAnirudh Venkataramanan 23370c3a6101SDave Ertman /* allow CONTROL frames egress from main VSI if FW LLDP disabled */ 23380c3a6101SDave Ertman if (unlikely(skb->priority == TC_PRIO_CONTROL && 23390c3a6101SDave Ertman vsi->type == ICE_VSI_PF && 23400c3a6101SDave Ertman vsi->port_info->is_sw_lldp)) 23410c3a6101SDave Ertman offload.cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX | 23420c3a6101SDave Ertman ICE_TX_CTX_DESC_SWTCH_UPLINK << 23430c3a6101SDave Ertman ICE_TXD_CTX_QW1_CMD_S); 23440c3a6101SDave Ertman 23450c3a6101SDave Ertman if (offload.cd_qw1 & ICE_TX_DESC_DTYPE_CTX) { 2346d76a60baSAnirudh Venkataramanan struct ice_tx_ctx_desc *cdesc; 234788865fc4SKarol Kolacinski u16 i = tx_ring->next_to_use; 2348d76a60baSAnirudh Venkataramanan 2349d76a60baSAnirudh Venkataramanan /* grab the next descriptor */ 2350d76a60baSAnirudh Venkataramanan cdesc = ICE_TX_CTX_DESC(tx_ring, i); 2351d76a60baSAnirudh Venkataramanan i++; 2352d76a60baSAnirudh Venkataramanan tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2353d76a60baSAnirudh Venkataramanan 2354d76a60baSAnirudh Venkataramanan /* setup context descriptor */ 2355d76a60baSAnirudh Venkataramanan cdesc->tunneling_params = cpu_to_le32(offload.cd_tunnel_params); 2356d76a60baSAnirudh Venkataramanan cdesc->l2tag2 = cpu_to_le16(offload.cd_l2tag2); 2357d76a60baSAnirudh Venkataramanan cdesc->rsvd = cpu_to_le16(0); 2358d76a60baSAnirudh Venkataramanan cdesc->qw1 = cpu_to_le64(offload.cd_qw1); 2359d76a60baSAnirudh Venkataramanan } 2360d76a60baSAnirudh Venkataramanan 2361d76a60baSAnirudh Venkataramanan ice_tx_map(tx_ring, first, &offload); 23622b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 23632b245cb2SAnirudh Venkataramanan 23642b245cb2SAnirudh Venkataramanan out_drop: 23652b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 23662b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 23672b245cb2SAnirudh Venkataramanan } 23682b245cb2SAnirudh Venkataramanan 23692b245cb2SAnirudh Venkataramanan /** 23702b245cb2SAnirudh Venkataramanan * ice_start_xmit - Selects the correct VSI and Tx queue to send buffer 23712b245cb2SAnirudh Venkataramanan * @skb: send buffer 23722b245cb2SAnirudh Venkataramanan * @netdev: network interface device structure 23732b245cb2SAnirudh Venkataramanan * 23742b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 23752b245cb2SAnirudh Venkataramanan */ 23762b245cb2SAnirudh Venkataramanan netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev) 23772b245cb2SAnirudh Venkataramanan { 23782b245cb2SAnirudh Venkataramanan struct ice_netdev_priv *np = netdev_priv(netdev); 23792b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = np->vsi; 23802b245cb2SAnirudh Venkataramanan struct ice_ring *tx_ring; 23812b245cb2SAnirudh Venkataramanan 23822b245cb2SAnirudh Venkataramanan tx_ring = vsi->tx_rings[skb->queue_mapping]; 23832b245cb2SAnirudh Venkataramanan 23842b245cb2SAnirudh Venkataramanan /* hardware can't handle really short frames, hardware padding works 23852b245cb2SAnirudh Venkataramanan * beyond this point 23862b245cb2SAnirudh Venkataramanan */ 23872b245cb2SAnirudh Venkataramanan if (skb_put_padto(skb, ICE_MIN_TX_LEN)) 23882b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 23892b245cb2SAnirudh Venkataramanan 23902b245cb2SAnirudh Venkataramanan return ice_xmit_frame_ring(skb, tx_ring); 23912b245cb2SAnirudh Venkataramanan } 2392148beb61SHenry Tieman 2393148beb61SHenry Tieman /** 2394148beb61SHenry Tieman * ice_clean_ctrl_tx_irq - interrupt handler for flow director Tx queue 2395148beb61SHenry Tieman * @tx_ring: tx_ring to clean 2396148beb61SHenry Tieman */ 2397148beb61SHenry Tieman void ice_clean_ctrl_tx_irq(struct ice_ring *tx_ring) 2398148beb61SHenry Tieman { 2399148beb61SHenry Tieman struct ice_vsi *vsi = tx_ring->vsi; 2400148beb61SHenry Tieman s16 i = tx_ring->next_to_clean; 2401148beb61SHenry Tieman int budget = ICE_DFLT_IRQ_WORK; 2402148beb61SHenry Tieman struct ice_tx_desc *tx_desc; 2403148beb61SHenry Tieman struct ice_tx_buf *tx_buf; 2404148beb61SHenry Tieman 2405148beb61SHenry Tieman tx_buf = &tx_ring->tx_buf[i]; 2406148beb61SHenry Tieman tx_desc = ICE_TX_DESC(tx_ring, i); 2407148beb61SHenry Tieman i -= tx_ring->count; 2408148beb61SHenry Tieman 2409148beb61SHenry Tieman do { 2410148beb61SHenry Tieman struct ice_tx_desc *eop_desc = tx_buf->next_to_watch; 2411148beb61SHenry Tieman 2412148beb61SHenry Tieman /* if next_to_watch is not set then there is no pending work */ 2413148beb61SHenry Tieman if (!eop_desc) 2414148beb61SHenry Tieman break; 2415148beb61SHenry Tieman 2416148beb61SHenry Tieman /* prevent any other reads prior to eop_desc */ 2417148beb61SHenry Tieman smp_rmb(); 2418148beb61SHenry Tieman 2419148beb61SHenry Tieman /* if the descriptor isn't done, no work to do */ 2420148beb61SHenry Tieman if (!(eop_desc->cmd_type_offset_bsz & 2421148beb61SHenry Tieman cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) 2422148beb61SHenry Tieman break; 2423148beb61SHenry Tieman 2424148beb61SHenry Tieman /* clear next_to_watch to prevent false hangs */ 2425148beb61SHenry Tieman tx_buf->next_to_watch = NULL; 2426148beb61SHenry Tieman tx_desc->buf_addr = 0; 2427148beb61SHenry Tieman tx_desc->cmd_type_offset_bsz = 0; 2428148beb61SHenry Tieman 2429148beb61SHenry Tieman /* move past filter desc */ 2430148beb61SHenry Tieman tx_buf++; 2431148beb61SHenry Tieman tx_desc++; 2432148beb61SHenry Tieman i++; 2433148beb61SHenry Tieman if (unlikely(!i)) { 2434148beb61SHenry Tieman i -= tx_ring->count; 2435148beb61SHenry Tieman tx_buf = tx_ring->tx_buf; 2436148beb61SHenry Tieman tx_desc = ICE_TX_DESC(tx_ring, 0); 2437148beb61SHenry Tieman } 2438148beb61SHenry Tieman 2439148beb61SHenry Tieman /* unmap the data header */ 2440148beb61SHenry Tieman if (dma_unmap_len(tx_buf, len)) 2441148beb61SHenry Tieman dma_unmap_single(tx_ring->dev, 2442148beb61SHenry Tieman dma_unmap_addr(tx_buf, dma), 2443148beb61SHenry Tieman dma_unmap_len(tx_buf, len), 2444148beb61SHenry Tieman DMA_TO_DEVICE); 2445148beb61SHenry Tieman if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) 2446148beb61SHenry Tieman devm_kfree(tx_ring->dev, tx_buf->raw_buf); 2447148beb61SHenry Tieman 2448148beb61SHenry Tieman /* clear next_to_watch to prevent false hangs */ 2449148beb61SHenry Tieman tx_buf->raw_buf = NULL; 2450148beb61SHenry Tieman tx_buf->tx_flags = 0; 2451148beb61SHenry Tieman tx_buf->next_to_watch = NULL; 2452148beb61SHenry Tieman dma_unmap_len_set(tx_buf, len, 0); 2453148beb61SHenry Tieman tx_desc->buf_addr = 0; 2454148beb61SHenry Tieman tx_desc->cmd_type_offset_bsz = 0; 2455148beb61SHenry Tieman 2456148beb61SHenry Tieman /* move past eop_desc for start of next FD desc */ 2457148beb61SHenry Tieman tx_buf++; 2458148beb61SHenry Tieman tx_desc++; 2459148beb61SHenry Tieman i++; 2460148beb61SHenry Tieman if (unlikely(!i)) { 2461148beb61SHenry Tieman i -= tx_ring->count; 2462148beb61SHenry Tieman tx_buf = tx_ring->tx_buf; 2463148beb61SHenry Tieman tx_desc = ICE_TX_DESC(tx_ring, 0); 2464148beb61SHenry Tieman } 2465148beb61SHenry Tieman 2466148beb61SHenry Tieman budget--; 2467148beb61SHenry Tieman } while (likely(budget)); 2468148beb61SHenry Tieman 2469148beb61SHenry Tieman i += tx_ring->count; 2470148beb61SHenry Tieman tx_ring->next_to_clean = i; 2471148beb61SHenry Tieman 2472148beb61SHenry Tieman /* re-enable interrupt if needed */ 2473148beb61SHenry Tieman ice_irq_dynamic_ena(&vsi->back->hw, vsi, vsi->q_vectors[0]); 2474148beb61SHenry Tieman } 2475