1cdedef59SAnirudh Venkataramanan // SPDX-License-Identifier: GPL-2.0 2cdedef59SAnirudh Venkataramanan /* Copyright (c) 2018, Intel Corporation. */ 3cdedef59SAnirudh Venkataramanan 4cdedef59SAnirudh Venkataramanan /* The driver transmit and receive code */ 5cdedef59SAnirudh Venkataramanan 6cdedef59SAnirudh Venkataramanan #include <linux/prefetch.h> 7cdedef59SAnirudh Venkataramanan #include <linux/mm.h> 8cdedef59SAnirudh Venkataramanan #include "ice.h" 9cdedef59SAnirudh Venkataramanan 102b245cb2SAnirudh Venkataramanan #define ICE_RX_HDR_SIZE 256 112b245cb2SAnirudh Venkataramanan 12cdedef59SAnirudh Venkataramanan /** 13cdedef59SAnirudh Venkataramanan * ice_unmap_and_free_tx_buf - Release a Tx buffer 14cdedef59SAnirudh Venkataramanan * @ring: the ring that owns the buffer 15cdedef59SAnirudh Venkataramanan * @tx_buf: the buffer to free 16cdedef59SAnirudh Venkataramanan */ 17cdedef59SAnirudh Venkataramanan static void 18cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(struct ice_ring *ring, struct ice_tx_buf *tx_buf) 19cdedef59SAnirudh Venkataramanan { 20cdedef59SAnirudh Venkataramanan if (tx_buf->skb) { 21cdedef59SAnirudh Venkataramanan dev_kfree_skb_any(tx_buf->skb); 22cdedef59SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) 23cdedef59SAnirudh Venkataramanan dma_unmap_single(ring->dev, 24cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 25cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 26cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 27cdedef59SAnirudh Venkataramanan } else if (dma_unmap_len(tx_buf, len)) { 28cdedef59SAnirudh Venkataramanan dma_unmap_page(ring->dev, 29cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 30cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 31cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 32cdedef59SAnirudh Venkataramanan } 33cdedef59SAnirudh Venkataramanan 34cdedef59SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 35cdedef59SAnirudh Venkataramanan tx_buf->skb = NULL; 36cdedef59SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 37cdedef59SAnirudh Venkataramanan /* tx_buf must be completely set up in the transmit path */ 38cdedef59SAnirudh Venkataramanan } 39cdedef59SAnirudh Venkataramanan 40cdedef59SAnirudh Venkataramanan static struct netdev_queue *txring_txq(const struct ice_ring *ring) 41cdedef59SAnirudh Venkataramanan { 42cdedef59SAnirudh Venkataramanan return netdev_get_tx_queue(ring->netdev, ring->q_index); 43cdedef59SAnirudh Venkataramanan } 44cdedef59SAnirudh Venkataramanan 45cdedef59SAnirudh Venkataramanan /** 46cdedef59SAnirudh Venkataramanan * ice_clean_tx_ring - Free any empty Tx buffers 47cdedef59SAnirudh Venkataramanan * @tx_ring: ring to be cleaned 48cdedef59SAnirudh Venkataramanan */ 49cdedef59SAnirudh Venkataramanan void ice_clean_tx_ring(struct ice_ring *tx_ring) 50cdedef59SAnirudh Venkataramanan { 51cdedef59SAnirudh Venkataramanan unsigned long size; 52cdedef59SAnirudh Venkataramanan u16 i; 53cdedef59SAnirudh Venkataramanan 54cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 55cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 56cdedef59SAnirudh Venkataramanan return; 57cdedef59SAnirudh Venkataramanan 58cdedef59SAnirudh Venkataramanan /* Free all the Tx ring sk_bufss */ 59cdedef59SAnirudh Venkataramanan for (i = 0; i < tx_ring->count; i++) 60cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]); 61cdedef59SAnirudh Venkataramanan 62cdedef59SAnirudh Venkataramanan size = sizeof(struct ice_tx_buf) * tx_ring->count; 63cdedef59SAnirudh Venkataramanan memset(tx_ring->tx_buf, 0, size); 64cdedef59SAnirudh Venkataramanan 65cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 66cdedef59SAnirudh Venkataramanan memset(tx_ring->desc, 0, tx_ring->size); 67cdedef59SAnirudh Venkataramanan 68cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 69cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 70cdedef59SAnirudh Venkataramanan 71cdedef59SAnirudh Venkataramanan if (!tx_ring->netdev) 72cdedef59SAnirudh Venkataramanan return; 73cdedef59SAnirudh Venkataramanan 74cdedef59SAnirudh Venkataramanan /* cleanup Tx queue statistics */ 75cdedef59SAnirudh Venkataramanan netdev_tx_reset_queue(txring_txq(tx_ring)); 76cdedef59SAnirudh Venkataramanan } 77cdedef59SAnirudh Venkataramanan 78cdedef59SAnirudh Venkataramanan /** 79cdedef59SAnirudh Venkataramanan * ice_free_tx_ring - Free Tx resources per queue 80cdedef59SAnirudh Venkataramanan * @tx_ring: Tx descriptor ring for a specific queue 81cdedef59SAnirudh Venkataramanan * 82cdedef59SAnirudh Venkataramanan * Free all transmit software resources 83cdedef59SAnirudh Venkataramanan */ 84cdedef59SAnirudh Venkataramanan void ice_free_tx_ring(struct ice_ring *tx_ring) 85cdedef59SAnirudh Venkataramanan { 86cdedef59SAnirudh Venkataramanan ice_clean_tx_ring(tx_ring); 87cdedef59SAnirudh Venkataramanan devm_kfree(tx_ring->dev, tx_ring->tx_buf); 88cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 89cdedef59SAnirudh Venkataramanan 90cdedef59SAnirudh Venkataramanan if (tx_ring->desc) { 91cdedef59SAnirudh Venkataramanan dmam_free_coherent(tx_ring->dev, tx_ring->size, 92cdedef59SAnirudh Venkataramanan tx_ring->desc, tx_ring->dma); 93cdedef59SAnirudh Venkataramanan tx_ring->desc = NULL; 94cdedef59SAnirudh Venkataramanan } 95cdedef59SAnirudh Venkataramanan } 96cdedef59SAnirudh Venkataramanan 97cdedef59SAnirudh Venkataramanan /** 982b245cb2SAnirudh Venkataramanan * ice_clean_tx_irq - Reclaim resources after transmit completes 992b245cb2SAnirudh Venkataramanan * @vsi: the VSI we care about 1002b245cb2SAnirudh Venkataramanan * @tx_ring: Tx ring to clean 1012b245cb2SAnirudh Venkataramanan * @napi_budget: Used to determine if we are in netpoll 1022b245cb2SAnirudh Venkataramanan * 1032b245cb2SAnirudh Venkataramanan * Returns true if there's any budget left (e.g. the clean is finished) 1042b245cb2SAnirudh Venkataramanan */ 1052b245cb2SAnirudh Venkataramanan static bool ice_clean_tx_irq(struct ice_vsi *vsi, struct ice_ring *tx_ring, 1062b245cb2SAnirudh Venkataramanan int napi_budget) 1072b245cb2SAnirudh Venkataramanan { 1082b245cb2SAnirudh Venkataramanan unsigned int total_bytes = 0, total_pkts = 0; 1092b245cb2SAnirudh Venkataramanan unsigned int budget = vsi->work_lmt; 1102b245cb2SAnirudh Venkataramanan s16 i = tx_ring->next_to_clean; 1112b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 1122b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 1132b245cb2SAnirudh Venkataramanan 1142b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 1152b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 1162b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1172b245cb2SAnirudh Venkataramanan 1182b245cb2SAnirudh Venkataramanan do { 1192b245cb2SAnirudh Venkataramanan struct ice_tx_desc *eop_desc = tx_buf->next_to_watch; 1202b245cb2SAnirudh Venkataramanan 1212b245cb2SAnirudh Venkataramanan /* if next_to_watch is not set then there is no work pending */ 1222b245cb2SAnirudh Venkataramanan if (!eop_desc) 1232b245cb2SAnirudh Venkataramanan break; 1242b245cb2SAnirudh Venkataramanan 1252b245cb2SAnirudh Venkataramanan smp_rmb(); /* prevent any other reads prior to eop_desc */ 1262b245cb2SAnirudh Venkataramanan 1272b245cb2SAnirudh Venkataramanan /* if the descriptor isn't done, no work yet to do */ 1282b245cb2SAnirudh Venkataramanan if (!(eop_desc->cmd_type_offset_bsz & 1292b245cb2SAnirudh Venkataramanan cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) 1302b245cb2SAnirudh Venkataramanan break; 1312b245cb2SAnirudh Venkataramanan 1322b245cb2SAnirudh Venkataramanan /* clear next_to_watch to prevent false hangs */ 1332b245cb2SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 1342b245cb2SAnirudh Venkataramanan 1352b245cb2SAnirudh Venkataramanan /* update the statistics for this packet */ 1362b245cb2SAnirudh Venkataramanan total_bytes += tx_buf->bytecount; 1372b245cb2SAnirudh Venkataramanan total_pkts += tx_buf->gso_segs; 1382b245cb2SAnirudh Venkataramanan 1392b245cb2SAnirudh Venkataramanan /* free the skb */ 1402b245cb2SAnirudh Venkataramanan napi_consume_skb(tx_buf->skb, napi_budget); 1412b245cb2SAnirudh Venkataramanan 1422b245cb2SAnirudh Venkataramanan /* unmap skb header data */ 1432b245cb2SAnirudh Venkataramanan dma_unmap_single(tx_ring->dev, 1442b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 1452b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 1462b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1472b245cb2SAnirudh Venkataramanan 1482b245cb2SAnirudh Venkataramanan /* clear tx_buf data */ 1492b245cb2SAnirudh Venkataramanan tx_buf->skb = NULL; 1502b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 1512b245cb2SAnirudh Venkataramanan 1522b245cb2SAnirudh Venkataramanan /* unmap remaining buffers */ 1532b245cb2SAnirudh Venkataramanan while (tx_desc != eop_desc) { 1542b245cb2SAnirudh Venkataramanan tx_buf++; 1552b245cb2SAnirudh Venkataramanan tx_desc++; 1562b245cb2SAnirudh Venkataramanan i++; 1572b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 1582b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1592b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 1602b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1612b245cb2SAnirudh Venkataramanan } 1622b245cb2SAnirudh Venkataramanan 1632b245cb2SAnirudh Venkataramanan /* unmap any remaining paged data */ 1642b245cb2SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) { 1652b245cb2SAnirudh Venkataramanan dma_unmap_page(tx_ring->dev, 1662b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 1672b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 1682b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1692b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 1702b245cb2SAnirudh Venkataramanan } 1712b245cb2SAnirudh Venkataramanan } 1722b245cb2SAnirudh Venkataramanan 1732b245cb2SAnirudh Venkataramanan /* move us one more past the eop_desc for start of next pkt */ 1742b245cb2SAnirudh Venkataramanan tx_buf++; 1752b245cb2SAnirudh Venkataramanan tx_desc++; 1762b245cb2SAnirudh Venkataramanan i++; 1772b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 1782b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1792b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 1802b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1812b245cb2SAnirudh Venkataramanan } 1822b245cb2SAnirudh Venkataramanan 1832b245cb2SAnirudh Venkataramanan prefetch(tx_desc); 1842b245cb2SAnirudh Venkataramanan 1852b245cb2SAnirudh Venkataramanan /* update budget accounting */ 1862b245cb2SAnirudh Venkataramanan budget--; 1872b245cb2SAnirudh Venkataramanan } while (likely(budget)); 1882b245cb2SAnirudh Venkataramanan 1892b245cb2SAnirudh Venkataramanan i += tx_ring->count; 1902b245cb2SAnirudh Venkataramanan tx_ring->next_to_clean = i; 1912b245cb2SAnirudh Venkataramanan u64_stats_update_begin(&tx_ring->syncp); 1922b245cb2SAnirudh Venkataramanan tx_ring->stats.bytes += total_bytes; 1932b245cb2SAnirudh Venkataramanan tx_ring->stats.pkts += total_pkts; 1942b245cb2SAnirudh Venkataramanan u64_stats_update_end(&tx_ring->syncp); 1952b245cb2SAnirudh Venkataramanan tx_ring->q_vector->tx.total_bytes += total_bytes; 1962b245cb2SAnirudh Venkataramanan tx_ring->q_vector->tx.total_pkts += total_pkts; 1972b245cb2SAnirudh Venkataramanan 1982b245cb2SAnirudh Venkataramanan netdev_tx_completed_queue(txring_txq(tx_ring), total_pkts, 1992b245cb2SAnirudh Venkataramanan total_bytes); 2002b245cb2SAnirudh Venkataramanan 2012b245cb2SAnirudh Venkataramanan #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2)) 2022b245cb2SAnirudh Venkataramanan if (unlikely(total_pkts && netif_carrier_ok(tx_ring->netdev) && 2032b245cb2SAnirudh Venkataramanan (ICE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 2042b245cb2SAnirudh Venkataramanan /* Make sure that anybody stopping the queue after this 2052b245cb2SAnirudh Venkataramanan * sees the new next_to_clean. 2062b245cb2SAnirudh Venkataramanan */ 2072b245cb2SAnirudh Venkataramanan smp_mb(); 2082b245cb2SAnirudh Venkataramanan if (__netif_subqueue_stopped(tx_ring->netdev, 2092b245cb2SAnirudh Venkataramanan tx_ring->q_index) && 2102b245cb2SAnirudh Venkataramanan !test_bit(__ICE_DOWN, vsi->state)) { 2112b245cb2SAnirudh Venkataramanan netif_wake_subqueue(tx_ring->netdev, 2122b245cb2SAnirudh Venkataramanan tx_ring->q_index); 2132b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 2142b245cb2SAnirudh Venkataramanan } 2152b245cb2SAnirudh Venkataramanan } 2162b245cb2SAnirudh Venkataramanan 2172b245cb2SAnirudh Venkataramanan return !!budget; 2182b245cb2SAnirudh Venkataramanan } 2192b245cb2SAnirudh Venkataramanan 2202b245cb2SAnirudh Venkataramanan /** 221cdedef59SAnirudh Venkataramanan * ice_setup_tx_ring - Allocate the Tx descriptors 222cdedef59SAnirudh Venkataramanan * @tx_ring: the tx ring to set up 223cdedef59SAnirudh Venkataramanan * 224cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 225cdedef59SAnirudh Venkataramanan */ 226cdedef59SAnirudh Venkataramanan int ice_setup_tx_ring(struct ice_ring *tx_ring) 227cdedef59SAnirudh Venkataramanan { 228cdedef59SAnirudh Venkataramanan struct device *dev = tx_ring->dev; 229cdedef59SAnirudh Venkataramanan int bi_size; 230cdedef59SAnirudh Venkataramanan 231cdedef59SAnirudh Venkataramanan if (!dev) 232cdedef59SAnirudh Venkataramanan return -ENOMEM; 233cdedef59SAnirudh Venkataramanan 234cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 235cdedef59SAnirudh Venkataramanan WARN_ON(tx_ring->tx_buf); 236cdedef59SAnirudh Venkataramanan bi_size = sizeof(struct ice_tx_buf) * tx_ring->count; 237cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = devm_kzalloc(dev, bi_size, GFP_KERNEL); 238cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 239cdedef59SAnirudh Venkataramanan return -ENOMEM; 240cdedef59SAnirudh Venkataramanan 241cdedef59SAnirudh Venkataramanan /* round up to nearest 4K */ 242cdedef59SAnirudh Venkataramanan tx_ring->size = tx_ring->count * sizeof(struct ice_tx_desc); 243cdedef59SAnirudh Venkataramanan tx_ring->size = ALIGN(tx_ring->size, 4096); 244cdedef59SAnirudh Venkataramanan tx_ring->desc = dmam_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, 245cdedef59SAnirudh Venkataramanan GFP_KERNEL); 246cdedef59SAnirudh Venkataramanan if (!tx_ring->desc) { 247cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 248cdedef59SAnirudh Venkataramanan tx_ring->size); 249cdedef59SAnirudh Venkataramanan goto err; 250cdedef59SAnirudh Venkataramanan } 251cdedef59SAnirudh Venkataramanan 252cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 253cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 254cdedef59SAnirudh Venkataramanan return 0; 255cdedef59SAnirudh Venkataramanan 256cdedef59SAnirudh Venkataramanan err: 257cdedef59SAnirudh Venkataramanan devm_kfree(dev, tx_ring->tx_buf); 258cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 259cdedef59SAnirudh Venkataramanan return -ENOMEM; 260cdedef59SAnirudh Venkataramanan } 261cdedef59SAnirudh Venkataramanan 262cdedef59SAnirudh Venkataramanan /** 263cdedef59SAnirudh Venkataramanan * ice_clean_rx_ring - Free Rx buffers 264cdedef59SAnirudh Venkataramanan * @rx_ring: ring to be cleaned 265cdedef59SAnirudh Venkataramanan */ 266cdedef59SAnirudh Venkataramanan void ice_clean_rx_ring(struct ice_ring *rx_ring) 267cdedef59SAnirudh Venkataramanan { 268cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 269cdedef59SAnirudh Venkataramanan unsigned long size; 270cdedef59SAnirudh Venkataramanan u16 i; 271cdedef59SAnirudh Venkataramanan 272cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 273cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 274cdedef59SAnirudh Venkataramanan return; 275cdedef59SAnirudh Venkataramanan 276cdedef59SAnirudh Venkataramanan /* Free all the Rx ring sk_buffs */ 277cdedef59SAnirudh Venkataramanan for (i = 0; i < rx_ring->count; i++) { 278cdedef59SAnirudh Venkataramanan struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i]; 279cdedef59SAnirudh Venkataramanan 280cdedef59SAnirudh Venkataramanan if (rx_buf->skb) { 281cdedef59SAnirudh Venkataramanan dev_kfree_skb(rx_buf->skb); 282cdedef59SAnirudh Venkataramanan rx_buf->skb = NULL; 283cdedef59SAnirudh Venkataramanan } 284cdedef59SAnirudh Venkataramanan if (!rx_buf->page) 285cdedef59SAnirudh Venkataramanan continue; 286cdedef59SAnirudh Venkataramanan 287cdedef59SAnirudh Venkataramanan dma_unmap_page(dev, rx_buf->dma, PAGE_SIZE, DMA_FROM_DEVICE); 288cdedef59SAnirudh Venkataramanan __free_pages(rx_buf->page, 0); 289cdedef59SAnirudh Venkataramanan 290cdedef59SAnirudh Venkataramanan rx_buf->page = NULL; 291cdedef59SAnirudh Venkataramanan rx_buf->page_offset = 0; 292cdedef59SAnirudh Venkataramanan } 293cdedef59SAnirudh Venkataramanan 294cdedef59SAnirudh Venkataramanan size = sizeof(struct ice_rx_buf) * rx_ring->count; 295cdedef59SAnirudh Venkataramanan memset(rx_ring->rx_buf, 0, size); 296cdedef59SAnirudh Venkataramanan 297cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 298cdedef59SAnirudh Venkataramanan memset(rx_ring->desc, 0, rx_ring->size); 299cdedef59SAnirudh Venkataramanan 300cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = 0; 301cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 302cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 303cdedef59SAnirudh Venkataramanan } 304cdedef59SAnirudh Venkataramanan 305cdedef59SAnirudh Venkataramanan /** 306cdedef59SAnirudh Venkataramanan * ice_free_rx_ring - Free Rx resources 307cdedef59SAnirudh Venkataramanan * @rx_ring: ring to clean the resources from 308cdedef59SAnirudh Venkataramanan * 309cdedef59SAnirudh Venkataramanan * Free all receive software resources 310cdedef59SAnirudh Venkataramanan */ 311cdedef59SAnirudh Venkataramanan void ice_free_rx_ring(struct ice_ring *rx_ring) 312cdedef59SAnirudh Venkataramanan { 313cdedef59SAnirudh Venkataramanan ice_clean_rx_ring(rx_ring); 314cdedef59SAnirudh Venkataramanan devm_kfree(rx_ring->dev, rx_ring->rx_buf); 315cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 316cdedef59SAnirudh Venkataramanan 317cdedef59SAnirudh Venkataramanan if (rx_ring->desc) { 318cdedef59SAnirudh Venkataramanan dmam_free_coherent(rx_ring->dev, rx_ring->size, 319cdedef59SAnirudh Venkataramanan rx_ring->desc, rx_ring->dma); 320cdedef59SAnirudh Venkataramanan rx_ring->desc = NULL; 321cdedef59SAnirudh Venkataramanan } 322cdedef59SAnirudh Venkataramanan } 323cdedef59SAnirudh Venkataramanan 324cdedef59SAnirudh Venkataramanan /** 325cdedef59SAnirudh Venkataramanan * ice_setup_rx_ring - Allocate the Rx descriptors 326cdedef59SAnirudh Venkataramanan * @rx_ring: the rx ring to set up 327cdedef59SAnirudh Venkataramanan * 328cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 329cdedef59SAnirudh Venkataramanan */ 330cdedef59SAnirudh Venkataramanan int ice_setup_rx_ring(struct ice_ring *rx_ring) 331cdedef59SAnirudh Venkataramanan { 332cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 333cdedef59SAnirudh Venkataramanan int bi_size; 334cdedef59SAnirudh Venkataramanan 335cdedef59SAnirudh Venkataramanan if (!dev) 336cdedef59SAnirudh Venkataramanan return -ENOMEM; 337cdedef59SAnirudh Venkataramanan 338cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 339cdedef59SAnirudh Venkataramanan WARN_ON(rx_ring->rx_buf); 340cdedef59SAnirudh Venkataramanan bi_size = sizeof(struct ice_rx_buf) * rx_ring->count; 341cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = devm_kzalloc(dev, bi_size, GFP_KERNEL); 342cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 343cdedef59SAnirudh Venkataramanan return -ENOMEM; 344cdedef59SAnirudh Venkataramanan 345cdedef59SAnirudh Venkataramanan /* round up to nearest 4K */ 346cdedef59SAnirudh Venkataramanan rx_ring->size = rx_ring->count * sizeof(union ice_32byte_rx_desc); 347cdedef59SAnirudh Venkataramanan rx_ring->size = ALIGN(rx_ring->size, 4096); 348cdedef59SAnirudh Venkataramanan rx_ring->desc = dmam_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, 349cdedef59SAnirudh Venkataramanan GFP_KERNEL); 350cdedef59SAnirudh Venkataramanan if (!rx_ring->desc) { 351cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 352cdedef59SAnirudh Venkataramanan rx_ring->size); 353cdedef59SAnirudh Venkataramanan goto err; 354cdedef59SAnirudh Venkataramanan } 355cdedef59SAnirudh Venkataramanan 356cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 357cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 358cdedef59SAnirudh Venkataramanan return 0; 359cdedef59SAnirudh Venkataramanan 360cdedef59SAnirudh Venkataramanan err: 361cdedef59SAnirudh Venkataramanan devm_kfree(dev, rx_ring->rx_buf); 362cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 363cdedef59SAnirudh Venkataramanan return -ENOMEM; 364cdedef59SAnirudh Venkataramanan } 365cdedef59SAnirudh Venkataramanan 366cdedef59SAnirudh Venkataramanan /** 367cdedef59SAnirudh Venkataramanan * ice_release_rx_desc - Store the new tail and head values 368cdedef59SAnirudh Venkataramanan * @rx_ring: ring to bump 369cdedef59SAnirudh Venkataramanan * @val: new head index 370cdedef59SAnirudh Venkataramanan */ 371cdedef59SAnirudh Venkataramanan static void ice_release_rx_desc(struct ice_ring *rx_ring, u32 val) 372cdedef59SAnirudh Venkataramanan { 373cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = val; 374cdedef59SAnirudh Venkataramanan 375cdedef59SAnirudh Venkataramanan /* update next to alloc since we have filled the ring */ 376cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = val; 377cdedef59SAnirudh Venkataramanan 378cdedef59SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w 379cdedef59SAnirudh Venkataramanan * know there are new descriptors to fetch. (Only 380cdedef59SAnirudh Venkataramanan * applicable for weak-ordered memory model archs, 381cdedef59SAnirudh Venkataramanan * such as IA-64). 382cdedef59SAnirudh Venkataramanan */ 383cdedef59SAnirudh Venkataramanan wmb(); 384cdedef59SAnirudh Venkataramanan writel(val, rx_ring->tail); 385cdedef59SAnirudh Venkataramanan } 386cdedef59SAnirudh Venkataramanan 387cdedef59SAnirudh Venkataramanan /** 388cdedef59SAnirudh Venkataramanan * ice_alloc_mapped_page - recycle or make a new page 389cdedef59SAnirudh Venkataramanan * @rx_ring: ring to use 390cdedef59SAnirudh Venkataramanan * @bi: rx_buf struct to modify 391cdedef59SAnirudh Venkataramanan * 392cdedef59SAnirudh Venkataramanan * Returns true if the page was successfully allocated or 393cdedef59SAnirudh Venkataramanan * reused. 394cdedef59SAnirudh Venkataramanan */ 395cdedef59SAnirudh Venkataramanan static bool ice_alloc_mapped_page(struct ice_ring *rx_ring, 396cdedef59SAnirudh Venkataramanan struct ice_rx_buf *bi) 397cdedef59SAnirudh Venkataramanan { 398cdedef59SAnirudh Venkataramanan struct page *page = bi->page; 399cdedef59SAnirudh Venkataramanan dma_addr_t dma; 400cdedef59SAnirudh Venkataramanan 401cdedef59SAnirudh Venkataramanan /* since we are recycling buffers we should seldom need to alloc */ 4022b245cb2SAnirudh Venkataramanan if (likely(page)) { 4032b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 404cdedef59SAnirudh Venkataramanan return true; 4052b245cb2SAnirudh Venkataramanan } 406cdedef59SAnirudh Venkataramanan 407cdedef59SAnirudh Venkataramanan /* alloc new page for storage */ 408cdedef59SAnirudh Venkataramanan page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 4092b245cb2SAnirudh Venkataramanan if (unlikely(!page)) { 4102b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 411cdedef59SAnirudh Venkataramanan return false; 4122b245cb2SAnirudh Venkataramanan } 413cdedef59SAnirudh Venkataramanan 414cdedef59SAnirudh Venkataramanan /* map page for use */ 415cdedef59SAnirudh Venkataramanan dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 416cdedef59SAnirudh Venkataramanan 417cdedef59SAnirudh Venkataramanan /* if mapping failed free memory back to system since 418cdedef59SAnirudh Venkataramanan * there isn't much point in holding memory we can't use 419cdedef59SAnirudh Venkataramanan */ 420cdedef59SAnirudh Venkataramanan if (dma_mapping_error(rx_ring->dev, dma)) { 421cdedef59SAnirudh Venkataramanan __free_pages(page, 0); 4222b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 423cdedef59SAnirudh Venkataramanan return false; 424cdedef59SAnirudh Venkataramanan } 425cdedef59SAnirudh Venkataramanan 426cdedef59SAnirudh Venkataramanan bi->dma = dma; 427cdedef59SAnirudh Venkataramanan bi->page = page; 428cdedef59SAnirudh Venkataramanan bi->page_offset = 0; 429cdedef59SAnirudh Venkataramanan 430cdedef59SAnirudh Venkataramanan return true; 431cdedef59SAnirudh Venkataramanan } 432cdedef59SAnirudh Venkataramanan 433cdedef59SAnirudh Venkataramanan /** 434cdedef59SAnirudh Venkataramanan * ice_alloc_rx_bufs - Replace used receive buffers 435cdedef59SAnirudh Venkataramanan * @rx_ring: ring to place buffers on 436cdedef59SAnirudh Venkataramanan * @cleaned_count: number of buffers to replace 437cdedef59SAnirudh Venkataramanan * 438cdedef59SAnirudh Venkataramanan * Returns false if all allocations were successful, true if any fail 439cdedef59SAnirudh Venkataramanan */ 440cdedef59SAnirudh Venkataramanan bool ice_alloc_rx_bufs(struct ice_ring *rx_ring, u16 cleaned_count) 441cdedef59SAnirudh Venkataramanan { 442cdedef59SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 443cdedef59SAnirudh Venkataramanan u16 ntu = rx_ring->next_to_use; 444cdedef59SAnirudh Venkataramanan struct ice_rx_buf *bi; 445cdedef59SAnirudh Venkataramanan 446cdedef59SAnirudh Venkataramanan /* do nothing if no valid netdev defined */ 447cdedef59SAnirudh Venkataramanan if (!rx_ring->netdev || !cleaned_count) 448cdedef59SAnirudh Venkataramanan return false; 449cdedef59SAnirudh Venkataramanan 450cdedef59SAnirudh Venkataramanan /* get the RX descriptor and buffer based on next_to_use */ 451cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, ntu); 452cdedef59SAnirudh Venkataramanan bi = &rx_ring->rx_buf[ntu]; 453cdedef59SAnirudh Venkataramanan 454cdedef59SAnirudh Venkataramanan do { 455cdedef59SAnirudh Venkataramanan if (!ice_alloc_mapped_page(rx_ring, bi)) 456cdedef59SAnirudh Venkataramanan goto no_bufs; 457cdedef59SAnirudh Venkataramanan 458cdedef59SAnirudh Venkataramanan /* Refresh the desc even if buffer_addrs didn't change 459cdedef59SAnirudh Venkataramanan * because each write-back erases this info. 460cdedef59SAnirudh Venkataramanan */ 461cdedef59SAnirudh Venkataramanan rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 462cdedef59SAnirudh Venkataramanan 463cdedef59SAnirudh Venkataramanan rx_desc++; 464cdedef59SAnirudh Venkataramanan bi++; 465cdedef59SAnirudh Venkataramanan ntu++; 466cdedef59SAnirudh Venkataramanan if (unlikely(ntu == rx_ring->count)) { 467cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, 0); 468cdedef59SAnirudh Venkataramanan bi = rx_ring->rx_buf; 469cdedef59SAnirudh Venkataramanan ntu = 0; 470cdedef59SAnirudh Venkataramanan } 471cdedef59SAnirudh Venkataramanan 472cdedef59SAnirudh Venkataramanan /* clear the status bits for the next_to_use descriptor */ 473cdedef59SAnirudh Venkataramanan rx_desc->wb.status_error0 = 0; 474cdedef59SAnirudh Venkataramanan 475cdedef59SAnirudh Venkataramanan cleaned_count--; 476cdedef59SAnirudh Venkataramanan } while (cleaned_count); 477cdedef59SAnirudh Venkataramanan 478cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 479cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 480cdedef59SAnirudh Venkataramanan 481cdedef59SAnirudh Venkataramanan return false; 482cdedef59SAnirudh Venkataramanan 483cdedef59SAnirudh Venkataramanan no_bufs: 484cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 485cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 486cdedef59SAnirudh Venkataramanan 487cdedef59SAnirudh Venkataramanan /* make sure to come back via polling to try again after 488cdedef59SAnirudh Venkataramanan * allocation failure 489cdedef59SAnirudh Venkataramanan */ 490cdedef59SAnirudh Venkataramanan return true; 491cdedef59SAnirudh Venkataramanan } 4922b245cb2SAnirudh Venkataramanan 4932b245cb2SAnirudh Venkataramanan /** 4942b245cb2SAnirudh Venkataramanan * ice_page_is_reserved - check if reuse is possible 4952b245cb2SAnirudh Venkataramanan * @page: page struct to check 4962b245cb2SAnirudh Venkataramanan */ 4972b245cb2SAnirudh Venkataramanan static bool ice_page_is_reserved(struct page *page) 4982b245cb2SAnirudh Venkataramanan { 4992b245cb2SAnirudh Venkataramanan return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 5002b245cb2SAnirudh Venkataramanan } 5012b245cb2SAnirudh Venkataramanan 5022b245cb2SAnirudh Venkataramanan /** 5032b245cb2SAnirudh Venkataramanan * ice_add_rx_frag - Add contents of Rx buffer to sk_buff 5042b245cb2SAnirudh Venkataramanan * @rx_buf: buffer containing page to add 5052b245cb2SAnirudh Venkataramanan * @rx_desc: descriptor containing length of buffer written by hardware 5062b245cb2SAnirudh Venkataramanan * @skb: sk_buf to place the data into 5072b245cb2SAnirudh Venkataramanan * 5082b245cb2SAnirudh Venkataramanan * This function will add the data contained in rx_buf->page to the skb. 5092b245cb2SAnirudh Venkataramanan * This is done either through a direct copy if the data in the buffer is 5102b245cb2SAnirudh Venkataramanan * less than the skb header size, otherwise it will just attach the page as 5112b245cb2SAnirudh Venkataramanan * a frag to the skb. 5122b245cb2SAnirudh Venkataramanan * 5132b245cb2SAnirudh Venkataramanan * The function will then update the page offset if necessary and return 5142b245cb2SAnirudh Venkataramanan * true if the buffer can be reused by the adapter. 5152b245cb2SAnirudh Venkataramanan */ 5162b245cb2SAnirudh Venkataramanan static bool ice_add_rx_frag(struct ice_rx_buf *rx_buf, 5172b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc, 5182b245cb2SAnirudh Venkataramanan struct sk_buff *skb) 5192b245cb2SAnirudh Venkataramanan { 5202b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE < 8192) 5212b245cb2SAnirudh Venkataramanan unsigned int truesize = ICE_RXBUF_2048; 5222b245cb2SAnirudh Venkataramanan #else 5232b245cb2SAnirudh Venkataramanan unsigned int last_offset = PAGE_SIZE - ICE_RXBUF_2048; 5242b245cb2SAnirudh Venkataramanan unsigned int truesize; 5252b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE < 8192) */ 5262b245cb2SAnirudh Venkataramanan 5272b245cb2SAnirudh Venkataramanan struct page *page; 5282b245cb2SAnirudh Venkataramanan unsigned int size; 5292b245cb2SAnirudh Venkataramanan 5302b245cb2SAnirudh Venkataramanan size = le16_to_cpu(rx_desc->wb.pkt_len) & 5312b245cb2SAnirudh Venkataramanan ICE_RX_FLX_DESC_PKT_LEN_M; 5322b245cb2SAnirudh Venkataramanan 5332b245cb2SAnirudh Venkataramanan page = rx_buf->page; 5342b245cb2SAnirudh Venkataramanan 5352b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE >= 8192) 5362b245cb2SAnirudh Venkataramanan truesize = ALIGN(size, L1_CACHE_BYTES); 5372b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE >= 8192) */ 5382b245cb2SAnirudh Venkataramanan 5392b245cb2SAnirudh Venkataramanan /* will the data fit in the skb we allocated? if so, just 5402b245cb2SAnirudh Venkataramanan * copy it as it is pretty small anyway 5412b245cb2SAnirudh Venkataramanan */ 5422b245cb2SAnirudh Venkataramanan if (size <= ICE_RX_HDR_SIZE && !skb_is_nonlinear(skb)) { 5432b245cb2SAnirudh Venkataramanan unsigned char *va = page_address(page) + rx_buf->page_offset; 5442b245cb2SAnirudh Venkataramanan 5452b245cb2SAnirudh Venkataramanan memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 5462b245cb2SAnirudh Venkataramanan 5472b245cb2SAnirudh Venkataramanan /* page is not reserved, we can reuse buffer as-is */ 5482b245cb2SAnirudh Venkataramanan if (likely(!ice_page_is_reserved(page))) 5492b245cb2SAnirudh Venkataramanan return true; 5502b245cb2SAnirudh Venkataramanan 5512b245cb2SAnirudh Venkataramanan /* this page cannot be reused so discard it */ 5522b245cb2SAnirudh Venkataramanan __free_pages(page, 0); 5532b245cb2SAnirudh Venkataramanan return false; 5542b245cb2SAnirudh Venkataramanan } 5552b245cb2SAnirudh Venkataramanan 5562b245cb2SAnirudh Venkataramanan skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 5572b245cb2SAnirudh Venkataramanan rx_buf->page_offset, size, truesize); 5582b245cb2SAnirudh Venkataramanan 5592b245cb2SAnirudh Venkataramanan /* avoid re-using remote pages */ 5602b245cb2SAnirudh Venkataramanan if (unlikely(ice_page_is_reserved(page))) 5612b245cb2SAnirudh Venkataramanan return false; 5622b245cb2SAnirudh Venkataramanan 5632b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE < 8192) 5642b245cb2SAnirudh Venkataramanan /* if we are only owner of page we can reuse it */ 5652b245cb2SAnirudh Venkataramanan if (unlikely(page_count(page) != 1)) 5662b245cb2SAnirudh Venkataramanan return false; 5672b245cb2SAnirudh Venkataramanan 5682b245cb2SAnirudh Venkataramanan /* flip page offset to other buffer */ 5692b245cb2SAnirudh Venkataramanan rx_buf->page_offset ^= truesize; 5702b245cb2SAnirudh Venkataramanan #else 5712b245cb2SAnirudh Venkataramanan /* move offset up to the next cache line */ 5722b245cb2SAnirudh Venkataramanan rx_buf->page_offset += truesize; 5732b245cb2SAnirudh Venkataramanan 5742b245cb2SAnirudh Venkataramanan if (rx_buf->page_offset > last_offset) 5752b245cb2SAnirudh Venkataramanan return false; 5762b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE < 8192) */ 5772b245cb2SAnirudh Venkataramanan 5782b245cb2SAnirudh Venkataramanan /* Even if we own the page, we are not allowed to use atomic_set() 5792b245cb2SAnirudh Venkataramanan * This would break get_page_unless_zero() users. 5802b245cb2SAnirudh Venkataramanan */ 5812b245cb2SAnirudh Venkataramanan get_page(rx_buf->page); 5822b245cb2SAnirudh Venkataramanan 5832b245cb2SAnirudh Venkataramanan return true; 5842b245cb2SAnirudh Venkataramanan } 5852b245cb2SAnirudh Venkataramanan 5862b245cb2SAnirudh Venkataramanan /** 5872b245cb2SAnirudh Venkataramanan * ice_reuse_rx_page - page flip buffer and store it back on the ring 5882b245cb2SAnirudh Venkataramanan * @rx_ring: rx descriptor ring to store buffers on 5892b245cb2SAnirudh Venkataramanan * @old_buf: donor buffer to have page reused 5902b245cb2SAnirudh Venkataramanan * 5912b245cb2SAnirudh Venkataramanan * Synchronizes page for reuse by the adapter 5922b245cb2SAnirudh Venkataramanan */ 5932b245cb2SAnirudh Venkataramanan static void ice_reuse_rx_page(struct ice_ring *rx_ring, 5942b245cb2SAnirudh Venkataramanan struct ice_rx_buf *old_buf) 5952b245cb2SAnirudh Venkataramanan { 5962b245cb2SAnirudh Venkataramanan u16 nta = rx_ring->next_to_alloc; 5972b245cb2SAnirudh Venkataramanan struct ice_rx_buf *new_buf; 5982b245cb2SAnirudh Venkataramanan 5992b245cb2SAnirudh Venkataramanan new_buf = &rx_ring->rx_buf[nta]; 6002b245cb2SAnirudh Venkataramanan 6012b245cb2SAnirudh Venkataramanan /* update, and store next to alloc */ 6022b245cb2SAnirudh Venkataramanan nta++; 6032b245cb2SAnirudh Venkataramanan rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 6042b245cb2SAnirudh Venkataramanan 6052b245cb2SAnirudh Venkataramanan /* transfer page from old buffer to new buffer */ 6062b245cb2SAnirudh Venkataramanan *new_buf = *old_buf; 6072b245cb2SAnirudh Venkataramanan } 6082b245cb2SAnirudh Venkataramanan 6092b245cb2SAnirudh Venkataramanan /** 6102b245cb2SAnirudh Venkataramanan * ice_fetch_rx_buf - Allocate skb and populate it 6112b245cb2SAnirudh Venkataramanan * @rx_ring: rx descriptor ring to transact packets on 6122b245cb2SAnirudh Venkataramanan * @rx_desc: descriptor containing info written by hardware 6132b245cb2SAnirudh Venkataramanan * 6142b245cb2SAnirudh Venkataramanan * This function allocates an skb on the fly, and populates it with the page 6152b245cb2SAnirudh Venkataramanan * data from the current receive descriptor, taking care to set up the skb 6162b245cb2SAnirudh Venkataramanan * correctly, as well as handling calling the page recycle function if 6172b245cb2SAnirudh Venkataramanan * necessary. 6182b245cb2SAnirudh Venkataramanan */ 6192b245cb2SAnirudh Venkataramanan static struct sk_buff *ice_fetch_rx_buf(struct ice_ring *rx_ring, 6202b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc) 6212b245cb2SAnirudh Venkataramanan { 6222b245cb2SAnirudh Venkataramanan struct ice_rx_buf *rx_buf; 6232b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 6242b245cb2SAnirudh Venkataramanan struct page *page; 6252b245cb2SAnirudh Venkataramanan 6262b245cb2SAnirudh Venkataramanan rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean]; 6272b245cb2SAnirudh Venkataramanan page = rx_buf->page; 6282b245cb2SAnirudh Venkataramanan prefetchw(page); 6292b245cb2SAnirudh Venkataramanan 6302b245cb2SAnirudh Venkataramanan skb = rx_buf->skb; 6312b245cb2SAnirudh Venkataramanan 6322b245cb2SAnirudh Venkataramanan if (likely(!skb)) { 6332b245cb2SAnirudh Venkataramanan u8 *page_addr = page_address(page) + rx_buf->page_offset; 6342b245cb2SAnirudh Venkataramanan 6352b245cb2SAnirudh Venkataramanan /* prefetch first cache line of first page */ 6362b245cb2SAnirudh Venkataramanan prefetch(page_addr); 6372b245cb2SAnirudh Venkataramanan #if L1_CACHE_BYTES < 128 6382b245cb2SAnirudh Venkataramanan prefetch((void *)(page_addr + L1_CACHE_BYTES)); 6392b245cb2SAnirudh Venkataramanan #endif /* L1_CACHE_BYTES */ 6402b245cb2SAnirudh Venkataramanan 6412b245cb2SAnirudh Venkataramanan /* allocate a skb to store the frags */ 6422b245cb2SAnirudh Venkataramanan skb = __napi_alloc_skb(&rx_ring->q_vector->napi, 6432b245cb2SAnirudh Venkataramanan ICE_RX_HDR_SIZE, 6442b245cb2SAnirudh Venkataramanan GFP_ATOMIC | __GFP_NOWARN); 6452b245cb2SAnirudh Venkataramanan if (unlikely(!skb)) { 6462b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_buf_failed++; 6472b245cb2SAnirudh Venkataramanan return NULL; 6482b245cb2SAnirudh Venkataramanan } 6492b245cb2SAnirudh Venkataramanan 6502b245cb2SAnirudh Venkataramanan /* we will be copying header into skb->data in 6512b245cb2SAnirudh Venkataramanan * pskb_may_pull so it is in our interest to prefetch 6522b245cb2SAnirudh Venkataramanan * it now to avoid a possible cache miss 6532b245cb2SAnirudh Venkataramanan */ 6542b245cb2SAnirudh Venkataramanan prefetchw(skb->data); 6552b245cb2SAnirudh Venkataramanan 6562b245cb2SAnirudh Venkataramanan skb_record_rx_queue(skb, rx_ring->q_index); 6572b245cb2SAnirudh Venkataramanan } else { 6582b245cb2SAnirudh Venkataramanan /* we are reusing so sync this buffer for CPU use */ 6592b245cb2SAnirudh Venkataramanan dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 6602b245cb2SAnirudh Venkataramanan rx_buf->page_offset, 6612b245cb2SAnirudh Venkataramanan ICE_RXBUF_2048, 6622b245cb2SAnirudh Venkataramanan DMA_FROM_DEVICE); 6632b245cb2SAnirudh Venkataramanan 6642b245cb2SAnirudh Venkataramanan rx_buf->skb = NULL; 6652b245cb2SAnirudh Venkataramanan } 6662b245cb2SAnirudh Venkataramanan 6672b245cb2SAnirudh Venkataramanan /* pull page into skb */ 6682b245cb2SAnirudh Venkataramanan if (ice_add_rx_frag(rx_buf, rx_desc, skb)) { 6692b245cb2SAnirudh Venkataramanan /* hand second half of page back to the ring */ 6702b245cb2SAnirudh Venkataramanan ice_reuse_rx_page(rx_ring, rx_buf); 6712b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 6722b245cb2SAnirudh Venkataramanan } else { 6732b245cb2SAnirudh Venkataramanan /* we are not reusing the buffer so unmap it */ 6742b245cb2SAnirudh Venkataramanan dma_unmap_page(rx_ring->dev, rx_buf->dma, PAGE_SIZE, 6752b245cb2SAnirudh Venkataramanan DMA_FROM_DEVICE); 6762b245cb2SAnirudh Venkataramanan } 6772b245cb2SAnirudh Venkataramanan 6782b245cb2SAnirudh Venkataramanan /* clear contents of buffer_info */ 6792b245cb2SAnirudh Venkataramanan rx_buf->page = NULL; 6802b245cb2SAnirudh Venkataramanan 6812b245cb2SAnirudh Venkataramanan return skb; 6822b245cb2SAnirudh Venkataramanan } 6832b245cb2SAnirudh Venkataramanan 6842b245cb2SAnirudh Venkataramanan /** 6852b245cb2SAnirudh Venkataramanan * ice_pull_tail - ice specific version of skb_pull_tail 6862b245cb2SAnirudh Venkataramanan * @skb: pointer to current skb being adjusted 6872b245cb2SAnirudh Venkataramanan * 6882b245cb2SAnirudh Venkataramanan * This function is an ice specific version of __pskb_pull_tail. The 6892b245cb2SAnirudh Venkataramanan * main difference between this version and the original function is that 6902b245cb2SAnirudh Venkataramanan * this function can make several assumptions about the state of things 6912b245cb2SAnirudh Venkataramanan * that allow for significant optimizations versus the standard function. 6922b245cb2SAnirudh Venkataramanan * As a result we can do things like drop a frag and maintain an accurate 6932b245cb2SAnirudh Venkataramanan * truesize for the skb. 6942b245cb2SAnirudh Venkataramanan */ 6952b245cb2SAnirudh Venkataramanan static void ice_pull_tail(struct sk_buff *skb) 6962b245cb2SAnirudh Venkataramanan { 6972b245cb2SAnirudh Venkataramanan struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 6982b245cb2SAnirudh Venkataramanan unsigned int pull_len; 6992b245cb2SAnirudh Venkataramanan unsigned char *va; 7002b245cb2SAnirudh Venkataramanan 7012b245cb2SAnirudh Venkataramanan /* it is valid to use page_address instead of kmap since we are 7022b245cb2SAnirudh Venkataramanan * working with pages allocated out of the lomem pool per 7032b245cb2SAnirudh Venkataramanan * alloc_page(GFP_ATOMIC) 7042b245cb2SAnirudh Venkataramanan */ 7052b245cb2SAnirudh Venkataramanan va = skb_frag_address(frag); 7062b245cb2SAnirudh Venkataramanan 7072b245cb2SAnirudh Venkataramanan /* we need the header to contain the greater of either ETH_HLEN or 7082b245cb2SAnirudh Venkataramanan * 60 bytes if the skb->len is less than 60 for skb_pad. 7092b245cb2SAnirudh Venkataramanan */ 7102b245cb2SAnirudh Venkataramanan pull_len = eth_get_headlen(va, ICE_RX_HDR_SIZE); 7112b245cb2SAnirudh Venkataramanan 7122b245cb2SAnirudh Venkataramanan /* align pull length to size of long to optimize memcpy performance */ 7132b245cb2SAnirudh Venkataramanan skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 7142b245cb2SAnirudh Venkataramanan 7152b245cb2SAnirudh Venkataramanan /* update all of the pointers */ 7162b245cb2SAnirudh Venkataramanan skb_frag_size_sub(frag, pull_len); 7172b245cb2SAnirudh Venkataramanan frag->page_offset += pull_len; 7182b245cb2SAnirudh Venkataramanan skb->data_len -= pull_len; 7192b245cb2SAnirudh Venkataramanan skb->tail += pull_len; 7202b245cb2SAnirudh Venkataramanan } 7212b245cb2SAnirudh Venkataramanan 7222b245cb2SAnirudh Venkataramanan /** 7232b245cb2SAnirudh Venkataramanan * ice_cleanup_headers - Correct empty headers 7242b245cb2SAnirudh Venkataramanan * @skb: pointer to current skb being fixed 7252b245cb2SAnirudh Venkataramanan * 7262b245cb2SAnirudh Venkataramanan * Also address the case where we are pulling data in on pages only 7272b245cb2SAnirudh Venkataramanan * and as such no data is present in the skb header. 7282b245cb2SAnirudh Venkataramanan * 7292b245cb2SAnirudh Venkataramanan * In addition if skb is not at least 60 bytes we need to pad it so that 7302b245cb2SAnirudh Venkataramanan * it is large enough to qualify as a valid Ethernet frame. 7312b245cb2SAnirudh Venkataramanan * 7322b245cb2SAnirudh Venkataramanan * Returns true if an error was encountered and skb was freed. 7332b245cb2SAnirudh Venkataramanan */ 7342b245cb2SAnirudh Venkataramanan static bool ice_cleanup_headers(struct sk_buff *skb) 7352b245cb2SAnirudh Venkataramanan { 7362b245cb2SAnirudh Venkataramanan /* place header in linear portion of buffer */ 7372b245cb2SAnirudh Venkataramanan if (skb_is_nonlinear(skb)) 7382b245cb2SAnirudh Venkataramanan ice_pull_tail(skb); 7392b245cb2SAnirudh Venkataramanan 7402b245cb2SAnirudh Venkataramanan /* if eth_skb_pad returns an error the skb was freed */ 7412b245cb2SAnirudh Venkataramanan if (eth_skb_pad(skb)) 7422b245cb2SAnirudh Venkataramanan return true; 7432b245cb2SAnirudh Venkataramanan 7442b245cb2SAnirudh Venkataramanan return false; 7452b245cb2SAnirudh Venkataramanan } 7462b245cb2SAnirudh Venkataramanan 7472b245cb2SAnirudh Venkataramanan /** 7482b245cb2SAnirudh Venkataramanan * ice_test_staterr - tests bits in Rx descriptor status and error fields 7492b245cb2SAnirudh Venkataramanan * @rx_desc: pointer to receive descriptor (in le64 format) 7502b245cb2SAnirudh Venkataramanan * @stat_err_bits: value to mask 7512b245cb2SAnirudh Venkataramanan * 7522b245cb2SAnirudh Venkataramanan * This function does some fast chicanery in order to return the 7532b245cb2SAnirudh Venkataramanan * value of the mask which is really only used for boolean tests. 7542b245cb2SAnirudh Venkataramanan * The status_error_len doesn't need to be shifted because it begins 7552b245cb2SAnirudh Venkataramanan * at offset zero. 7562b245cb2SAnirudh Venkataramanan */ 7572b245cb2SAnirudh Venkataramanan static bool ice_test_staterr(union ice_32b_rx_flex_desc *rx_desc, 7582b245cb2SAnirudh Venkataramanan const u16 stat_err_bits) 7592b245cb2SAnirudh Venkataramanan { 7602b245cb2SAnirudh Venkataramanan return !!(rx_desc->wb.status_error0 & 7612b245cb2SAnirudh Venkataramanan cpu_to_le16(stat_err_bits)); 7622b245cb2SAnirudh Venkataramanan } 7632b245cb2SAnirudh Venkataramanan 7642b245cb2SAnirudh Venkataramanan /** 7652b245cb2SAnirudh Venkataramanan * ice_is_non_eop - process handling of non-EOP buffers 7662b245cb2SAnirudh Venkataramanan * @rx_ring: Rx ring being processed 7672b245cb2SAnirudh Venkataramanan * @rx_desc: Rx descriptor for current buffer 7682b245cb2SAnirudh Venkataramanan * @skb: Current socket buffer containing buffer in progress 7692b245cb2SAnirudh Venkataramanan * 7702b245cb2SAnirudh Venkataramanan * This function updates next to clean. If the buffer is an EOP buffer 7712b245cb2SAnirudh Venkataramanan * this function exits returning false, otherwise it will place the 7722b245cb2SAnirudh Venkataramanan * sk_buff in the next buffer to be chained and return true indicating 7732b245cb2SAnirudh Venkataramanan * that this is in fact a non-EOP buffer. 7742b245cb2SAnirudh Venkataramanan */ 7752b245cb2SAnirudh Venkataramanan static bool ice_is_non_eop(struct ice_ring *rx_ring, 7762b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc, 7772b245cb2SAnirudh Venkataramanan struct sk_buff *skb) 7782b245cb2SAnirudh Venkataramanan { 7792b245cb2SAnirudh Venkataramanan u32 ntc = rx_ring->next_to_clean + 1; 7802b245cb2SAnirudh Venkataramanan 7812b245cb2SAnirudh Venkataramanan /* fetch, update, and store next to clean */ 7822b245cb2SAnirudh Venkataramanan ntc = (ntc < rx_ring->count) ? ntc : 0; 7832b245cb2SAnirudh Venkataramanan rx_ring->next_to_clean = ntc; 7842b245cb2SAnirudh Venkataramanan 7852b245cb2SAnirudh Venkataramanan prefetch(ICE_RX_DESC(rx_ring, ntc)); 7862b245cb2SAnirudh Venkataramanan 7872b245cb2SAnirudh Venkataramanan /* if we are the last buffer then there is nothing else to do */ 7882b245cb2SAnirudh Venkataramanan #define ICE_RXD_EOF BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S) 7892b245cb2SAnirudh Venkataramanan if (likely(ice_test_staterr(rx_desc, ICE_RXD_EOF))) 7902b245cb2SAnirudh Venkataramanan return false; 7912b245cb2SAnirudh Venkataramanan 7922b245cb2SAnirudh Venkataramanan /* place skb in next buffer to be received */ 7932b245cb2SAnirudh Venkataramanan rx_ring->rx_buf[ntc].skb = skb; 7942b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.non_eop_descs++; 7952b245cb2SAnirudh Venkataramanan 7962b245cb2SAnirudh Venkataramanan return true; 7972b245cb2SAnirudh Venkataramanan } 7982b245cb2SAnirudh Venkataramanan 7992b245cb2SAnirudh Venkataramanan /** 8002b245cb2SAnirudh Venkataramanan * ice_receive_skb - Send a completed packet up the stack 8012b245cb2SAnirudh Venkataramanan * @rx_ring: rx ring in play 8022b245cb2SAnirudh Venkataramanan * @skb: packet to send up 8032b245cb2SAnirudh Venkataramanan * @vlan_tag: vlan tag for packet 8042b245cb2SAnirudh Venkataramanan * 8052b245cb2SAnirudh Venkataramanan * This function sends the completed packet (via. skb) up the stack using 8062b245cb2SAnirudh Venkataramanan * gro receive functions (with/without vlan tag) 8072b245cb2SAnirudh Venkataramanan */ 8082b245cb2SAnirudh Venkataramanan static void ice_receive_skb(struct ice_ring *rx_ring, struct sk_buff *skb, 8092b245cb2SAnirudh Venkataramanan u16 vlan_tag) 8102b245cb2SAnirudh Venkataramanan { 8112b245cb2SAnirudh Venkataramanan if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 8122b245cb2SAnirudh Venkataramanan (vlan_tag & VLAN_VID_MASK)) { 8132b245cb2SAnirudh Venkataramanan __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 8142b245cb2SAnirudh Venkataramanan } 8152b245cb2SAnirudh Venkataramanan napi_gro_receive(&rx_ring->q_vector->napi, skb); 8162b245cb2SAnirudh Venkataramanan } 8172b245cb2SAnirudh Venkataramanan 8182b245cb2SAnirudh Venkataramanan /** 8192b245cb2SAnirudh Venkataramanan * ice_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 8202b245cb2SAnirudh Venkataramanan * @rx_ring: rx descriptor ring to transact packets on 8212b245cb2SAnirudh Venkataramanan * @budget: Total limit on number of packets to process 8222b245cb2SAnirudh Venkataramanan * 8232b245cb2SAnirudh Venkataramanan * This function provides a "bounce buffer" approach to Rx interrupt 8242b245cb2SAnirudh Venkataramanan * processing. The advantage to this is that on systems that have 8252b245cb2SAnirudh Venkataramanan * expensive overhead for IOMMU access this provides a means of avoiding 8262b245cb2SAnirudh Venkataramanan * it by maintaining the mapping of the page to the system. 8272b245cb2SAnirudh Venkataramanan * 8282b245cb2SAnirudh Venkataramanan * Returns amount of work completed 8292b245cb2SAnirudh Venkataramanan */ 8302b245cb2SAnirudh Venkataramanan static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) 8312b245cb2SAnirudh Venkataramanan { 8322b245cb2SAnirudh Venkataramanan unsigned int total_rx_bytes = 0, total_rx_pkts = 0; 8332b245cb2SAnirudh Venkataramanan u16 cleaned_count = ICE_DESC_UNUSED(rx_ring); 8342b245cb2SAnirudh Venkataramanan bool failure = false; 8352b245cb2SAnirudh Venkataramanan 8362b245cb2SAnirudh Venkataramanan /* start the loop to process RX packets bounded by 'budget' */ 8372b245cb2SAnirudh Venkataramanan while (likely(total_rx_pkts < (unsigned int)budget)) { 8382b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 8392b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 8402b245cb2SAnirudh Venkataramanan u16 stat_err_bits; 8412b245cb2SAnirudh Venkataramanan u16 vlan_tag = 0; 8422b245cb2SAnirudh Venkataramanan 8432b245cb2SAnirudh Venkataramanan /* return some buffers to hardware, one at a time is too slow */ 8442b245cb2SAnirudh Venkataramanan if (cleaned_count >= ICE_RX_BUF_WRITE) { 8452b245cb2SAnirudh Venkataramanan failure = failure || 8462b245cb2SAnirudh Venkataramanan ice_alloc_rx_bufs(rx_ring, cleaned_count); 8472b245cb2SAnirudh Venkataramanan cleaned_count = 0; 8482b245cb2SAnirudh Venkataramanan } 8492b245cb2SAnirudh Venkataramanan 8502b245cb2SAnirudh Venkataramanan /* get the RX desc from RX ring based on 'next_to_clean' */ 8512b245cb2SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); 8522b245cb2SAnirudh Venkataramanan 8532b245cb2SAnirudh Venkataramanan /* status_error_len will always be zero for unused descriptors 8542b245cb2SAnirudh Venkataramanan * because it's cleared in cleanup, and overlaps with hdr_addr 8552b245cb2SAnirudh Venkataramanan * which is always zero because packet split isn't used, if the 8562b245cb2SAnirudh Venkataramanan * hardware wrote DD then it will be non-zero 8572b245cb2SAnirudh Venkataramanan */ 8582b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 8592b245cb2SAnirudh Venkataramanan if (!ice_test_staterr(rx_desc, stat_err_bits)) 8602b245cb2SAnirudh Venkataramanan break; 8612b245cb2SAnirudh Venkataramanan 8622b245cb2SAnirudh Venkataramanan /* This memory barrier is needed to keep us from reading 8632b245cb2SAnirudh Venkataramanan * any other fields out of the rx_desc until we know the 8642b245cb2SAnirudh Venkataramanan * DD bit is set. 8652b245cb2SAnirudh Venkataramanan */ 8662b245cb2SAnirudh Venkataramanan dma_rmb(); 8672b245cb2SAnirudh Venkataramanan 8682b245cb2SAnirudh Venkataramanan /* allocate (if needed) and populate skb */ 8692b245cb2SAnirudh Venkataramanan skb = ice_fetch_rx_buf(rx_ring, rx_desc); 8702b245cb2SAnirudh Venkataramanan if (!skb) 8712b245cb2SAnirudh Venkataramanan break; 8722b245cb2SAnirudh Venkataramanan 8732b245cb2SAnirudh Venkataramanan cleaned_count++; 8742b245cb2SAnirudh Venkataramanan 8752b245cb2SAnirudh Venkataramanan /* skip if it is NOP desc */ 8762b245cb2SAnirudh Venkataramanan if (ice_is_non_eop(rx_ring, rx_desc, skb)) 8772b245cb2SAnirudh Venkataramanan continue; 8782b245cb2SAnirudh Venkataramanan 8792b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S); 8802b245cb2SAnirudh Venkataramanan if (unlikely(ice_test_staterr(rx_desc, stat_err_bits))) { 8812b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 8822b245cb2SAnirudh Venkataramanan continue; 8832b245cb2SAnirudh Venkataramanan } 8842b245cb2SAnirudh Venkataramanan 8852b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S); 8862b245cb2SAnirudh Venkataramanan if (ice_test_staterr(rx_desc, stat_err_bits)) 8872b245cb2SAnirudh Venkataramanan vlan_tag = le16_to_cpu(rx_desc->wb.l2tag1); 8882b245cb2SAnirudh Venkataramanan 8892b245cb2SAnirudh Venkataramanan /* correct empty headers and pad skb if needed (to make valid 8902b245cb2SAnirudh Venkataramanan * ethernet frame 8912b245cb2SAnirudh Venkataramanan */ 8922b245cb2SAnirudh Venkataramanan if (ice_cleanup_headers(skb)) { 8932b245cb2SAnirudh Venkataramanan skb = NULL; 8942b245cb2SAnirudh Venkataramanan continue; 8952b245cb2SAnirudh Venkataramanan } 8962b245cb2SAnirudh Venkataramanan 8972b245cb2SAnirudh Venkataramanan /* probably a little skewed due to removing CRC */ 8982b245cb2SAnirudh Venkataramanan total_rx_bytes += skb->len; 8992b245cb2SAnirudh Venkataramanan 9002b245cb2SAnirudh Venkataramanan /* send completed skb up the stack */ 9012b245cb2SAnirudh Venkataramanan ice_receive_skb(rx_ring, skb, vlan_tag); 9022b245cb2SAnirudh Venkataramanan 9032b245cb2SAnirudh Venkataramanan /* update budget accounting */ 9042b245cb2SAnirudh Venkataramanan total_rx_pkts++; 9052b245cb2SAnirudh Venkataramanan } 9062b245cb2SAnirudh Venkataramanan 9072b245cb2SAnirudh Venkataramanan /* update queue and vector specific stats */ 9082b245cb2SAnirudh Venkataramanan u64_stats_update_begin(&rx_ring->syncp); 9092b245cb2SAnirudh Venkataramanan rx_ring->stats.pkts += total_rx_pkts; 9102b245cb2SAnirudh Venkataramanan rx_ring->stats.bytes += total_rx_bytes; 9112b245cb2SAnirudh Venkataramanan u64_stats_update_end(&rx_ring->syncp); 9122b245cb2SAnirudh Venkataramanan rx_ring->q_vector->rx.total_pkts += total_rx_pkts; 9132b245cb2SAnirudh Venkataramanan rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 9142b245cb2SAnirudh Venkataramanan 9152b245cb2SAnirudh Venkataramanan /* guarantee a trip back through this routine if there was a failure */ 9162b245cb2SAnirudh Venkataramanan return failure ? budget : (int)total_rx_pkts; 9172b245cb2SAnirudh Venkataramanan } 9182b245cb2SAnirudh Venkataramanan 9192b245cb2SAnirudh Venkataramanan /** 9202b245cb2SAnirudh Venkataramanan * ice_napi_poll - NAPI polling Rx/Tx cleanup routine 9212b245cb2SAnirudh Venkataramanan * @napi: napi struct with our devices info in it 9222b245cb2SAnirudh Venkataramanan * @budget: amount of work driver is allowed to do this pass, in packets 9232b245cb2SAnirudh Venkataramanan * 9242b245cb2SAnirudh Venkataramanan * This function will clean all queues associated with a q_vector. 9252b245cb2SAnirudh Venkataramanan * 9262b245cb2SAnirudh Venkataramanan * Returns the amount of work done 9272b245cb2SAnirudh Venkataramanan */ 9282b245cb2SAnirudh Venkataramanan int ice_napi_poll(struct napi_struct *napi, int budget) 9292b245cb2SAnirudh Venkataramanan { 9302b245cb2SAnirudh Venkataramanan struct ice_q_vector *q_vector = 9312b245cb2SAnirudh Venkataramanan container_of(napi, struct ice_q_vector, napi); 9322b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = q_vector->vsi; 9332b245cb2SAnirudh Venkataramanan struct ice_pf *pf = vsi->back; 9342b245cb2SAnirudh Venkataramanan bool clean_complete = true; 9352b245cb2SAnirudh Venkataramanan int budget_per_ring = 0; 9362b245cb2SAnirudh Venkataramanan struct ice_ring *ring; 9372b245cb2SAnirudh Venkataramanan int work_done = 0; 9382b245cb2SAnirudh Venkataramanan 9392b245cb2SAnirudh Venkataramanan /* Since the actual Tx work is minimal, we can give the Tx a larger 9402b245cb2SAnirudh Venkataramanan * budget and be more aggressive about cleaning up the Tx descriptors. 9412b245cb2SAnirudh Venkataramanan */ 9422b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->tx) 9432b245cb2SAnirudh Venkataramanan if (!ice_clean_tx_irq(vsi, ring, budget)) 9442b245cb2SAnirudh Venkataramanan clean_complete = false; 9452b245cb2SAnirudh Venkataramanan 9462b245cb2SAnirudh Venkataramanan /* Handle case where we are called by netpoll with a budget of 0 */ 9472b245cb2SAnirudh Venkataramanan if (budget <= 0) 9482b245cb2SAnirudh Venkataramanan return budget; 9492b245cb2SAnirudh Venkataramanan 9502b245cb2SAnirudh Venkataramanan /* We attempt to distribute budget to each Rx queue fairly, but don't 9512b245cb2SAnirudh Venkataramanan * allow the budget to go below 1 because that would exit polling early. 9522b245cb2SAnirudh Venkataramanan */ 9532b245cb2SAnirudh Venkataramanan if (q_vector->num_ring_rx) 9542b245cb2SAnirudh Venkataramanan budget_per_ring = max(budget / q_vector->num_ring_rx, 1); 9552b245cb2SAnirudh Venkataramanan 9562b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->rx) { 9572b245cb2SAnirudh Venkataramanan int cleaned; 9582b245cb2SAnirudh Venkataramanan 9592b245cb2SAnirudh Venkataramanan cleaned = ice_clean_rx_irq(ring, budget_per_ring); 9602b245cb2SAnirudh Venkataramanan work_done += cleaned; 9612b245cb2SAnirudh Venkataramanan /* if we clean as many as budgeted, we must not be done */ 9622b245cb2SAnirudh Venkataramanan if (cleaned >= budget_per_ring) 9632b245cb2SAnirudh Venkataramanan clean_complete = false; 9642b245cb2SAnirudh Venkataramanan } 9652b245cb2SAnirudh Venkataramanan 9662b245cb2SAnirudh Venkataramanan /* If work not completed, return budget and polling will return */ 9672b245cb2SAnirudh Venkataramanan if (!clean_complete) 9682b245cb2SAnirudh Venkataramanan return budget; 9692b245cb2SAnirudh Venkataramanan 9702b245cb2SAnirudh Venkataramanan /* Work is done so exit the polling mode and re-enable the interrupt */ 9712b245cb2SAnirudh Venkataramanan napi_complete_done(napi, work_done); 9722b245cb2SAnirudh Venkataramanan if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 9732b245cb2SAnirudh Venkataramanan ice_irq_dynamic_ena(&vsi->back->hw, vsi, q_vector); 9742b245cb2SAnirudh Venkataramanan return 0; 9752b245cb2SAnirudh Venkataramanan } 9762b245cb2SAnirudh Venkataramanan 9772b245cb2SAnirudh Venkataramanan /* helper function for building cmd/type/offset */ 9782b245cb2SAnirudh Venkataramanan static __le64 9792b245cb2SAnirudh Venkataramanan build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) 9802b245cb2SAnirudh Venkataramanan { 9812b245cb2SAnirudh Venkataramanan return cpu_to_le64(ICE_TX_DESC_DTYPE_DATA | 9822b245cb2SAnirudh Venkataramanan (td_cmd << ICE_TXD_QW1_CMD_S) | 9832b245cb2SAnirudh Venkataramanan (td_offset << ICE_TXD_QW1_OFFSET_S) | 9842b245cb2SAnirudh Venkataramanan ((u64)size << ICE_TXD_QW1_TX_BUF_SZ_S) | 9852b245cb2SAnirudh Venkataramanan (td_tag << ICE_TXD_QW1_L2TAG1_S)); 9862b245cb2SAnirudh Venkataramanan } 9872b245cb2SAnirudh Venkataramanan 9882b245cb2SAnirudh Venkataramanan /** 9892b245cb2SAnirudh Venkataramanan * __ice_maybe_stop_tx - 2nd level check for tx stop conditions 9902b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 9912b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 9922b245cb2SAnirudh Venkataramanan * 9932b245cb2SAnirudh Venkataramanan * Returns -EBUSY if a stop is needed, else 0 9942b245cb2SAnirudh Venkataramanan */ 9952b245cb2SAnirudh Venkataramanan static int __ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 9962b245cb2SAnirudh Venkataramanan { 9972b245cb2SAnirudh Venkataramanan netif_stop_subqueue(tx_ring->netdev, tx_ring->q_index); 9982b245cb2SAnirudh Venkataramanan /* Memory barrier before checking head and tail */ 9992b245cb2SAnirudh Venkataramanan smp_mb(); 10002b245cb2SAnirudh Venkataramanan 10012b245cb2SAnirudh Venkataramanan /* Check again in a case another CPU has just made room available. */ 10022b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) < size)) 10032b245cb2SAnirudh Venkataramanan return -EBUSY; 10042b245cb2SAnirudh Venkataramanan 10052b245cb2SAnirudh Venkataramanan /* A reprieve! - use start_subqueue because it doesn't call schedule */ 10062b245cb2SAnirudh Venkataramanan netif_start_subqueue(tx_ring->netdev, tx_ring->q_index); 10072b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 10082b245cb2SAnirudh Venkataramanan return 0; 10092b245cb2SAnirudh Venkataramanan } 10102b245cb2SAnirudh Venkataramanan 10112b245cb2SAnirudh Venkataramanan /** 10122b245cb2SAnirudh Venkataramanan * ice_maybe_stop_tx - 1st level check for tx stop conditions 10132b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 10142b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 10152b245cb2SAnirudh Venkataramanan * 10162b245cb2SAnirudh Venkataramanan * Returns 0 if stop is not needed 10172b245cb2SAnirudh Venkataramanan */ 10182b245cb2SAnirudh Venkataramanan static int ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 10192b245cb2SAnirudh Venkataramanan { 10202b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) >= size)) 10212b245cb2SAnirudh Venkataramanan return 0; 10222b245cb2SAnirudh Venkataramanan return __ice_maybe_stop_tx(tx_ring, size); 10232b245cb2SAnirudh Venkataramanan } 10242b245cb2SAnirudh Venkataramanan 10252b245cb2SAnirudh Venkataramanan /** 10262b245cb2SAnirudh Venkataramanan * ice_tx_map - Build the Tx descriptor 10272b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 10282b245cb2SAnirudh Venkataramanan * @first: first buffer info buffer to use 10292b245cb2SAnirudh Venkataramanan * 10302b245cb2SAnirudh Venkataramanan * This function loops over the skb data pointed to by *first 10312b245cb2SAnirudh Venkataramanan * and gets a physical address for each memory location and programs 10322b245cb2SAnirudh Venkataramanan * it and the length into the transmit descriptor. 10332b245cb2SAnirudh Venkataramanan */ 10342b245cb2SAnirudh Venkataramanan static void ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first) 10352b245cb2SAnirudh Venkataramanan { 10362b245cb2SAnirudh Venkataramanan u64 td_offset = 0, td_tag = 0, td_cmd = 0; 10372b245cb2SAnirudh Venkataramanan u16 i = tx_ring->next_to_use; 10382b245cb2SAnirudh Venkataramanan struct skb_frag_struct *frag; 10392b245cb2SAnirudh Venkataramanan unsigned int data_len, size; 10402b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 10412b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 10422b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 10432b245cb2SAnirudh Venkataramanan dma_addr_t dma; 10442b245cb2SAnirudh Venkataramanan 10452b245cb2SAnirudh Venkataramanan skb = first->skb; 10462b245cb2SAnirudh Venkataramanan 10472b245cb2SAnirudh Venkataramanan data_len = skb->data_len; 10482b245cb2SAnirudh Venkataramanan size = skb_headlen(skb); 10492b245cb2SAnirudh Venkataramanan 10502b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 10512b245cb2SAnirudh Venkataramanan 10522b245cb2SAnirudh Venkataramanan dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 10532b245cb2SAnirudh Venkataramanan 10542b245cb2SAnirudh Venkataramanan tx_buf = first; 10552b245cb2SAnirudh Venkataramanan 10562b245cb2SAnirudh Venkataramanan for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 10572b245cb2SAnirudh Venkataramanan unsigned int max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 10582b245cb2SAnirudh Venkataramanan 10592b245cb2SAnirudh Venkataramanan if (dma_mapping_error(tx_ring->dev, dma)) 10602b245cb2SAnirudh Venkataramanan goto dma_error; 10612b245cb2SAnirudh Venkataramanan 10622b245cb2SAnirudh Venkataramanan /* record length, and DMA address */ 10632b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, size); 10642b245cb2SAnirudh Venkataramanan dma_unmap_addr_set(tx_buf, dma, dma); 10652b245cb2SAnirudh Venkataramanan 10662b245cb2SAnirudh Venkataramanan /* align size to end of page */ 10672b245cb2SAnirudh Venkataramanan max_data += -dma & (ICE_MAX_READ_REQ_SIZE - 1); 10682b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 10692b245cb2SAnirudh Venkataramanan 10702b245cb2SAnirudh Venkataramanan /* account for data chunks larger than the hardware 10712b245cb2SAnirudh Venkataramanan * can handle 10722b245cb2SAnirudh Venkataramanan */ 10732b245cb2SAnirudh Venkataramanan while (unlikely(size > ICE_MAX_DATA_PER_TXD)) { 10742b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 10752b245cb2SAnirudh Venkataramanan build_ctob(td_cmd, td_offset, max_data, td_tag); 10762b245cb2SAnirudh Venkataramanan 10772b245cb2SAnirudh Venkataramanan tx_desc++; 10782b245cb2SAnirudh Venkataramanan i++; 10792b245cb2SAnirudh Venkataramanan 10802b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 10812b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 10822b245cb2SAnirudh Venkataramanan i = 0; 10832b245cb2SAnirudh Venkataramanan } 10842b245cb2SAnirudh Venkataramanan 10852b245cb2SAnirudh Venkataramanan dma += max_data; 10862b245cb2SAnirudh Venkataramanan size -= max_data; 10872b245cb2SAnirudh Venkataramanan 10882b245cb2SAnirudh Venkataramanan max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 10892b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 10902b245cb2SAnirudh Venkataramanan } 10912b245cb2SAnirudh Venkataramanan 10922b245cb2SAnirudh Venkataramanan if (likely(!data_len)) 10932b245cb2SAnirudh Venkataramanan break; 10942b245cb2SAnirudh Venkataramanan 10952b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 10962b245cb2SAnirudh Venkataramanan size, td_tag); 10972b245cb2SAnirudh Venkataramanan 10982b245cb2SAnirudh Venkataramanan tx_desc++; 10992b245cb2SAnirudh Venkataramanan i++; 11002b245cb2SAnirudh Venkataramanan 11012b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 11022b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 11032b245cb2SAnirudh Venkataramanan i = 0; 11042b245cb2SAnirudh Venkataramanan } 11052b245cb2SAnirudh Venkataramanan 11062b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag); 11072b245cb2SAnirudh Venkataramanan data_len -= size; 11082b245cb2SAnirudh Venkataramanan 11092b245cb2SAnirudh Venkataramanan dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 11102b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 11112b245cb2SAnirudh Venkataramanan 11122b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 11132b245cb2SAnirudh Venkataramanan } 11142b245cb2SAnirudh Venkataramanan 11152b245cb2SAnirudh Venkataramanan /* record bytecount for BQL */ 11162b245cb2SAnirudh Venkataramanan netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 11172b245cb2SAnirudh Venkataramanan 11182b245cb2SAnirudh Venkataramanan /* record SW timestamp if HW timestamp is not available */ 11192b245cb2SAnirudh Venkataramanan skb_tx_timestamp(first->skb); 11202b245cb2SAnirudh Venkataramanan 11212b245cb2SAnirudh Venkataramanan i++; 11222b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) 11232b245cb2SAnirudh Venkataramanan i = 0; 11242b245cb2SAnirudh Venkataramanan 11252b245cb2SAnirudh Venkataramanan /* write last descriptor with RS and EOP bits */ 11262b245cb2SAnirudh Venkataramanan td_cmd |= (u64)(ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS); 11272b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 11282b245cb2SAnirudh Venkataramanan build_ctob(td_cmd, td_offset, size, td_tag); 11292b245cb2SAnirudh Venkataramanan 11302b245cb2SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w know there 11312b245cb2SAnirudh Venkataramanan * are new descriptors to fetch. 11322b245cb2SAnirudh Venkataramanan * 11332b245cb2SAnirudh Venkataramanan * We also use this memory barrier to make certain all of the 11342b245cb2SAnirudh Venkataramanan * status bits have been updated before next_to_watch is written. 11352b245cb2SAnirudh Venkataramanan */ 11362b245cb2SAnirudh Venkataramanan wmb(); 11372b245cb2SAnirudh Venkataramanan 11382b245cb2SAnirudh Venkataramanan /* set next_to_watch value indicating a packet is present */ 11392b245cb2SAnirudh Venkataramanan first->next_to_watch = tx_desc; 11402b245cb2SAnirudh Venkataramanan 11412b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 11422b245cb2SAnirudh Venkataramanan 11432b245cb2SAnirudh Venkataramanan ice_maybe_stop_tx(tx_ring, DESC_NEEDED); 11442b245cb2SAnirudh Venkataramanan 11452b245cb2SAnirudh Venkataramanan /* notify HW of packet */ 11462b245cb2SAnirudh Venkataramanan if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 11472b245cb2SAnirudh Venkataramanan writel(i, tx_ring->tail); 11482b245cb2SAnirudh Venkataramanan 11492b245cb2SAnirudh Venkataramanan /* we need this if more than one processor can write to our tail 11502b245cb2SAnirudh Venkataramanan * at a time, it synchronizes IO on IA64/Altix systems 11512b245cb2SAnirudh Venkataramanan */ 11522b245cb2SAnirudh Venkataramanan mmiowb(); 11532b245cb2SAnirudh Venkataramanan } 11542b245cb2SAnirudh Venkataramanan 11552b245cb2SAnirudh Venkataramanan return; 11562b245cb2SAnirudh Venkataramanan 11572b245cb2SAnirudh Venkataramanan dma_error: 11582b245cb2SAnirudh Venkataramanan /* clear dma mappings for failed tx_buf map */ 11592b245cb2SAnirudh Venkataramanan for (;;) { 11602b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 11612b245cb2SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, tx_buf); 11622b245cb2SAnirudh Venkataramanan if (tx_buf == first) 11632b245cb2SAnirudh Venkataramanan break; 11642b245cb2SAnirudh Venkataramanan if (i == 0) 11652b245cb2SAnirudh Venkataramanan i = tx_ring->count; 11662b245cb2SAnirudh Venkataramanan i--; 11672b245cb2SAnirudh Venkataramanan } 11682b245cb2SAnirudh Venkataramanan 11692b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 11702b245cb2SAnirudh Venkataramanan } 11712b245cb2SAnirudh Venkataramanan 11722b245cb2SAnirudh Venkataramanan /** 11732b245cb2SAnirudh Venkataramanan * ice_txd_use_count - estimate the number of descriptors needed for Tx 11742b245cb2SAnirudh Venkataramanan * @size: transmit request size in bytes 11752b245cb2SAnirudh Venkataramanan * 11762b245cb2SAnirudh Venkataramanan * Due to hardware alignment restrictions (4K alignment), we need to 11772b245cb2SAnirudh Venkataramanan * assume that we can have no more than 12K of data per descriptor, even 11782b245cb2SAnirudh Venkataramanan * though each descriptor can take up to 16K - 1 bytes of aligned memory. 11792b245cb2SAnirudh Venkataramanan * Thus, we need to divide by 12K. But division is slow! Instead, 11802b245cb2SAnirudh Venkataramanan * we decompose the operation into shifts and one relatively cheap 11812b245cb2SAnirudh Venkataramanan * multiply operation. 11822b245cb2SAnirudh Venkataramanan * 11832b245cb2SAnirudh Venkataramanan * To divide by 12K, we first divide by 4K, then divide by 3: 11842b245cb2SAnirudh Venkataramanan * To divide by 4K, shift right by 12 bits 11852b245cb2SAnirudh Venkataramanan * To divide by 3, multiply by 85, then divide by 256 11862b245cb2SAnirudh Venkataramanan * (Divide by 256 is done by shifting right by 8 bits) 11872b245cb2SAnirudh Venkataramanan * Finally, we add one to round up. Because 256 isn't an exact multiple of 11882b245cb2SAnirudh Venkataramanan * 3, we'll underestimate near each multiple of 12K. This is actually more 11892b245cb2SAnirudh Venkataramanan * accurate as we have 4K - 1 of wiggle room that we can fit into the last 11902b245cb2SAnirudh Venkataramanan * segment. For our purposes this is accurate out to 1M which is orders of 11912b245cb2SAnirudh Venkataramanan * magnitude greater than our largest possible GSO size. 11922b245cb2SAnirudh Venkataramanan * 11932b245cb2SAnirudh Venkataramanan * This would then be implemented as: 11942b245cb2SAnirudh Venkataramanan * return (((size >> 12) * 85) >> 8) + 1; 11952b245cb2SAnirudh Venkataramanan * 11962b245cb2SAnirudh Venkataramanan * Since multiplication and division are commutative, we can reorder 11972b245cb2SAnirudh Venkataramanan * operations into: 11982b245cb2SAnirudh Venkataramanan * return ((size * 85) >> 20) + 1; 11992b245cb2SAnirudh Venkataramanan */ 12002b245cb2SAnirudh Venkataramanan static unsigned int ice_txd_use_count(unsigned int size) 12012b245cb2SAnirudh Venkataramanan { 12022b245cb2SAnirudh Venkataramanan return ((size * 85) >> 20) + 1; 12032b245cb2SAnirudh Venkataramanan } 12042b245cb2SAnirudh Venkataramanan 12052b245cb2SAnirudh Venkataramanan /** 12062b245cb2SAnirudh Venkataramanan * ice_xmit_desc_count - calculate number of tx descriptors needed 12072b245cb2SAnirudh Venkataramanan * @skb: send buffer 12082b245cb2SAnirudh Venkataramanan * 12092b245cb2SAnirudh Venkataramanan * Returns number of data descriptors needed for this skb. 12102b245cb2SAnirudh Venkataramanan */ 12112b245cb2SAnirudh Venkataramanan static unsigned int ice_xmit_desc_count(struct sk_buff *skb) 12122b245cb2SAnirudh Venkataramanan { 12132b245cb2SAnirudh Venkataramanan const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 12142b245cb2SAnirudh Venkataramanan unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 12152b245cb2SAnirudh Venkataramanan unsigned int count = 0, size = skb_headlen(skb); 12162b245cb2SAnirudh Venkataramanan 12172b245cb2SAnirudh Venkataramanan for (;;) { 12182b245cb2SAnirudh Venkataramanan count += ice_txd_use_count(size); 12192b245cb2SAnirudh Venkataramanan 12202b245cb2SAnirudh Venkataramanan if (!nr_frags--) 12212b245cb2SAnirudh Venkataramanan break; 12222b245cb2SAnirudh Venkataramanan 12232b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag++); 12242b245cb2SAnirudh Venkataramanan } 12252b245cb2SAnirudh Venkataramanan 12262b245cb2SAnirudh Venkataramanan return count; 12272b245cb2SAnirudh Venkataramanan } 12282b245cb2SAnirudh Venkataramanan 12292b245cb2SAnirudh Venkataramanan /** 12302b245cb2SAnirudh Venkataramanan * __ice_chk_linearize - Check if there are more than 8 buffers per packet 12312b245cb2SAnirudh Venkataramanan * @skb: send buffer 12322b245cb2SAnirudh Venkataramanan * 12332b245cb2SAnirudh Venkataramanan * Note: This HW can't DMA more than 8 buffers to build a packet on the wire 12342b245cb2SAnirudh Venkataramanan * and so we need to figure out the cases where we need to linearize the skb. 12352b245cb2SAnirudh Venkataramanan * 12362b245cb2SAnirudh Venkataramanan * For TSO we need to count the TSO header and segment payload separately. 12372b245cb2SAnirudh Venkataramanan * As such we need to check cases where we have 7 fragments or more as we 12382b245cb2SAnirudh Venkataramanan * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 12392b245cb2SAnirudh Venkataramanan * the segment payload in the first descriptor, and another 7 for the 12402b245cb2SAnirudh Venkataramanan * fragments. 12412b245cb2SAnirudh Venkataramanan */ 12422b245cb2SAnirudh Venkataramanan static bool __ice_chk_linearize(struct sk_buff *skb) 12432b245cb2SAnirudh Venkataramanan { 12442b245cb2SAnirudh Venkataramanan const struct skb_frag_struct *frag, *stale; 12452b245cb2SAnirudh Venkataramanan int nr_frags, sum; 12462b245cb2SAnirudh Venkataramanan 12472b245cb2SAnirudh Venkataramanan /* no need to check if number of frags is less than 7 */ 12482b245cb2SAnirudh Venkataramanan nr_frags = skb_shinfo(skb)->nr_frags; 12492b245cb2SAnirudh Venkataramanan if (nr_frags < (ICE_MAX_BUF_TXD - 1)) 12502b245cb2SAnirudh Venkataramanan return false; 12512b245cb2SAnirudh Venkataramanan 12522b245cb2SAnirudh Venkataramanan /* We need to walk through the list and validate that each group 12532b245cb2SAnirudh Venkataramanan * of 6 fragments totals at least gso_size. 12542b245cb2SAnirudh Venkataramanan */ 12552b245cb2SAnirudh Venkataramanan nr_frags -= ICE_MAX_BUF_TXD - 2; 12562b245cb2SAnirudh Venkataramanan frag = &skb_shinfo(skb)->frags[0]; 12572b245cb2SAnirudh Venkataramanan 12582b245cb2SAnirudh Venkataramanan /* Initialize size to the negative value of gso_size minus 1. We 12592b245cb2SAnirudh Venkataramanan * use this as the worst case scenerio in which the frag ahead 12602b245cb2SAnirudh Venkataramanan * of us only provides one byte which is why we are limited to 6 12612b245cb2SAnirudh Venkataramanan * descriptors for a single transmit as the header and previous 12622b245cb2SAnirudh Venkataramanan * fragment are already consuming 2 descriptors. 12632b245cb2SAnirudh Venkataramanan */ 12642b245cb2SAnirudh Venkataramanan sum = 1 - skb_shinfo(skb)->gso_size; 12652b245cb2SAnirudh Venkataramanan 12662b245cb2SAnirudh Venkataramanan /* Add size of frags 0 through 4 to create our initial sum */ 12672b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 12682b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 12692b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 12702b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 12712b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 12722b245cb2SAnirudh Venkataramanan 12732b245cb2SAnirudh Venkataramanan /* Walk through fragments adding latest fragment, testing it, and 12742b245cb2SAnirudh Venkataramanan * then removing stale fragments from the sum. 12752b245cb2SAnirudh Venkataramanan */ 12762b245cb2SAnirudh Venkataramanan stale = &skb_shinfo(skb)->frags[0]; 12772b245cb2SAnirudh Venkataramanan for (;;) { 12782b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 12792b245cb2SAnirudh Venkataramanan 12802b245cb2SAnirudh Venkataramanan /* if sum is negative we failed to make sufficient progress */ 12812b245cb2SAnirudh Venkataramanan if (sum < 0) 12822b245cb2SAnirudh Venkataramanan return true; 12832b245cb2SAnirudh Venkataramanan 12842b245cb2SAnirudh Venkataramanan if (!nr_frags--) 12852b245cb2SAnirudh Venkataramanan break; 12862b245cb2SAnirudh Venkataramanan 12872b245cb2SAnirudh Venkataramanan sum -= skb_frag_size(stale++); 12882b245cb2SAnirudh Venkataramanan } 12892b245cb2SAnirudh Venkataramanan 12902b245cb2SAnirudh Venkataramanan return false; 12912b245cb2SAnirudh Venkataramanan } 12922b245cb2SAnirudh Venkataramanan 12932b245cb2SAnirudh Venkataramanan /** 12942b245cb2SAnirudh Venkataramanan * ice_chk_linearize - Check if there are more than 8 fragments per packet 12952b245cb2SAnirudh Venkataramanan * @skb: send buffer 12962b245cb2SAnirudh Venkataramanan * @count: number of buffers used 12972b245cb2SAnirudh Venkataramanan * 12982b245cb2SAnirudh Venkataramanan * Note: Our HW can't scatter-gather more than 8 fragments to build 12992b245cb2SAnirudh Venkataramanan * a packet on the wire and so we need to figure out the cases where we 13002b245cb2SAnirudh Venkataramanan * need to linearize the skb. 13012b245cb2SAnirudh Venkataramanan */ 13022b245cb2SAnirudh Venkataramanan static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count) 13032b245cb2SAnirudh Venkataramanan { 13042b245cb2SAnirudh Venkataramanan /* Both TSO and single send will work if count is less than 8 */ 13052b245cb2SAnirudh Venkataramanan if (likely(count < ICE_MAX_BUF_TXD)) 13062b245cb2SAnirudh Venkataramanan return false; 13072b245cb2SAnirudh Venkataramanan 13082b245cb2SAnirudh Venkataramanan if (skb_is_gso(skb)) 13092b245cb2SAnirudh Venkataramanan return __ice_chk_linearize(skb); 13102b245cb2SAnirudh Venkataramanan 13112b245cb2SAnirudh Venkataramanan /* we can support up to 8 data buffers for a single send */ 13122b245cb2SAnirudh Venkataramanan return count != ICE_MAX_BUF_TXD; 13132b245cb2SAnirudh Venkataramanan } 13142b245cb2SAnirudh Venkataramanan 13152b245cb2SAnirudh Venkataramanan /** 13162b245cb2SAnirudh Venkataramanan * ice_xmit_frame_ring - Sends buffer on Tx ring 13172b245cb2SAnirudh Venkataramanan * @skb: send buffer 13182b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 13192b245cb2SAnirudh Venkataramanan * 13202b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 13212b245cb2SAnirudh Venkataramanan */ 13222b245cb2SAnirudh Venkataramanan static netdev_tx_t 13232b245cb2SAnirudh Venkataramanan ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) 13242b245cb2SAnirudh Venkataramanan { 13252b245cb2SAnirudh Venkataramanan struct ice_tx_buf *first; 13262b245cb2SAnirudh Venkataramanan unsigned int count; 13272b245cb2SAnirudh Venkataramanan 13282b245cb2SAnirudh Venkataramanan count = ice_xmit_desc_count(skb); 13292b245cb2SAnirudh Venkataramanan if (ice_chk_linearize(skb, count)) { 13302b245cb2SAnirudh Venkataramanan if (__skb_linearize(skb)) 13312b245cb2SAnirudh Venkataramanan goto out_drop; 13322b245cb2SAnirudh Venkataramanan count = ice_txd_use_count(skb->len); 13332b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_linearize++; 13342b245cb2SAnirudh Venkataramanan } 13352b245cb2SAnirudh Venkataramanan 13362b245cb2SAnirudh Venkataramanan /* need: 1 descriptor per page * PAGE_SIZE/ICE_MAX_DATA_PER_TXD, 13372b245cb2SAnirudh Venkataramanan * + 1 desc for skb_head_len/ICE_MAX_DATA_PER_TXD, 13382b245cb2SAnirudh Venkataramanan * + 4 desc gap to avoid the cache line where head is, 13392b245cb2SAnirudh Venkataramanan * + 1 desc for context descriptor, 13402b245cb2SAnirudh Venkataramanan * otherwise try next time 13412b245cb2SAnirudh Venkataramanan */ 13422b245cb2SAnirudh Venkataramanan if (ice_maybe_stop_tx(tx_ring, count + 4 + 1)) { 13432b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_busy++; 13442b245cb2SAnirudh Venkataramanan return NETDEV_TX_BUSY; 13452b245cb2SAnirudh Venkataramanan } 13462b245cb2SAnirudh Venkataramanan 13472b245cb2SAnirudh Venkataramanan /* record the location of the first descriptor for this packet */ 13482b245cb2SAnirudh Venkataramanan first = &tx_ring->tx_buf[tx_ring->next_to_use]; 13492b245cb2SAnirudh Venkataramanan first->skb = skb; 13502b245cb2SAnirudh Venkataramanan first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 13512b245cb2SAnirudh Venkataramanan first->gso_segs = 1; 13522b245cb2SAnirudh Venkataramanan 13532b245cb2SAnirudh Venkataramanan ice_tx_map(tx_ring, first); 13542b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 13552b245cb2SAnirudh Venkataramanan 13562b245cb2SAnirudh Venkataramanan out_drop: 13572b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 13582b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 13592b245cb2SAnirudh Venkataramanan } 13602b245cb2SAnirudh Venkataramanan 13612b245cb2SAnirudh Venkataramanan /** 13622b245cb2SAnirudh Venkataramanan * ice_start_xmit - Selects the correct VSI and Tx queue to send buffer 13632b245cb2SAnirudh Venkataramanan * @skb: send buffer 13642b245cb2SAnirudh Venkataramanan * @netdev: network interface device structure 13652b245cb2SAnirudh Venkataramanan * 13662b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 13672b245cb2SAnirudh Venkataramanan */ 13682b245cb2SAnirudh Venkataramanan netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev) 13692b245cb2SAnirudh Venkataramanan { 13702b245cb2SAnirudh Venkataramanan struct ice_netdev_priv *np = netdev_priv(netdev); 13712b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = np->vsi; 13722b245cb2SAnirudh Venkataramanan struct ice_ring *tx_ring; 13732b245cb2SAnirudh Venkataramanan 13742b245cb2SAnirudh Venkataramanan tx_ring = vsi->tx_rings[skb->queue_mapping]; 13752b245cb2SAnirudh Venkataramanan 13762b245cb2SAnirudh Venkataramanan /* hardware can't handle really short frames, hardware padding works 13772b245cb2SAnirudh Venkataramanan * beyond this point 13782b245cb2SAnirudh Venkataramanan */ 13792b245cb2SAnirudh Venkataramanan if (skb_put_padto(skb, ICE_MIN_TX_LEN)) 13802b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 13812b245cb2SAnirudh Venkataramanan 13822b245cb2SAnirudh Venkataramanan return ice_xmit_frame_ring(skb, tx_ring); 13832b245cb2SAnirudh Venkataramanan } 1384