1cdedef59SAnirudh Venkataramanan // SPDX-License-Identifier: GPL-2.0 2cdedef59SAnirudh Venkataramanan /* Copyright (c) 2018, Intel Corporation. */ 3cdedef59SAnirudh Venkataramanan 4cdedef59SAnirudh Venkataramanan /* The driver transmit and receive code */ 5cdedef59SAnirudh Venkataramanan 6cdedef59SAnirudh Venkataramanan #include <linux/prefetch.h> 7cdedef59SAnirudh Venkataramanan #include <linux/mm.h> 8cdedef59SAnirudh Venkataramanan #include "ice.h" 9cdedef59SAnirudh Venkataramanan 102b245cb2SAnirudh Venkataramanan #define ICE_RX_HDR_SIZE 256 112b245cb2SAnirudh Venkataramanan 12cdedef59SAnirudh Venkataramanan /** 13cdedef59SAnirudh Venkataramanan * ice_unmap_and_free_tx_buf - Release a Tx buffer 14cdedef59SAnirudh Venkataramanan * @ring: the ring that owns the buffer 15cdedef59SAnirudh Venkataramanan * @tx_buf: the buffer to free 16cdedef59SAnirudh Venkataramanan */ 17cdedef59SAnirudh Venkataramanan static void 18cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(struct ice_ring *ring, struct ice_tx_buf *tx_buf) 19cdedef59SAnirudh Venkataramanan { 20cdedef59SAnirudh Venkataramanan if (tx_buf->skb) { 21cdedef59SAnirudh Venkataramanan dev_kfree_skb_any(tx_buf->skb); 22cdedef59SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) 23cdedef59SAnirudh Venkataramanan dma_unmap_single(ring->dev, 24cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 25cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 26cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 27cdedef59SAnirudh Venkataramanan } else if (dma_unmap_len(tx_buf, len)) { 28cdedef59SAnirudh Venkataramanan dma_unmap_page(ring->dev, 29cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 30cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 31cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 32cdedef59SAnirudh Venkataramanan } 33cdedef59SAnirudh Venkataramanan 34cdedef59SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 35cdedef59SAnirudh Venkataramanan tx_buf->skb = NULL; 36cdedef59SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 37cdedef59SAnirudh Venkataramanan /* tx_buf must be completely set up in the transmit path */ 38cdedef59SAnirudh Venkataramanan } 39cdedef59SAnirudh Venkataramanan 40cdedef59SAnirudh Venkataramanan static struct netdev_queue *txring_txq(const struct ice_ring *ring) 41cdedef59SAnirudh Venkataramanan { 42cdedef59SAnirudh Venkataramanan return netdev_get_tx_queue(ring->netdev, ring->q_index); 43cdedef59SAnirudh Venkataramanan } 44cdedef59SAnirudh Venkataramanan 45cdedef59SAnirudh Venkataramanan /** 46cdedef59SAnirudh Venkataramanan * ice_clean_tx_ring - Free any empty Tx buffers 47cdedef59SAnirudh Venkataramanan * @tx_ring: ring to be cleaned 48cdedef59SAnirudh Venkataramanan */ 49cdedef59SAnirudh Venkataramanan void ice_clean_tx_ring(struct ice_ring *tx_ring) 50cdedef59SAnirudh Venkataramanan { 51cdedef59SAnirudh Venkataramanan u16 i; 52cdedef59SAnirudh Venkataramanan 53cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 54cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 55cdedef59SAnirudh Venkataramanan return; 56cdedef59SAnirudh Venkataramanan 57cdedef59SAnirudh Venkataramanan /* Free all the Tx ring sk_bufss */ 58cdedef59SAnirudh Venkataramanan for (i = 0; i < tx_ring->count; i++) 59cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]); 60cdedef59SAnirudh Venkataramanan 61c6dfd690SBruce Allan memset(tx_ring->tx_buf, 0, sizeof(*tx_ring->tx_buf) * tx_ring->count); 62cdedef59SAnirudh Venkataramanan 63cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 64cdedef59SAnirudh Venkataramanan memset(tx_ring->desc, 0, tx_ring->size); 65cdedef59SAnirudh Venkataramanan 66cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 67cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 68cdedef59SAnirudh Venkataramanan 69cdedef59SAnirudh Venkataramanan if (!tx_ring->netdev) 70cdedef59SAnirudh Venkataramanan return; 71cdedef59SAnirudh Venkataramanan 72cdedef59SAnirudh Venkataramanan /* cleanup Tx queue statistics */ 73cdedef59SAnirudh Venkataramanan netdev_tx_reset_queue(txring_txq(tx_ring)); 74cdedef59SAnirudh Venkataramanan } 75cdedef59SAnirudh Venkataramanan 76cdedef59SAnirudh Venkataramanan /** 77cdedef59SAnirudh Venkataramanan * ice_free_tx_ring - Free Tx resources per queue 78cdedef59SAnirudh Venkataramanan * @tx_ring: Tx descriptor ring for a specific queue 79cdedef59SAnirudh Venkataramanan * 80cdedef59SAnirudh Venkataramanan * Free all transmit software resources 81cdedef59SAnirudh Venkataramanan */ 82cdedef59SAnirudh Venkataramanan void ice_free_tx_ring(struct ice_ring *tx_ring) 83cdedef59SAnirudh Venkataramanan { 84cdedef59SAnirudh Venkataramanan ice_clean_tx_ring(tx_ring); 85cdedef59SAnirudh Venkataramanan devm_kfree(tx_ring->dev, tx_ring->tx_buf); 86cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 87cdedef59SAnirudh Venkataramanan 88cdedef59SAnirudh Venkataramanan if (tx_ring->desc) { 89cdedef59SAnirudh Venkataramanan dmam_free_coherent(tx_ring->dev, tx_ring->size, 90cdedef59SAnirudh Venkataramanan tx_ring->desc, tx_ring->dma); 91cdedef59SAnirudh Venkataramanan tx_ring->desc = NULL; 92cdedef59SAnirudh Venkataramanan } 93cdedef59SAnirudh Venkataramanan } 94cdedef59SAnirudh Venkataramanan 95cdedef59SAnirudh Venkataramanan /** 962b245cb2SAnirudh Venkataramanan * ice_clean_tx_irq - Reclaim resources after transmit completes 972b245cb2SAnirudh Venkataramanan * @vsi: the VSI we care about 982b245cb2SAnirudh Venkataramanan * @tx_ring: Tx ring to clean 992b245cb2SAnirudh Venkataramanan * @napi_budget: Used to determine if we are in netpoll 1002b245cb2SAnirudh Venkataramanan * 1012b245cb2SAnirudh Venkataramanan * Returns true if there's any budget left (e.g. the clean is finished) 1022b245cb2SAnirudh Venkataramanan */ 103c8b7abddSBruce Allan static bool 104c8b7abddSBruce Allan ice_clean_tx_irq(struct ice_vsi *vsi, struct ice_ring *tx_ring, int napi_budget) 1052b245cb2SAnirudh Venkataramanan { 1062b245cb2SAnirudh Venkataramanan unsigned int total_bytes = 0, total_pkts = 0; 1072b245cb2SAnirudh Venkataramanan unsigned int budget = vsi->work_lmt; 1082b245cb2SAnirudh Venkataramanan s16 i = tx_ring->next_to_clean; 1092b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 1102b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 1112b245cb2SAnirudh Venkataramanan 1122b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 1132b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 1142b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1152b245cb2SAnirudh Venkataramanan 1162b245cb2SAnirudh Venkataramanan do { 1172b245cb2SAnirudh Venkataramanan struct ice_tx_desc *eop_desc = tx_buf->next_to_watch; 1182b245cb2SAnirudh Venkataramanan 1192b245cb2SAnirudh Venkataramanan /* if next_to_watch is not set then there is no work pending */ 1202b245cb2SAnirudh Venkataramanan if (!eop_desc) 1212b245cb2SAnirudh Venkataramanan break; 1222b245cb2SAnirudh Venkataramanan 1232b245cb2SAnirudh Venkataramanan smp_rmb(); /* prevent any other reads prior to eop_desc */ 1242b245cb2SAnirudh Venkataramanan 1252b245cb2SAnirudh Venkataramanan /* if the descriptor isn't done, no work yet to do */ 1262b245cb2SAnirudh Venkataramanan if (!(eop_desc->cmd_type_offset_bsz & 1272b245cb2SAnirudh Venkataramanan cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) 1282b245cb2SAnirudh Venkataramanan break; 1292b245cb2SAnirudh Venkataramanan 1302b245cb2SAnirudh Venkataramanan /* clear next_to_watch to prevent false hangs */ 1312b245cb2SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 1322b245cb2SAnirudh Venkataramanan 1332b245cb2SAnirudh Venkataramanan /* update the statistics for this packet */ 1342b245cb2SAnirudh Venkataramanan total_bytes += tx_buf->bytecount; 1352b245cb2SAnirudh Venkataramanan total_pkts += tx_buf->gso_segs; 1362b245cb2SAnirudh Venkataramanan 1372b245cb2SAnirudh Venkataramanan /* free the skb */ 1382b245cb2SAnirudh Venkataramanan napi_consume_skb(tx_buf->skb, napi_budget); 1392b245cb2SAnirudh Venkataramanan 1402b245cb2SAnirudh Venkataramanan /* unmap skb header data */ 1412b245cb2SAnirudh Venkataramanan dma_unmap_single(tx_ring->dev, 1422b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 1432b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 1442b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1452b245cb2SAnirudh Venkataramanan 1462b245cb2SAnirudh Venkataramanan /* clear tx_buf data */ 1472b245cb2SAnirudh Venkataramanan tx_buf->skb = NULL; 1482b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 1492b245cb2SAnirudh Venkataramanan 1502b245cb2SAnirudh Venkataramanan /* unmap remaining buffers */ 1512b245cb2SAnirudh Venkataramanan while (tx_desc != eop_desc) { 1522b245cb2SAnirudh Venkataramanan tx_buf++; 1532b245cb2SAnirudh Venkataramanan tx_desc++; 1542b245cb2SAnirudh Venkataramanan i++; 1552b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 1562b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1572b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 1582b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1592b245cb2SAnirudh Venkataramanan } 1602b245cb2SAnirudh Venkataramanan 1612b245cb2SAnirudh Venkataramanan /* unmap any remaining paged data */ 1622b245cb2SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) { 1632b245cb2SAnirudh Venkataramanan dma_unmap_page(tx_ring->dev, 1642b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 1652b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 1662b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1672b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 1682b245cb2SAnirudh Venkataramanan } 1692b245cb2SAnirudh Venkataramanan } 1702b245cb2SAnirudh Venkataramanan 1712b245cb2SAnirudh Venkataramanan /* move us one more past the eop_desc for start of next pkt */ 1722b245cb2SAnirudh Venkataramanan tx_buf++; 1732b245cb2SAnirudh Venkataramanan tx_desc++; 1742b245cb2SAnirudh Venkataramanan i++; 1752b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 1762b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 1772b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 1782b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1792b245cb2SAnirudh Venkataramanan } 1802b245cb2SAnirudh Venkataramanan 1812b245cb2SAnirudh Venkataramanan prefetch(tx_desc); 1822b245cb2SAnirudh Venkataramanan 1832b245cb2SAnirudh Venkataramanan /* update budget accounting */ 1842b245cb2SAnirudh Venkataramanan budget--; 1852b245cb2SAnirudh Venkataramanan } while (likely(budget)); 1862b245cb2SAnirudh Venkataramanan 1872b245cb2SAnirudh Venkataramanan i += tx_ring->count; 1882b245cb2SAnirudh Venkataramanan tx_ring->next_to_clean = i; 1892b245cb2SAnirudh Venkataramanan u64_stats_update_begin(&tx_ring->syncp); 1902b245cb2SAnirudh Venkataramanan tx_ring->stats.bytes += total_bytes; 1912b245cb2SAnirudh Venkataramanan tx_ring->stats.pkts += total_pkts; 1922b245cb2SAnirudh Venkataramanan u64_stats_update_end(&tx_ring->syncp); 1932b245cb2SAnirudh Venkataramanan tx_ring->q_vector->tx.total_bytes += total_bytes; 1942b245cb2SAnirudh Venkataramanan tx_ring->q_vector->tx.total_pkts += total_pkts; 1952b245cb2SAnirudh Venkataramanan 1962b245cb2SAnirudh Venkataramanan netdev_tx_completed_queue(txring_txq(tx_ring), total_pkts, 1972b245cb2SAnirudh Venkataramanan total_bytes); 1982b245cb2SAnirudh Venkataramanan 1992b245cb2SAnirudh Venkataramanan #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2)) 2002b245cb2SAnirudh Venkataramanan if (unlikely(total_pkts && netif_carrier_ok(tx_ring->netdev) && 2012b245cb2SAnirudh Venkataramanan (ICE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 2022b245cb2SAnirudh Venkataramanan /* Make sure that anybody stopping the queue after this 2032b245cb2SAnirudh Venkataramanan * sees the new next_to_clean. 2042b245cb2SAnirudh Venkataramanan */ 2052b245cb2SAnirudh Venkataramanan smp_mb(); 2062b245cb2SAnirudh Venkataramanan if (__netif_subqueue_stopped(tx_ring->netdev, 2072b245cb2SAnirudh Venkataramanan tx_ring->q_index) && 2082b245cb2SAnirudh Venkataramanan !test_bit(__ICE_DOWN, vsi->state)) { 2092b245cb2SAnirudh Venkataramanan netif_wake_subqueue(tx_ring->netdev, 2102b245cb2SAnirudh Venkataramanan tx_ring->q_index); 2112b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 2122b245cb2SAnirudh Venkataramanan } 2132b245cb2SAnirudh Venkataramanan } 2142b245cb2SAnirudh Venkataramanan 2152b245cb2SAnirudh Venkataramanan return !!budget; 2162b245cb2SAnirudh Venkataramanan } 2172b245cb2SAnirudh Venkataramanan 2182b245cb2SAnirudh Venkataramanan /** 219cdedef59SAnirudh Venkataramanan * ice_setup_tx_ring - Allocate the Tx descriptors 220d337f2afSAnirudh Venkataramanan * @tx_ring: the Tx ring to set up 221cdedef59SAnirudh Venkataramanan * 222cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 223cdedef59SAnirudh Venkataramanan */ 224cdedef59SAnirudh Venkataramanan int ice_setup_tx_ring(struct ice_ring *tx_ring) 225cdedef59SAnirudh Venkataramanan { 226cdedef59SAnirudh Venkataramanan struct device *dev = tx_ring->dev; 227cdedef59SAnirudh Venkataramanan 228cdedef59SAnirudh Venkataramanan if (!dev) 229cdedef59SAnirudh Venkataramanan return -ENOMEM; 230cdedef59SAnirudh Venkataramanan 231cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 232cdedef59SAnirudh Venkataramanan WARN_ON(tx_ring->tx_buf); 233c6dfd690SBruce Allan tx_ring->tx_buf = 234c6dfd690SBruce Allan devm_kzalloc(dev, sizeof(*tx_ring->tx_buf) * tx_ring->count, 235c6dfd690SBruce Allan GFP_KERNEL); 236cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 237cdedef59SAnirudh Venkataramanan return -ENOMEM; 238cdedef59SAnirudh Venkataramanan 239ad71b256SBrett Creeley /* round up to nearest page */ 240c6dfd690SBruce Allan tx_ring->size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc), 241ad71b256SBrett Creeley PAGE_SIZE); 242cdedef59SAnirudh Venkataramanan tx_ring->desc = dmam_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, 243cdedef59SAnirudh Venkataramanan GFP_KERNEL); 244cdedef59SAnirudh Venkataramanan if (!tx_ring->desc) { 245cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 246cdedef59SAnirudh Venkataramanan tx_ring->size); 247cdedef59SAnirudh Venkataramanan goto err; 248cdedef59SAnirudh Venkataramanan } 249cdedef59SAnirudh Venkataramanan 250cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 251cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 252b3969fd7SSudheer Mogilappagari tx_ring->tx_stats.prev_pkt = -1; 253cdedef59SAnirudh Venkataramanan return 0; 254cdedef59SAnirudh Venkataramanan 255cdedef59SAnirudh Venkataramanan err: 256cdedef59SAnirudh Venkataramanan devm_kfree(dev, tx_ring->tx_buf); 257cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 258cdedef59SAnirudh Venkataramanan return -ENOMEM; 259cdedef59SAnirudh Venkataramanan } 260cdedef59SAnirudh Venkataramanan 261cdedef59SAnirudh Venkataramanan /** 262cdedef59SAnirudh Venkataramanan * ice_clean_rx_ring - Free Rx buffers 263cdedef59SAnirudh Venkataramanan * @rx_ring: ring to be cleaned 264cdedef59SAnirudh Venkataramanan */ 265cdedef59SAnirudh Venkataramanan void ice_clean_rx_ring(struct ice_ring *rx_ring) 266cdedef59SAnirudh Venkataramanan { 267cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 268cdedef59SAnirudh Venkataramanan u16 i; 269cdedef59SAnirudh Venkataramanan 270cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 271cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 272cdedef59SAnirudh Venkataramanan return; 273cdedef59SAnirudh Venkataramanan 274cdedef59SAnirudh Venkataramanan /* Free all the Rx ring sk_buffs */ 275cdedef59SAnirudh Venkataramanan for (i = 0; i < rx_ring->count; i++) { 276cdedef59SAnirudh Venkataramanan struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i]; 277cdedef59SAnirudh Venkataramanan 278cdedef59SAnirudh Venkataramanan if (rx_buf->skb) { 279cdedef59SAnirudh Venkataramanan dev_kfree_skb(rx_buf->skb); 280cdedef59SAnirudh Venkataramanan rx_buf->skb = NULL; 281cdedef59SAnirudh Venkataramanan } 282cdedef59SAnirudh Venkataramanan if (!rx_buf->page) 283cdedef59SAnirudh Venkataramanan continue; 284cdedef59SAnirudh Venkataramanan 285a65f71feSMaciej Fijalkowski /* Invalidate cache lines that may have been written to by 286a65f71feSMaciej Fijalkowski * device so that we avoid corrupting memory. 287a65f71feSMaciej Fijalkowski */ 288a65f71feSMaciej Fijalkowski dma_sync_single_range_for_cpu(dev, rx_buf->dma, 289a65f71feSMaciej Fijalkowski rx_buf->page_offset, 290a65f71feSMaciej Fijalkowski ICE_RXBUF_2048, DMA_FROM_DEVICE); 291a65f71feSMaciej Fijalkowski 292a65f71feSMaciej Fijalkowski /* free resources associated with mapping */ 293a65f71feSMaciej Fijalkowski dma_unmap_page_attrs(dev, rx_buf->dma, PAGE_SIZE, 294a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 29503c66a13SMaciej Fijalkowski __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias); 296cdedef59SAnirudh Venkataramanan 297cdedef59SAnirudh Venkataramanan rx_buf->page = NULL; 298cdedef59SAnirudh Venkataramanan rx_buf->page_offset = 0; 299cdedef59SAnirudh Venkataramanan } 300cdedef59SAnirudh Venkataramanan 301c6dfd690SBruce Allan memset(rx_ring->rx_buf, 0, sizeof(*rx_ring->rx_buf) * rx_ring->count); 302cdedef59SAnirudh Venkataramanan 303cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 304cdedef59SAnirudh Venkataramanan memset(rx_ring->desc, 0, rx_ring->size); 305cdedef59SAnirudh Venkataramanan 306cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = 0; 307cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 308cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 309cdedef59SAnirudh Venkataramanan } 310cdedef59SAnirudh Venkataramanan 311cdedef59SAnirudh Venkataramanan /** 312cdedef59SAnirudh Venkataramanan * ice_free_rx_ring - Free Rx resources 313cdedef59SAnirudh Venkataramanan * @rx_ring: ring to clean the resources from 314cdedef59SAnirudh Venkataramanan * 315cdedef59SAnirudh Venkataramanan * Free all receive software resources 316cdedef59SAnirudh Venkataramanan */ 317cdedef59SAnirudh Venkataramanan void ice_free_rx_ring(struct ice_ring *rx_ring) 318cdedef59SAnirudh Venkataramanan { 319cdedef59SAnirudh Venkataramanan ice_clean_rx_ring(rx_ring); 320cdedef59SAnirudh Venkataramanan devm_kfree(rx_ring->dev, rx_ring->rx_buf); 321cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 322cdedef59SAnirudh Venkataramanan 323cdedef59SAnirudh Venkataramanan if (rx_ring->desc) { 324cdedef59SAnirudh Venkataramanan dmam_free_coherent(rx_ring->dev, rx_ring->size, 325cdedef59SAnirudh Venkataramanan rx_ring->desc, rx_ring->dma); 326cdedef59SAnirudh Venkataramanan rx_ring->desc = NULL; 327cdedef59SAnirudh Venkataramanan } 328cdedef59SAnirudh Venkataramanan } 329cdedef59SAnirudh Venkataramanan 330cdedef59SAnirudh Venkataramanan /** 331cdedef59SAnirudh Venkataramanan * ice_setup_rx_ring - Allocate the Rx descriptors 332d337f2afSAnirudh Venkataramanan * @rx_ring: the Rx ring to set up 333cdedef59SAnirudh Venkataramanan * 334cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 335cdedef59SAnirudh Venkataramanan */ 336cdedef59SAnirudh Venkataramanan int ice_setup_rx_ring(struct ice_ring *rx_ring) 337cdedef59SAnirudh Venkataramanan { 338cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 339cdedef59SAnirudh Venkataramanan 340cdedef59SAnirudh Venkataramanan if (!dev) 341cdedef59SAnirudh Venkataramanan return -ENOMEM; 342cdedef59SAnirudh Venkataramanan 343cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 344cdedef59SAnirudh Venkataramanan WARN_ON(rx_ring->rx_buf); 345c6dfd690SBruce Allan rx_ring->rx_buf = 346c6dfd690SBruce Allan devm_kzalloc(dev, sizeof(*rx_ring->rx_buf) * rx_ring->count, 347c6dfd690SBruce Allan GFP_KERNEL); 348cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 349cdedef59SAnirudh Venkataramanan return -ENOMEM; 350cdedef59SAnirudh Venkataramanan 351ad71b256SBrett Creeley /* round up to nearest page */ 352ad71b256SBrett Creeley rx_ring->size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc), 353ad71b256SBrett Creeley PAGE_SIZE); 354cdedef59SAnirudh Venkataramanan rx_ring->desc = dmam_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, 355cdedef59SAnirudh Venkataramanan GFP_KERNEL); 356cdedef59SAnirudh Venkataramanan if (!rx_ring->desc) { 357cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 358cdedef59SAnirudh Venkataramanan rx_ring->size); 359cdedef59SAnirudh Venkataramanan goto err; 360cdedef59SAnirudh Venkataramanan } 361cdedef59SAnirudh Venkataramanan 362cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 363cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 364cdedef59SAnirudh Venkataramanan return 0; 365cdedef59SAnirudh Venkataramanan 366cdedef59SAnirudh Venkataramanan err: 367cdedef59SAnirudh Venkataramanan devm_kfree(dev, rx_ring->rx_buf); 368cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 369cdedef59SAnirudh Venkataramanan return -ENOMEM; 370cdedef59SAnirudh Venkataramanan } 371cdedef59SAnirudh Venkataramanan 372cdedef59SAnirudh Venkataramanan /** 373cdedef59SAnirudh Venkataramanan * ice_release_rx_desc - Store the new tail and head values 374cdedef59SAnirudh Venkataramanan * @rx_ring: ring to bump 375cdedef59SAnirudh Venkataramanan * @val: new head index 376cdedef59SAnirudh Venkataramanan */ 377cdedef59SAnirudh Venkataramanan static void ice_release_rx_desc(struct ice_ring *rx_ring, u32 val) 378cdedef59SAnirudh Venkataramanan { 379cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = val; 380cdedef59SAnirudh Venkataramanan 381cdedef59SAnirudh Venkataramanan /* update next to alloc since we have filled the ring */ 382cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = val; 383cdedef59SAnirudh Venkataramanan 384cdedef59SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w 385cdedef59SAnirudh Venkataramanan * know there are new descriptors to fetch. (Only 386cdedef59SAnirudh Venkataramanan * applicable for weak-ordered memory model archs, 387cdedef59SAnirudh Venkataramanan * such as IA-64). 388cdedef59SAnirudh Venkataramanan */ 389cdedef59SAnirudh Venkataramanan wmb(); 390cdedef59SAnirudh Venkataramanan writel(val, rx_ring->tail); 391cdedef59SAnirudh Venkataramanan } 392cdedef59SAnirudh Venkataramanan 393cdedef59SAnirudh Venkataramanan /** 394cdedef59SAnirudh Venkataramanan * ice_alloc_mapped_page - recycle or make a new page 395cdedef59SAnirudh Venkataramanan * @rx_ring: ring to use 396cdedef59SAnirudh Venkataramanan * @bi: rx_buf struct to modify 397cdedef59SAnirudh Venkataramanan * 398cdedef59SAnirudh Venkataramanan * Returns true if the page was successfully allocated or 399cdedef59SAnirudh Venkataramanan * reused. 400cdedef59SAnirudh Venkataramanan */ 401c8b7abddSBruce Allan static bool 402c8b7abddSBruce Allan ice_alloc_mapped_page(struct ice_ring *rx_ring, struct ice_rx_buf *bi) 403cdedef59SAnirudh Venkataramanan { 404cdedef59SAnirudh Venkataramanan struct page *page = bi->page; 405cdedef59SAnirudh Venkataramanan dma_addr_t dma; 406cdedef59SAnirudh Venkataramanan 407cdedef59SAnirudh Venkataramanan /* since we are recycling buffers we should seldom need to alloc */ 4082b245cb2SAnirudh Venkataramanan if (likely(page)) { 4092b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 410cdedef59SAnirudh Venkataramanan return true; 4112b245cb2SAnirudh Venkataramanan } 412cdedef59SAnirudh Venkataramanan 413cdedef59SAnirudh Venkataramanan /* alloc new page for storage */ 414cdedef59SAnirudh Venkataramanan page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 4152b245cb2SAnirudh Venkataramanan if (unlikely(!page)) { 4162b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 417cdedef59SAnirudh Venkataramanan return false; 4182b245cb2SAnirudh Venkataramanan } 419cdedef59SAnirudh Venkataramanan 420cdedef59SAnirudh Venkataramanan /* map page for use */ 421a65f71feSMaciej Fijalkowski dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, 422a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 423cdedef59SAnirudh Venkataramanan 424cdedef59SAnirudh Venkataramanan /* if mapping failed free memory back to system since 425cdedef59SAnirudh Venkataramanan * there isn't much point in holding memory we can't use 426cdedef59SAnirudh Venkataramanan */ 427cdedef59SAnirudh Venkataramanan if (dma_mapping_error(rx_ring->dev, dma)) { 428cdedef59SAnirudh Venkataramanan __free_pages(page, 0); 4292b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 430cdedef59SAnirudh Venkataramanan return false; 431cdedef59SAnirudh Venkataramanan } 432cdedef59SAnirudh Venkataramanan 433cdedef59SAnirudh Venkataramanan bi->dma = dma; 434cdedef59SAnirudh Venkataramanan bi->page = page; 435cdedef59SAnirudh Venkataramanan bi->page_offset = 0; 43603c66a13SMaciej Fijalkowski page_ref_add(page, USHRT_MAX - 1); 43703c66a13SMaciej Fijalkowski bi->pagecnt_bias = USHRT_MAX; 438cdedef59SAnirudh Venkataramanan 439cdedef59SAnirudh Venkataramanan return true; 440cdedef59SAnirudh Venkataramanan } 441cdedef59SAnirudh Venkataramanan 442cdedef59SAnirudh Venkataramanan /** 443cdedef59SAnirudh Venkataramanan * ice_alloc_rx_bufs - Replace used receive buffers 444cdedef59SAnirudh Venkataramanan * @rx_ring: ring to place buffers on 445cdedef59SAnirudh Venkataramanan * @cleaned_count: number of buffers to replace 446cdedef59SAnirudh Venkataramanan * 447cdedef59SAnirudh Venkataramanan * Returns false if all allocations were successful, true if any fail 448cdedef59SAnirudh Venkataramanan */ 449cdedef59SAnirudh Venkataramanan bool ice_alloc_rx_bufs(struct ice_ring *rx_ring, u16 cleaned_count) 450cdedef59SAnirudh Venkataramanan { 451cdedef59SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 452cdedef59SAnirudh Venkataramanan u16 ntu = rx_ring->next_to_use; 453cdedef59SAnirudh Venkataramanan struct ice_rx_buf *bi; 454cdedef59SAnirudh Venkataramanan 455cdedef59SAnirudh Venkataramanan /* do nothing if no valid netdev defined */ 456cdedef59SAnirudh Venkataramanan if (!rx_ring->netdev || !cleaned_count) 457cdedef59SAnirudh Venkataramanan return false; 458cdedef59SAnirudh Venkataramanan 459cdedef59SAnirudh Venkataramanan /* get the RX descriptor and buffer based on next_to_use */ 460cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, ntu); 461cdedef59SAnirudh Venkataramanan bi = &rx_ring->rx_buf[ntu]; 462cdedef59SAnirudh Venkataramanan 463cdedef59SAnirudh Venkataramanan do { 464cdedef59SAnirudh Venkataramanan if (!ice_alloc_mapped_page(rx_ring, bi)) 465cdedef59SAnirudh Venkataramanan goto no_bufs; 466cdedef59SAnirudh Venkataramanan 467a65f71feSMaciej Fijalkowski /* sync the buffer for use by the device */ 468a65f71feSMaciej Fijalkowski dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 469a65f71feSMaciej Fijalkowski bi->page_offset, 470a65f71feSMaciej Fijalkowski ICE_RXBUF_2048, 471a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE); 472a65f71feSMaciej Fijalkowski 473cdedef59SAnirudh Venkataramanan /* Refresh the desc even if buffer_addrs didn't change 474cdedef59SAnirudh Venkataramanan * because each write-back erases this info. 475cdedef59SAnirudh Venkataramanan */ 476cdedef59SAnirudh Venkataramanan rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 477cdedef59SAnirudh Venkataramanan 478cdedef59SAnirudh Venkataramanan rx_desc++; 479cdedef59SAnirudh Venkataramanan bi++; 480cdedef59SAnirudh Venkataramanan ntu++; 481cdedef59SAnirudh Venkataramanan if (unlikely(ntu == rx_ring->count)) { 482cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, 0); 483cdedef59SAnirudh Venkataramanan bi = rx_ring->rx_buf; 484cdedef59SAnirudh Venkataramanan ntu = 0; 485cdedef59SAnirudh Venkataramanan } 486cdedef59SAnirudh Venkataramanan 487cdedef59SAnirudh Venkataramanan /* clear the status bits for the next_to_use descriptor */ 488cdedef59SAnirudh Venkataramanan rx_desc->wb.status_error0 = 0; 489cdedef59SAnirudh Venkataramanan 490cdedef59SAnirudh Venkataramanan cleaned_count--; 491cdedef59SAnirudh Venkataramanan } while (cleaned_count); 492cdedef59SAnirudh Venkataramanan 493cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 494cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 495cdedef59SAnirudh Venkataramanan 496cdedef59SAnirudh Venkataramanan return false; 497cdedef59SAnirudh Venkataramanan 498cdedef59SAnirudh Venkataramanan no_bufs: 499cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 500cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 501cdedef59SAnirudh Venkataramanan 502cdedef59SAnirudh Venkataramanan /* make sure to come back via polling to try again after 503cdedef59SAnirudh Venkataramanan * allocation failure 504cdedef59SAnirudh Venkataramanan */ 505cdedef59SAnirudh Venkataramanan return true; 506cdedef59SAnirudh Venkataramanan } 5072b245cb2SAnirudh Venkataramanan 5082b245cb2SAnirudh Venkataramanan /** 5092b245cb2SAnirudh Venkataramanan * ice_page_is_reserved - check if reuse is possible 5102b245cb2SAnirudh Venkataramanan * @page: page struct to check 5112b245cb2SAnirudh Venkataramanan */ 5122b245cb2SAnirudh Venkataramanan static bool ice_page_is_reserved(struct page *page) 5132b245cb2SAnirudh Venkataramanan { 5142b245cb2SAnirudh Venkataramanan return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 5152b245cb2SAnirudh Venkataramanan } 5162b245cb2SAnirudh Venkataramanan 5172b245cb2SAnirudh Venkataramanan /** 5181d032bc7SMaciej Fijalkowski * ice_rx_buf_adjust_pg_offset - Prepare Rx buffer for reuse 5191d032bc7SMaciej Fijalkowski * @rx_buf: Rx buffer to adjust 5201d032bc7SMaciej Fijalkowski * @size: Size of adjustment 5211d032bc7SMaciej Fijalkowski * 5221d032bc7SMaciej Fijalkowski * Update the offset within page so that Rx buf will be ready to be reused. 5231d032bc7SMaciej Fijalkowski * For systems with PAGE_SIZE < 8192 this function will flip the page offset 5241d032bc7SMaciej Fijalkowski * so the second half of page assigned to Rx buffer will be used, otherwise 5251d032bc7SMaciej Fijalkowski * the offset is moved by the @size bytes 5261d032bc7SMaciej Fijalkowski */ 5271d032bc7SMaciej Fijalkowski static void 5281d032bc7SMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size) 5291d032bc7SMaciej Fijalkowski { 5301d032bc7SMaciej Fijalkowski #if (PAGE_SIZE < 8192) 5311d032bc7SMaciej Fijalkowski /* flip page offset to other buffer */ 5321d032bc7SMaciej Fijalkowski rx_buf->page_offset ^= size; 5331d032bc7SMaciej Fijalkowski #else 5341d032bc7SMaciej Fijalkowski /* move offset up to the next cache line */ 5351d032bc7SMaciej Fijalkowski rx_buf->page_offset += size; 5361d032bc7SMaciej Fijalkowski #endif 5371d032bc7SMaciej Fijalkowski } 5381d032bc7SMaciej Fijalkowski 5391d032bc7SMaciej Fijalkowski /** 540bbb97808SMaciej Fijalkowski * ice_can_reuse_rx_page - Determine if page can be reused for another Rx 541bbb97808SMaciej Fijalkowski * @rx_buf: buffer containing the page 542bbb97808SMaciej Fijalkowski * 543bbb97808SMaciej Fijalkowski * If page is reusable, we have a green light for calling ice_reuse_rx_page, 544bbb97808SMaciej Fijalkowski * which will assign the current buffer to the buffer that next_to_alloc is 545bbb97808SMaciej Fijalkowski * pointing to; otherwise, the DMA mapping needs to be destroyed and 546bbb97808SMaciej Fijalkowski * page freed 547bbb97808SMaciej Fijalkowski */ 5481d032bc7SMaciej Fijalkowski static bool ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf) 549bbb97808SMaciej Fijalkowski { 5501d032bc7SMaciej Fijalkowski #if (PAGE_SIZE >= 8192) 5511d032bc7SMaciej Fijalkowski unsigned int last_offset = PAGE_SIZE - ICE_RXBUF_2048; 5521d032bc7SMaciej Fijalkowski #endif 55303c66a13SMaciej Fijalkowski unsigned int pagecnt_bias = rx_buf->pagecnt_bias; 554bbb97808SMaciej Fijalkowski struct page *page = rx_buf->page; 555bbb97808SMaciej Fijalkowski 556bbb97808SMaciej Fijalkowski /* avoid re-using remote pages */ 557bbb97808SMaciej Fijalkowski if (unlikely(ice_page_is_reserved(page))) 558bbb97808SMaciej Fijalkowski return false; 559bbb97808SMaciej Fijalkowski 560bbb97808SMaciej Fijalkowski #if (PAGE_SIZE < 8192) 561bbb97808SMaciej Fijalkowski /* if we are only owner of page we can reuse it */ 56203c66a13SMaciej Fijalkowski if (unlikely((page_count(page) - pagecnt_bias) > 1)) 563bbb97808SMaciej Fijalkowski return false; 564bbb97808SMaciej Fijalkowski #else 5651d032bc7SMaciej Fijalkowski if (rx_buf->page_offset > last_offset) 566bbb97808SMaciej Fijalkowski return false; 567bbb97808SMaciej Fijalkowski #endif /* PAGE_SIZE < 8192) */ 568bbb97808SMaciej Fijalkowski 56903c66a13SMaciej Fijalkowski /* If we have drained the page fragment pool we need to update 57003c66a13SMaciej Fijalkowski * the pagecnt_bias and page count so that we fully restock the 57103c66a13SMaciej Fijalkowski * number of references the driver holds. 572bbb97808SMaciej Fijalkowski */ 57303c66a13SMaciej Fijalkowski if (unlikely(pagecnt_bias == 1)) { 57403c66a13SMaciej Fijalkowski page_ref_add(page, USHRT_MAX - 1); 57503c66a13SMaciej Fijalkowski rx_buf->pagecnt_bias = USHRT_MAX; 57603c66a13SMaciej Fijalkowski } 577bbb97808SMaciej Fijalkowski 578bbb97808SMaciej Fijalkowski return true; 579bbb97808SMaciej Fijalkowski } 580bbb97808SMaciej Fijalkowski 581bbb97808SMaciej Fijalkowski /** 582712edbbbSMaciej Fijalkowski * ice_add_rx_frag - Add contents of Rx buffer to sk_buff as a frag 5832b245cb2SAnirudh Venkataramanan * @rx_buf: buffer containing page to add 584712edbbbSMaciej Fijalkowski * @skb: sk_buff to place the data into 585712edbbbSMaciej Fijalkowski * @size: packet length from rx_desc 5862b245cb2SAnirudh Venkataramanan * 5872b245cb2SAnirudh Venkataramanan * This function will add the data contained in rx_buf->page to the skb. 588712edbbbSMaciej Fijalkowski * It will just attach the page as a frag to the skb. 589712edbbbSMaciej Fijalkowski * The function will then update the page offset. 5902b245cb2SAnirudh Venkataramanan */ 5911d032bc7SMaciej Fijalkowski static void 5926c869cb7SMaciej Fijalkowski ice_add_rx_frag(struct ice_rx_buf *rx_buf, struct sk_buff *skb, 5936c869cb7SMaciej Fijalkowski unsigned int size) 5942b245cb2SAnirudh Venkataramanan { 595712edbbbSMaciej Fijalkowski #if (PAGE_SIZE >= 8192) 596712edbbbSMaciej Fijalkowski unsigned int truesize = SKB_DATA_ALIGN(size); 5972b245cb2SAnirudh Venkataramanan #else 598712edbbbSMaciej Fijalkowski unsigned int truesize = ICE_RXBUF_2048; 599712edbbbSMaciej Fijalkowski #endif 6001857ca42SMaciej Fijalkowski 601712edbbbSMaciej Fijalkowski skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page, 602712edbbbSMaciej Fijalkowski rx_buf->page_offset, size, truesize); 6032b245cb2SAnirudh Venkataramanan 604712edbbbSMaciej Fijalkowski /* page is being used so we must update the page offset */ 6051d032bc7SMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(rx_buf, truesize); 6062b245cb2SAnirudh Venkataramanan } 6072b245cb2SAnirudh Venkataramanan 6082b245cb2SAnirudh Venkataramanan /** 6092b245cb2SAnirudh Venkataramanan * ice_reuse_rx_page - page flip buffer and store it back on the ring 610d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to store buffers on 6112b245cb2SAnirudh Venkataramanan * @old_buf: donor buffer to have page reused 6122b245cb2SAnirudh Venkataramanan * 6132b245cb2SAnirudh Venkataramanan * Synchronizes page for reuse by the adapter 6142b245cb2SAnirudh Venkataramanan */ 615c8b7abddSBruce Allan static void 616c8b7abddSBruce Allan ice_reuse_rx_page(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf) 6172b245cb2SAnirudh Venkataramanan { 6182b245cb2SAnirudh Venkataramanan u16 nta = rx_ring->next_to_alloc; 6192b245cb2SAnirudh Venkataramanan struct ice_rx_buf *new_buf; 6202b245cb2SAnirudh Venkataramanan 6212b245cb2SAnirudh Venkataramanan new_buf = &rx_ring->rx_buf[nta]; 6222b245cb2SAnirudh Venkataramanan 6232b245cb2SAnirudh Venkataramanan /* update, and store next to alloc */ 6242b245cb2SAnirudh Venkataramanan nta++; 6252b245cb2SAnirudh Venkataramanan rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 6262b245cb2SAnirudh Venkataramanan 627712edbbbSMaciej Fijalkowski /* Transfer page from old buffer to new buffer. 628712edbbbSMaciej Fijalkowski * Move each member individually to avoid possible store 629712edbbbSMaciej Fijalkowski * forwarding stalls and unnecessary copy of skb. 630712edbbbSMaciej Fijalkowski */ 631712edbbbSMaciej Fijalkowski new_buf->dma = old_buf->dma; 632712edbbbSMaciej Fijalkowski new_buf->page = old_buf->page; 633712edbbbSMaciej Fijalkowski new_buf->page_offset = old_buf->page_offset; 634712edbbbSMaciej Fijalkowski new_buf->pagecnt_bias = old_buf->pagecnt_bias; 6352b245cb2SAnirudh Venkataramanan } 6362b245cb2SAnirudh Venkataramanan 6372b245cb2SAnirudh Venkataramanan /** 6386c869cb7SMaciej Fijalkowski * ice_get_rx_buf - Fetch Rx buffer and synchronize data for use 6396c869cb7SMaciej Fijalkowski * @rx_ring: Rx descriptor ring to transact packets on 640712edbbbSMaciej Fijalkowski * @skb: skb to be used 6416c869cb7SMaciej Fijalkowski * @size: size of buffer to add to skb 6426c869cb7SMaciej Fijalkowski * 6436c869cb7SMaciej Fijalkowski * This function will pull an Rx buffer from the ring and synchronize it 6446c869cb7SMaciej Fijalkowski * for use by the CPU. 6456c869cb7SMaciej Fijalkowski */ 6466c869cb7SMaciej Fijalkowski static struct ice_rx_buf * 647712edbbbSMaciej Fijalkowski ice_get_rx_buf(struct ice_ring *rx_ring, struct sk_buff **skb, 648712edbbbSMaciej Fijalkowski const unsigned int size) 6496c869cb7SMaciej Fijalkowski { 6506c869cb7SMaciej Fijalkowski struct ice_rx_buf *rx_buf; 6516c869cb7SMaciej Fijalkowski 6526c869cb7SMaciej Fijalkowski rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean]; 6536c869cb7SMaciej Fijalkowski prefetchw(rx_buf->page); 654712edbbbSMaciej Fijalkowski *skb = rx_buf->skb; 6556c869cb7SMaciej Fijalkowski 6566c869cb7SMaciej Fijalkowski /* we are reusing so sync this buffer for CPU use */ 6576c869cb7SMaciej Fijalkowski dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 6586c869cb7SMaciej Fijalkowski rx_buf->page_offset, size, 6596c869cb7SMaciej Fijalkowski DMA_FROM_DEVICE); 6606c869cb7SMaciej Fijalkowski 66103c66a13SMaciej Fijalkowski /* We have pulled a buffer for use, so decrement pagecnt_bias */ 66203c66a13SMaciej Fijalkowski rx_buf->pagecnt_bias--; 66303c66a13SMaciej Fijalkowski 6646c869cb7SMaciej Fijalkowski return rx_buf; 6656c869cb7SMaciej Fijalkowski } 6666c869cb7SMaciej Fijalkowski 6676c869cb7SMaciej Fijalkowski /** 668712edbbbSMaciej Fijalkowski * ice_construct_skb - Allocate skb and populate it 669d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to transact packets on 6706c869cb7SMaciej Fijalkowski * @rx_buf: Rx buffer to pull data from 6716c869cb7SMaciej Fijalkowski * @size: the length of the packet 6722b245cb2SAnirudh Venkataramanan * 673712edbbbSMaciej Fijalkowski * This function allocates an skb. It then populates it with the page 674712edbbbSMaciej Fijalkowski * data from the current receive descriptor, taking care to set up the 675712edbbbSMaciej Fijalkowski * skb correctly. 6762b245cb2SAnirudh Venkataramanan */ 677c8b7abddSBruce Allan static struct sk_buff * 678712edbbbSMaciej Fijalkowski ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf, 6796c869cb7SMaciej Fijalkowski unsigned int size) 6802b245cb2SAnirudh Venkataramanan { 681712edbbbSMaciej Fijalkowski void *va = page_address(rx_buf->page) + rx_buf->page_offset; 682712edbbbSMaciej Fijalkowski unsigned int headlen; 683712edbbbSMaciej Fijalkowski struct sk_buff *skb; 6842b245cb2SAnirudh Venkataramanan 6852b245cb2SAnirudh Venkataramanan /* prefetch first cache line of first page */ 686712edbbbSMaciej Fijalkowski prefetch(va); 6872b245cb2SAnirudh Venkataramanan #if L1_CACHE_BYTES < 128 688712edbbbSMaciej Fijalkowski prefetch((u8 *)va + L1_CACHE_BYTES); 6892b245cb2SAnirudh Venkataramanan #endif /* L1_CACHE_BYTES */ 6902b245cb2SAnirudh Venkataramanan 6912b245cb2SAnirudh Venkataramanan /* allocate a skb to store the frags */ 692712edbbbSMaciej Fijalkowski skb = __napi_alloc_skb(&rx_ring->q_vector->napi, ICE_RX_HDR_SIZE, 6932b245cb2SAnirudh Venkataramanan GFP_ATOMIC | __GFP_NOWARN); 694712edbbbSMaciej Fijalkowski if (unlikely(!skb)) 6952b245cb2SAnirudh Venkataramanan return NULL; 6962b245cb2SAnirudh Venkataramanan 6972b245cb2SAnirudh Venkataramanan skb_record_rx_queue(skb, rx_ring->q_index); 698712edbbbSMaciej Fijalkowski /* Determine available headroom for copy */ 699712edbbbSMaciej Fijalkowski headlen = size; 700712edbbbSMaciej Fijalkowski if (headlen > ICE_RX_HDR_SIZE) 701712edbbbSMaciej Fijalkowski headlen = eth_get_headlen(va, ICE_RX_HDR_SIZE); 7022b245cb2SAnirudh Venkataramanan 703712edbbbSMaciej Fijalkowski /* align pull length to size of long to optimize memcpy performance */ 704712edbbbSMaciej Fijalkowski memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 705712edbbbSMaciej Fijalkowski 706712edbbbSMaciej Fijalkowski /* if we exhaust the linear part then add what is left as a frag */ 707712edbbbSMaciej Fijalkowski size -= headlen; 708712edbbbSMaciej Fijalkowski if (size) { 709712edbbbSMaciej Fijalkowski #if (PAGE_SIZE >= 8192) 710712edbbbSMaciej Fijalkowski unsigned int truesize = SKB_DATA_ALIGN(size); 711712edbbbSMaciej Fijalkowski #else 712712edbbbSMaciej Fijalkowski unsigned int truesize = ICE_RXBUF_2048; 713712edbbbSMaciej Fijalkowski #endif 714712edbbbSMaciej Fijalkowski skb_add_rx_frag(skb, 0, rx_buf->page, 715712edbbbSMaciej Fijalkowski rx_buf->page_offset + headlen, size, truesize); 716712edbbbSMaciej Fijalkowski /* buffer is used by skb, update page_offset */ 717712edbbbSMaciej Fijalkowski ice_rx_buf_adjust_pg_offset(rx_buf, truesize); 718712edbbbSMaciej Fijalkowski } else { 719712edbbbSMaciej Fijalkowski /* buffer is unused, reset bias back to rx_buf; data was copied 720712edbbbSMaciej Fijalkowski * onto skb's linear part so there's no need for adjusting 721712edbbbSMaciej Fijalkowski * page offset and we can reuse this buffer as-is 722712edbbbSMaciej Fijalkowski */ 723712edbbbSMaciej Fijalkowski rx_buf->pagecnt_bias++; 724712edbbbSMaciej Fijalkowski } 7251d032bc7SMaciej Fijalkowski 7261d032bc7SMaciej Fijalkowski return skb; 7271d032bc7SMaciej Fijalkowski } 7281d032bc7SMaciej Fijalkowski 7291d032bc7SMaciej Fijalkowski /** 7301d032bc7SMaciej Fijalkowski * ice_put_rx_buf - Clean up used buffer and either recycle or free 7311d032bc7SMaciej Fijalkowski * @rx_ring: Rx descriptor ring to transact packets on 7321d032bc7SMaciej Fijalkowski * @rx_buf: Rx buffer to pull data from 7331d032bc7SMaciej Fijalkowski * 7341d032bc7SMaciej Fijalkowski * This function will clean up the contents of the rx_buf. It will 7351d032bc7SMaciej Fijalkowski * either recycle the buffer or unmap it and free the associated resources. 7361d032bc7SMaciej Fijalkowski */ 7371d032bc7SMaciej Fijalkowski static void ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf) 7381d032bc7SMaciej Fijalkowski { 7392b245cb2SAnirudh Venkataramanan /* hand second half of page back to the ring */ 7401d032bc7SMaciej Fijalkowski if (ice_can_reuse_rx_page(rx_buf)) { 7412b245cb2SAnirudh Venkataramanan ice_reuse_rx_page(rx_ring, rx_buf); 7422b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 7432b245cb2SAnirudh Venkataramanan } else { 7442b245cb2SAnirudh Venkataramanan /* we are not reusing the buffer so unmap it */ 745a65f71feSMaciej Fijalkowski dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma, PAGE_SIZE, 746a65f71feSMaciej Fijalkowski DMA_FROM_DEVICE, ICE_RX_DMA_ATTR); 74703c66a13SMaciej Fijalkowski __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias); 7482b245cb2SAnirudh Venkataramanan } 7492b245cb2SAnirudh Venkataramanan 7502b245cb2SAnirudh Venkataramanan /* clear contents of buffer_info */ 7512b245cb2SAnirudh Venkataramanan rx_buf->page = NULL; 752712edbbbSMaciej Fijalkowski rx_buf->skb = NULL; 7532b245cb2SAnirudh Venkataramanan } 7542b245cb2SAnirudh Venkataramanan 7552b245cb2SAnirudh Venkataramanan /** 7562b245cb2SAnirudh Venkataramanan * ice_cleanup_headers - Correct empty headers 7572b245cb2SAnirudh Venkataramanan * @skb: pointer to current skb being fixed 7582b245cb2SAnirudh Venkataramanan * 7592b245cb2SAnirudh Venkataramanan * Also address the case where we are pulling data in on pages only 7602b245cb2SAnirudh Venkataramanan * and as such no data is present in the skb header. 7612b245cb2SAnirudh Venkataramanan * 7622b245cb2SAnirudh Venkataramanan * In addition if skb is not at least 60 bytes we need to pad it so that 7632b245cb2SAnirudh Venkataramanan * it is large enough to qualify as a valid Ethernet frame. 7642b245cb2SAnirudh Venkataramanan * 7652b245cb2SAnirudh Venkataramanan * Returns true if an error was encountered and skb was freed. 7662b245cb2SAnirudh Venkataramanan */ 7672b245cb2SAnirudh Venkataramanan static bool ice_cleanup_headers(struct sk_buff *skb) 7682b245cb2SAnirudh Venkataramanan { 7692b245cb2SAnirudh Venkataramanan /* if eth_skb_pad returns an error the skb was freed */ 7702b245cb2SAnirudh Venkataramanan if (eth_skb_pad(skb)) 7712b245cb2SAnirudh Venkataramanan return true; 7722b245cb2SAnirudh Venkataramanan 7732b245cb2SAnirudh Venkataramanan return false; 7742b245cb2SAnirudh Venkataramanan } 7752b245cb2SAnirudh Venkataramanan 7762b245cb2SAnirudh Venkataramanan /** 7772b245cb2SAnirudh Venkataramanan * ice_test_staterr - tests bits in Rx descriptor status and error fields 7782b245cb2SAnirudh Venkataramanan * @rx_desc: pointer to receive descriptor (in le64 format) 7792b245cb2SAnirudh Venkataramanan * @stat_err_bits: value to mask 7802b245cb2SAnirudh Venkataramanan * 7812b245cb2SAnirudh Venkataramanan * This function does some fast chicanery in order to return the 7822b245cb2SAnirudh Venkataramanan * value of the mask which is really only used for boolean tests. 7832b245cb2SAnirudh Venkataramanan * The status_error_len doesn't need to be shifted because it begins 7842b245cb2SAnirudh Venkataramanan * at offset zero. 7852b245cb2SAnirudh Venkataramanan */ 786c8b7abddSBruce Allan static bool 787c8b7abddSBruce Allan ice_test_staterr(union ice_32b_rx_flex_desc *rx_desc, const u16 stat_err_bits) 7882b245cb2SAnirudh Venkataramanan { 7892b245cb2SAnirudh Venkataramanan return !!(rx_desc->wb.status_error0 & 7902b245cb2SAnirudh Venkataramanan cpu_to_le16(stat_err_bits)); 7912b245cb2SAnirudh Venkataramanan } 7922b245cb2SAnirudh Venkataramanan 7932b245cb2SAnirudh Venkataramanan /** 7942b245cb2SAnirudh Venkataramanan * ice_is_non_eop - process handling of non-EOP buffers 7952b245cb2SAnirudh Venkataramanan * @rx_ring: Rx ring being processed 7962b245cb2SAnirudh Venkataramanan * @rx_desc: Rx descriptor for current buffer 7972b245cb2SAnirudh Venkataramanan * @skb: Current socket buffer containing buffer in progress 7982b245cb2SAnirudh Venkataramanan * 7992b245cb2SAnirudh Venkataramanan * This function updates next to clean. If the buffer is an EOP buffer 8002b245cb2SAnirudh Venkataramanan * this function exits returning false, otherwise it will place the 8012b245cb2SAnirudh Venkataramanan * sk_buff in the next buffer to be chained and return true indicating 8022b245cb2SAnirudh Venkataramanan * that this is in fact a non-EOP buffer. 8032b245cb2SAnirudh Venkataramanan */ 804c8b7abddSBruce Allan static bool 805c8b7abddSBruce Allan ice_is_non_eop(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, 8062b245cb2SAnirudh Venkataramanan struct sk_buff *skb) 8072b245cb2SAnirudh Venkataramanan { 8082b245cb2SAnirudh Venkataramanan u32 ntc = rx_ring->next_to_clean + 1; 8092b245cb2SAnirudh Venkataramanan 8102b245cb2SAnirudh Venkataramanan /* fetch, update, and store next to clean */ 8112b245cb2SAnirudh Venkataramanan ntc = (ntc < rx_ring->count) ? ntc : 0; 8122b245cb2SAnirudh Venkataramanan rx_ring->next_to_clean = ntc; 8132b245cb2SAnirudh Venkataramanan 8142b245cb2SAnirudh Venkataramanan prefetch(ICE_RX_DESC(rx_ring, ntc)); 8152b245cb2SAnirudh Venkataramanan 8162b245cb2SAnirudh Venkataramanan /* if we are the last buffer then there is nothing else to do */ 8172b245cb2SAnirudh Venkataramanan #define ICE_RXD_EOF BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S) 8182b245cb2SAnirudh Venkataramanan if (likely(ice_test_staterr(rx_desc, ICE_RXD_EOF))) 8192b245cb2SAnirudh Venkataramanan return false; 8202b245cb2SAnirudh Venkataramanan 8212b245cb2SAnirudh Venkataramanan /* place skb in next buffer to be received */ 8222b245cb2SAnirudh Venkataramanan rx_ring->rx_buf[ntc].skb = skb; 8232b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.non_eop_descs++; 8242b245cb2SAnirudh Venkataramanan 8252b245cb2SAnirudh Venkataramanan return true; 8262b245cb2SAnirudh Venkataramanan } 8272b245cb2SAnirudh Venkataramanan 8282b245cb2SAnirudh Venkataramanan /** 829d76a60baSAnirudh Venkataramanan * ice_ptype_to_htype - get a hash type 830d76a60baSAnirudh Venkataramanan * @ptype: the ptype value from the descriptor 831d76a60baSAnirudh Venkataramanan * 832d76a60baSAnirudh Venkataramanan * Returns a hash type to be used by skb_set_hash 833d76a60baSAnirudh Venkataramanan */ 834d76a60baSAnirudh Venkataramanan static enum pkt_hash_types ice_ptype_to_htype(u8 __always_unused ptype) 835d76a60baSAnirudh Venkataramanan { 836d76a60baSAnirudh Venkataramanan return PKT_HASH_TYPE_NONE; 837d76a60baSAnirudh Venkataramanan } 838d76a60baSAnirudh Venkataramanan 839d76a60baSAnirudh Venkataramanan /** 840d76a60baSAnirudh Venkataramanan * ice_rx_hash - set the hash value in the skb 841d76a60baSAnirudh Venkataramanan * @rx_ring: descriptor ring 842d76a60baSAnirudh Venkataramanan * @rx_desc: specific descriptor 843d76a60baSAnirudh Venkataramanan * @skb: pointer to current skb 844d76a60baSAnirudh Venkataramanan * @rx_ptype: the ptype value from the descriptor 845d76a60baSAnirudh Venkataramanan */ 846d76a60baSAnirudh Venkataramanan static void 847d76a60baSAnirudh Venkataramanan ice_rx_hash(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, 848d76a60baSAnirudh Venkataramanan struct sk_buff *skb, u8 rx_ptype) 849d76a60baSAnirudh Venkataramanan { 850d76a60baSAnirudh Venkataramanan struct ice_32b_rx_flex_desc_nic *nic_mdid; 851d76a60baSAnirudh Venkataramanan u32 hash; 852d76a60baSAnirudh Venkataramanan 853d76a60baSAnirudh Venkataramanan if (!(rx_ring->netdev->features & NETIF_F_RXHASH)) 854d76a60baSAnirudh Venkataramanan return; 855d76a60baSAnirudh Venkataramanan 856d76a60baSAnirudh Venkataramanan if (rx_desc->wb.rxdid != ICE_RXDID_FLEX_NIC) 857d76a60baSAnirudh Venkataramanan return; 858d76a60baSAnirudh Venkataramanan 859d76a60baSAnirudh Venkataramanan nic_mdid = (struct ice_32b_rx_flex_desc_nic *)rx_desc; 860d76a60baSAnirudh Venkataramanan hash = le32_to_cpu(nic_mdid->rss_hash); 861d76a60baSAnirudh Venkataramanan skb_set_hash(skb, hash, ice_ptype_to_htype(rx_ptype)); 862d76a60baSAnirudh Venkataramanan } 863d76a60baSAnirudh Venkataramanan 864d76a60baSAnirudh Venkataramanan /** 865d76a60baSAnirudh Venkataramanan * ice_rx_csum - Indicate in skb if checksum is good 866d76a60baSAnirudh Venkataramanan * @vsi: the VSI we care about 867d76a60baSAnirudh Venkataramanan * @skb: skb currently being received and modified 868d76a60baSAnirudh Venkataramanan * @rx_desc: the receive descriptor 869d76a60baSAnirudh Venkataramanan * @ptype: the packet type decoded by hardware 870d76a60baSAnirudh Venkataramanan * 871d76a60baSAnirudh Venkataramanan * skb->protocol must be set before this function is called 872d76a60baSAnirudh Venkataramanan */ 873c8b7abddSBruce Allan static void 874c8b7abddSBruce Allan ice_rx_csum(struct ice_vsi *vsi, struct sk_buff *skb, 875d76a60baSAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc, u8 ptype) 876d76a60baSAnirudh Venkataramanan { 877d76a60baSAnirudh Venkataramanan struct ice_rx_ptype_decoded decoded; 878d76a60baSAnirudh Venkataramanan u32 rx_error, rx_status; 879d76a60baSAnirudh Venkataramanan bool ipv4, ipv6; 880d76a60baSAnirudh Venkataramanan 881d76a60baSAnirudh Venkataramanan rx_status = le16_to_cpu(rx_desc->wb.status_error0); 882d76a60baSAnirudh Venkataramanan rx_error = rx_status; 883d76a60baSAnirudh Venkataramanan 884d76a60baSAnirudh Venkataramanan decoded = ice_decode_rx_desc_ptype(ptype); 885d76a60baSAnirudh Venkataramanan 886d76a60baSAnirudh Venkataramanan /* Start with CHECKSUM_NONE and by default csum_level = 0 */ 887d76a60baSAnirudh Venkataramanan skb->ip_summed = CHECKSUM_NONE; 888d76a60baSAnirudh Venkataramanan skb_checksum_none_assert(skb); 889d76a60baSAnirudh Venkataramanan 890d76a60baSAnirudh Venkataramanan /* check if Rx checksum is enabled */ 891d76a60baSAnirudh Venkataramanan if (!(vsi->netdev->features & NETIF_F_RXCSUM)) 892d76a60baSAnirudh Venkataramanan return; 893d76a60baSAnirudh Venkataramanan 894d76a60baSAnirudh Venkataramanan /* check if HW has decoded the packet and checksum */ 895d76a60baSAnirudh Venkataramanan if (!(rx_status & BIT(ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))) 896d76a60baSAnirudh Venkataramanan return; 897d76a60baSAnirudh Venkataramanan 898d76a60baSAnirudh Venkataramanan if (!(decoded.known && decoded.outer_ip)) 899d76a60baSAnirudh Venkataramanan return; 900d76a60baSAnirudh Venkataramanan 901d76a60baSAnirudh Venkataramanan ipv4 = (decoded.outer_ip == ICE_RX_PTYPE_OUTER_IP) && 902d76a60baSAnirudh Venkataramanan (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV4); 903d76a60baSAnirudh Venkataramanan ipv6 = (decoded.outer_ip == ICE_RX_PTYPE_OUTER_IP) && 904d76a60baSAnirudh Venkataramanan (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV6); 905d76a60baSAnirudh Venkataramanan 906d76a60baSAnirudh Venkataramanan if (ipv4 && (rx_error & (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | 907d76a60baSAnirudh Venkataramanan BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))) 908d76a60baSAnirudh Venkataramanan goto checksum_fail; 909d76a60baSAnirudh Venkataramanan else if (ipv6 && (rx_status & 910d76a60baSAnirudh Venkataramanan (BIT(ICE_RX_FLEX_DESC_STATUS0_IPV6EXADD_S)))) 911d76a60baSAnirudh Venkataramanan goto checksum_fail; 912d76a60baSAnirudh Venkataramanan 913d76a60baSAnirudh Venkataramanan /* check for L4 errors and handle packets that were not able to be 914d76a60baSAnirudh Venkataramanan * checksummed due to arrival speed 915d76a60baSAnirudh Venkataramanan */ 916d76a60baSAnirudh Venkataramanan if (rx_error & BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)) 917d76a60baSAnirudh Venkataramanan goto checksum_fail; 918d76a60baSAnirudh Venkataramanan 919d76a60baSAnirudh Venkataramanan /* Only report checksum unnecessary for TCP, UDP, or SCTP */ 920d76a60baSAnirudh Venkataramanan switch (decoded.inner_prot) { 921d76a60baSAnirudh Venkataramanan case ICE_RX_PTYPE_INNER_PROT_TCP: 922d76a60baSAnirudh Venkataramanan case ICE_RX_PTYPE_INNER_PROT_UDP: 923d76a60baSAnirudh Venkataramanan case ICE_RX_PTYPE_INNER_PROT_SCTP: 924d76a60baSAnirudh Venkataramanan skb->ip_summed = CHECKSUM_UNNECESSARY; 925d76a60baSAnirudh Venkataramanan default: 926d76a60baSAnirudh Venkataramanan break; 927d76a60baSAnirudh Venkataramanan } 928d76a60baSAnirudh Venkataramanan return; 929d76a60baSAnirudh Venkataramanan 930d76a60baSAnirudh Venkataramanan checksum_fail: 931d76a60baSAnirudh Venkataramanan vsi->back->hw_csum_rx_error++; 932d76a60baSAnirudh Venkataramanan } 933d76a60baSAnirudh Venkataramanan 934d76a60baSAnirudh Venkataramanan /** 935d76a60baSAnirudh Venkataramanan * ice_process_skb_fields - Populate skb header fields from Rx descriptor 936d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring packet is being transacted on 937d76a60baSAnirudh Venkataramanan * @rx_desc: pointer to the EOP Rx descriptor 938d76a60baSAnirudh Venkataramanan * @skb: pointer to current skb being populated 939d76a60baSAnirudh Venkataramanan * @ptype: the packet type decoded by hardware 940d76a60baSAnirudh Venkataramanan * 941d76a60baSAnirudh Venkataramanan * This function checks the ring, descriptor, and packet information in 942d76a60baSAnirudh Venkataramanan * order to populate the hash, checksum, VLAN, protocol, and 943d76a60baSAnirudh Venkataramanan * other fields within the skb. 944d76a60baSAnirudh Venkataramanan */ 945c8b7abddSBruce Allan static void 946c8b7abddSBruce Allan ice_process_skb_fields(struct ice_ring *rx_ring, 947d76a60baSAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc, 948d76a60baSAnirudh Venkataramanan struct sk_buff *skb, u8 ptype) 949d76a60baSAnirudh Venkataramanan { 950d76a60baSAnirudh Venkataramanan ice_rx_hash(rx_ring, rx_desc, skb, ptype); 951d76a60baSAnirudh Venkataramanan 952d76a60baSAnirudh Venkataramanan /* modifies the skb - consumes the enet header */ 953d76a60baSAnirudh Venkataramanan skb->protocol = eth_type_trans(skb, rx_ring->netdev); 954d76a60baSAnirudh Venkataramanan 955d76a60baSAnirudh Venkataramanan ice_rx_csum(rx_ring->vsi, skb, rx_desc, ptype); 956d76a60baSAnirudh Venkataramanan } 957d76a60baSAnirudh Venkataramanan 958d76a60baSAnirudh Venkataramanan /** 9592b245cb2SAnirudh Venkataramanan * ice_receive_skb - Send a completed packet up the stack 960d337f2afSAnirudh Venkataramanan * @rx_ring: Rx ring in play 9612b245cb2SAnirudh Venkataramanan * @skb: packet to send up 9622b245cb2SAnirudh Venkataramanan * @vlan_tag: vlan tag for packet 9632b245cb2SAnirudh Venkataramanan * 9642b245cb2SAnirudh Venkataramanan * This function sends the completed packet (via. skb) up the stack using 9652b245cb2SAnirudh Venkataramanan * gro receive functions (with/without vlan tag) 9662b245cb2SAnirudh Venkataramanan */ 967c8b7abddSBruce Allan static void 968c8b7abddSBruce Allan ice_receive_skb(struct ice_ring *rx_ring, struct sk_buff *skb, u16 vlan_tag) 9692b245cb2SAnirudh Venkataramanan { 9702b245cb2SAnirudh Venkataramanan if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 971ac466755SAnirudh Venkataramanan (vlan_tag & VLAN_VID_MASK)) 9722b245cb2SAnirudh Venkataramanan __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 9732b245cb2SAnirudh Venkataramanan napi_gro_receive(&rx_ring->q_vector->napi, skb); 9742b245cb2SAnirudh Venkataramanan } 9752b245cb2SAnirudh Venkataramanan 9762b245cb2SAnirudh Venkataramanan /** 9772b245cb2SAnirudh Venkataramanan * ice_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 978d337f2afSAnirudh Venkataramanan * @rx_ring: Rx descriptor ring to transact packets on 9792b245cb2SAnirudh Venkataramanan * @budget: Total limit on number of packets to process 9802b245cb2SAnirudh Venkataramanan * 9812b245cb2SAnirudh Venkataramanan * This function provides a "bounce buffer" approach to Rx interrupt 9822b245cb2SAnirudh Venkataramanan * processing. The advantage to this is that on systems that have 9832b245cb2SAnirudh Venkataramanan * expensive overhead for IOMMU access this provides a means of avoiding 9842b245cb2SAnirudh Venkataramanan * it by maintaining the mapping of the page to the system. 9852b245cb2SAnirudh Venkataramanan * 9862b245cb2SAnirudh Venkataramanan * Returns amount of work completed 9872b245cb2SAnirudh Venkataramanan */ 9882b245cb2SAnirudh Venkataramanan static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) 9892b245cb2SAnirudh Venkataramanan { 9902b245cb2SAnirudh Venkataramanan unsigned int total_rx_bytes = 0, total_rx_pkts = 0; 9912b245cb2SAnirudh Venkataramanan u16 cleaned_count = ICE_DESC_UNUSED(rx_ring); 9922b245cb2SAnirudh Venkataramanan bool failure = false; 9932b245cb2SAnirudh Venkataramanan 9942b245cb2SAnirudh Venkataramanan /* start the loop to process RX packets bounded by 'budget' */ 9952b245cb2SAnirudh Venkataramanan while (likely(total_rx_pkts < (unsigned int)budget)) { 9962b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 9976c869cb7SMaciej Fijalkowski struct ice_rx_buf *rx_buf; 9982b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 9996c869cb7SMaciej Fijalkowski unsigned int size; 10002b245cb2SAnirudh Venkataramanan u16 stat_err_bits; 10012b245cb2SAnirudh Venkataramanan u16 vlan_tag = 0; 1002d76a60baSAnirudh Venkataramanan u8 rx_ptype; 10032b245cb2SAnirudh Venkataramanan 10042b245cb2SAnirudh Venkataramanan /* return some buffers to hardware, one at a time is too slow */ 10052b245cb2SAnirudh Venkataramanan if (cleaned_count >= ICE_RX_BUF_WRITE) { 10062b245cb2SAnirudh Venkataramanan failure = failure || 10072b245cb2SAnirudh Venkataramanan ice_alloc_rx_bufs(rx_ring, cleaned_count); 10082b245cb2SAnirudh Venkataramanan cleaned_count = 0; 10092b245cb2SAnirudh Venkataramanan } 10102b245cb2SAnirudh Venkataramanan 10112b245cb2SAnirudh Venkataramanan /* get the RX desc from RX ring based on 'next_to_clean' */ 10122b245cb2SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); 10132b245cb2SAnirudh Venkataramanan 10142b245cb2SAnirudh Venkataramanan /* status_error_len will always be zero for unused descriptors 10152b245cb2SAnirudh Venkataramanan * because it's cleared in cleanup, and overlaps with hdr_addr 10162b245cb2SAnirudh Venkataramanan * which is always zero because packet split isn't used, if the 10172b245cb2SAnirudh Venkataramanan * hardware wrote DD then it will be non-zero 10182b245cb2SAnirudh Venkataramanan */ 10192b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 10202b245cb2SAnirudh Venkataramanan if (!ice_test_staterr(rx_desc, stat_err_bits)) 10212b245cb2SAnirudh Venkataramanan break; 10222b245cb2SAnirudh Venkataramanan 10232b245cb2SAnirudh Venkataramanan /* This memory barrier is needed to keep us from reading 10242b245cb2SAnirudh Venkataramanan * any other fields out of the rx_desc until we know the 10252b245cb2SAnirudh Venkataramanan * DD bit is set. 10262b245cb2SAnirudh Venkataramanan */ 10272b245cb2SAnirudh Venkataramanan dma_rmb(); 10282b245cb2SAnirudh Venkataramanan 10296c869cb7SMaciej Fijalkowski size = le16_to_cpu(rx_desc->wb.pkt_len) & 10306c869cb7SMaciej Fijalkowski ICE_RX_FLX_DESC_PKT_LEN_M; 10316c869cb7SMaciej Fijalkowski 1032712edbbbSMaciej Fijalkowski rx_buf = ice_get_rx_buf(rx_ring, &skb, size); 10332b245cb2SAnirudh Venkataramanan /* allocate (if needed) and populate skb */ 1034712edbbbSMaciej Fijalkowski if (skb) 1035712edbbbSMaciej Fijalkowski ice_add_rx_frag(rx_buf, skb, size); 1036712edbbbSMaciej Fijalkowski else 1037712edbbbSMaciej Fijalkowski skb = ice_construct_skb(rx_ring, rx_buf, size); 1038712edbbbSMaciej Fijalkowski 1039712edbbbSMaciej Fijalkowski /* exit if we failed to retrieve a buffer */ 1040712edbbbSMaciej Fijalkowski if (!skb) { 1041712edbbbSMaciej Fijalkowski rx_ring->rx_stats.alloc_buf_failed++; 1042712edbbbSMaciej Fijalkowski rx_buf->pagecnt_bias++; 10432b245cb2SAnirudh Venkataramanan break; 1044712edbbbSMaciej Fijalkowski } 10452b245cb2SAnirudh Venkataramanan 10461d032bc7SMaciej Fijalkowski ice_put_rx_buf(rx_ring, rx_buf); 10472b245cb2SAnirudh Venkataramanan cleaned_count++; 10482b245cb2SAnirudh Venkataramanan 10492b245cb2SAnirudh Venkataramanan /* skip if it is NOP desc */ 10502b245cb2SAnirudh Venkataramanan if (ice_is_non_eop(rx_ring, rx_desc, skb)) 10512b245cb2SAnirudh Venkataramanan continue; 10522b245cb2SAnirudh Venkataramanan 10532b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S); 10542b245cb2SAnirudh Venkataramanan if (unlikely(ice_test_staterr(rx_desc, stat_err_bits))) { 10552b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 10562b245cb2SAnirudh Venkataramanan continue; 10572b245cb2SAnirudh Venkataramanan } 10582b245cb2SAnirudh Venkataramanan 1059d76a60baSAnirudh Venkataramanan rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) & 1060d76a60baSAnirudh Venkataramanan ICE_RX_FLEX_DESC_PTYPE_M; 1061d76a60baSAnirudh Venkataramanan 10622b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S); 10632b245cb2SAnirudh Venkataramanan if (ice_test_staterr(rx_desc, stat_err_bits)) 10642b245cb2SAnirudh Venkataramanan vlan_tag = le16_to_cpu(rx_desc->wb.l2tag1); 10652b245cb2SAnirudh Venkataramanan 10662b245cb2SAnirudh Venkataramanan /* correct empty headers and pad skb if needed (to make valid 10672b245cb2SAnirudh Venkataramanan * ethernet frame 10682b245cb2SAnirudh Venkataramanan */ 10692b245cb2SAnirudh Venkataramanan if (ice_cleanup_headers(skb)) { 10702b245cb2SAnirudh Venkataramanan skb = NULL; 10712b245cb2SAnirudh Venkataramanan continue; 10722b245cb2SAnirudh Venkataramanan } 10732b245cb2SAnirudh Venkataramanan 10742b245cb2SAnirudh Venkataramanan /* probably a little skewed due to removing CRC */ 10752b245cb2SAnirudh Venkataramanan total_rx_bytes += skb->len; 10762b245cb2SAnirudh Venkataramanan 1077d76a60baSAnirudh Venkataramanan /* populate checksum, VLAN, and protocol */ 1078d76a60baSAnirudh Venkataramanan ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); 1079d76a60baSAnirudh Venkataramanan 10802b245cb2SAnirudh Venkataramanan /* send completed skb up the stack */ 10812b245cb2SAnirudh Venkataramanan ice_receive_skb(rx_ring, skb, vlan_tag); 10822b245cb2SAnirudh Venkataramanan 10832b245cb2SAnirudh Venkataramanan /* update budget accounting */ 10842b245cb2SAnirudh Venkataramanan total_rx_pkts++; 10852b245cb2SAnirudh Venkataramanan } 10862b245cb2SAnirudh Venkataramanan 10872b245cb2SAnirudh Venkataramanan /* update queue and vector specific stats */ 10882b245cb2SAnirudh Venkataramanan u64_stats_update_begin(&rx_ring->syncp); 10892b245cb2SAnirudh Venkataramanan rx_ring->stats.pkts += total_rx_pkts; 10902b245cb2SAnirudh Venkataramanan rx_ring->stats.bytes += total_rx_bytes; 10912b245cb2SAnirudh Venkataramanan u64_stats_update_end(&rx_ring->syncp); 10922b245cb2SAnirudh Venkataramanan rx_ring->q_vector->rx.total_pkts += total_rx_pkts; 10932b245cb2SAnirudh Venkataramanan rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 10942b245cb2SAnirudh Venkataramanan 10952b245cb2SAnirudh Venkataramanan /* guarantee a trip back through this routine if there was a failure */ 10962b245cb2SAnirudh Venkataramanan return failure ? budget : (int)total_rx_pkts; 10972b245cb2SAnirudh Venkataramanan } 10982b245cb2SAnirudh Venkataramanan 109964a59d05SAnirudh Venkataramanan static unsigned int ice_itr_divisor(struct ice_port_info *pi) 110064a59d05SAnirudh Venkataramanan { 110164a59d05SAnirudh Venkataramanan switch (pi->phy.link_info.link_speed) { 110264a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_40GB: 110364a59d05SAnirudh Venkataramanan return ICE_ITR_ADAPTIVE_MIN_INC * 1024; 110464a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_25GB: 110564a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_20GB: 110664a59d05SAnirudh Venkataramanan return ICE_ITR_ADAPTIVE_MIN_INC * 512; 110764a59d05SAnirudh Venkataramanan case ICE_AQ_LINK_SPEED_100MB: 110864a59d05SAnirudh Venkataramanan return ICE_ITR_ADAPTIVE_MIN_INC * 32; 110964a59d05SAnirudh Venkataramanan default: 111064a59d05SAnirudh Venkataramanan return ICE_ITR_ADAPTIVE_MIN_INC * 256; 111164a59d05SAnirudh Venkataramanan } 111264a59d05SAnirudh Venkataramanan } 111364a59d05SAnirudh Venkataramanan 111464a59d05SAnirudh Venkataramanan /** 111564a59d05SAnirudh Venkataramanan * ice_update_itr - update the adaptive ITR value based on statistics 111664a59d05SAnirudh Venkataramanan * @q_vector: structure containing interrupt and ring information 111764a59d05SAnirudh Venkataramanan * @rc: structure containing ring performance data 111864a59d05SAnirudh Venkataramanan * 111964a59d05SAnirudh Venkataramanan * Stores a new ITR value based on packets and byte 112064a59d05SAnirudh Venkataramanan * counts during the last interrupt. The advantage of per interrupt 112164a59d05SAnirudh Venkataramanan * computation is faster updates and more accurate ITR for the current 112264a59d05SAnirudh Venkataramanan * traffic pattern. Constants in this function were computed 112364a59d05SAnirudh Venkataramanan * based on theoretical maximum wire speed and thresholds were set based 112464a59d05SAnirudh Venkataramanan * on testing data as well as attempting to minimize response time 112564a59d05SAnirudh Venkataramanan * while increasing bulk throughput. 112664a59d05SAnirudh Venkataramanan */ 112764a59d05SAnirudh Venkataramanan static void 112864a59d05SAnirudh Venkataramanan ice_update_itr(struct ice_q_vector *q_vector, struct ice_ring_container *rc) 112964a59d05SAnirudh Venkataramanan { 113064a59d05SAnirudh Venkataramanan unsigned int avg_wire_size, packets, bytes, itr; 113164a59d05SAnirudh Venkataramanan unsigned long next_update = jiffies; 113264a59d05SAnirudh Venkataramanan bool container_is_rx; 113364a59d05SAnirudh Venkataramanan 113464a59d05SAnirudh Venkataramanan if (!rc->ring || !ITR_IS_DYNAMIC(rc->itr_setting)) 113564a59d05SAnirudh Venkataramanan return; 113664a59d05SAnirudh Venkataramanan 113764a59d05SAnirudh Venkataramanan /* If itr_countdown is set it means we programmed an ITR within 113864a59d05SAnirudh Venkataramanan * the last 4 interrupt cycles. This has a side effect of us 113964a59d05SAnirudh Venkataramanan * potentially firing an early interrupt. In order to work around 114064a59d05SAnirudh Venkataramanan * this we need to throw out any data received for a few 114164a59d05SAnirudh Venkataramanan * interrupts following the update. 114264a59d05SAnirudh Venkataramanan */ 114364a59d05SAnirudh Venkataramanan if (q_vector->itr_countdown) { 114464a59d05SAnirudh Venkataramanan itr = rc->target_itr; 114564a59d05SAnirudh Venkataramanan goto clear_counts; 114664a59d05SAnirudh Venkataramanan } 114764a59d05SAnirudh Venkataramanan 114864a59d05SAnirudh Venkataramanan container_is_rx = (&q_vector->rx == rc); 114964a59d05SAnirudh Venkataramanan /* For Rx we want to push the delay up and default to low latency. 115064a59d05SAnirudh Venkataramanan * for Tx we want to pull the delay down and default to high latency. 115164a59d05SAnirudh Venkataramanan */ 115264a59d05SAnirudh Venkataramanan itr = container_is_rx ? 115364a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MIN_USECS | ICE_ITR_ADAPTIVE_LATENCY : 115464a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MAX_USECS | ICE_ITR_ADAPTIVE_LATENCY; 115564a59d05SAnirudh Venkataramanan 115664a59d05SAnirudh Venkataramanan /* If we didn't update within up to 1 - 2 jiffies we can assume 115764a59d05SAnirudh Venkataramanan * that either packets are coming in so slow there hasn't been 115864a59d05SAnirudh Venkataramanan * any work, or that there is so much work that NAPI is dealing 115964a59d05SAnirudh Venkataramanan * with interrupt moderation and we don't need to do anything. 116064a59d05SAnirudh Venkataramanan */ 116164a59d05SAnirudh Venkataramanan if (time_after(next_update, rc->next_update)) 116264a59d05SAnirudh Venkataramanan goto clear_counts; 116364a59d05SAnirudh Venkataramanan 116464a59d05SAnirudh Venkataramanan packets = rc->total_pkts; 116564a59d05SAnirudh Venkataramanan bytes = rc->total_bytes; 116664a59d05SAnirudh Venkataramanan 116764a59d05SAnirudh Venkataramanan if (container_is_rx) { 116864a59d05SAnirudh Venkataramanan /* If Rx there are 1 to 4 packets and bytes are less than 116964a59d05SAnirudh Venkataramanan * 9000 assume insufficient data to use bulk rate limiting 117064a59d05SAnirudh Venkataramanan * approach unless Tx is already in bulk rate limiting. We 117164a59d05SAnirudh Venkataramanan * are likely latency driven. 117264a59d05SAnirudh Venkataramanan */ 117364a59d05SAnirudh Venkataramanan if (packets && packets < 4 && bytes < 9000 && 117464a59d05SAnirudh Venkataramanan (q_vector->tx.target_itr & ICE_ITR_ADAPTIVE_LATENCY)) { 117564a59d05SAnirudh Venkataramanan itr = ICE_ITR_ADAPTIVE_LATENCY; 117664a59d05SAnirudh Venkataramanan goto adjust_by_size; 117764a59d05SAnirudh Venkataramanan } 117864a59d05SAnirudh Venkataramanan } else if (packets < 4) { 117964a59d05SAnirudh Venkataramanan /* If we have Tx and Rx ITR maxed and Tx ITR is running in 118064a59d05SAnirudh Venkataramanan * bulk mode and we are receiving 4 or fewer packets just 118164a59d05SAnirudh Venkataramanan * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so 118264a59d05SAnirudh Venkataramanan * that the Rx can relax. 118364a59d05SAnirudh Venkataramanan */ 118464a59d05SAnirudh Venkataramanan if (rc->target_itr == ICE_ITR_ADAPTIVE_MAX_USECS && 118564a59d05SAnirudh Venkataramanan (q_vector->rx.target_itr & ICE_ITR_MASK) == 118664a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MAX_USECS) 118764a59d05SAnirudh Venkataramanan goto clear_counts; 118864a59d05SAnirudh Venkataramanan } else if (packets > 32) { 118964a59d05SAnirudh Venkataramanan /* If we have processed over 32 packets in a single interrupt 119064a59d05SAnirudh Venkataramanan * for Tx assume we need to switch over to "bulk" mode. 119164a59d05SAnirudh Venkataramanan */ 119264a59d05SAnirudh Venkataramanan rc->target_itr &= ~ICE_ITR_ADAPTIVE_LATENCY; 119364a59d05SAnirudh Venkataramanan } 119464a59d05SAnirudh Venkataramanan 119564a59d05SAnirudh Venkataramanan /* We have no packets to actually measure against. This means 119664a59d05SAnirudh Venkataramanan * either one of the other queues on this vector is active or 119764a59d05SAnirudh Venkataramanan * we are a Tx queue doing TSO with too high of an interrupt rate. 119864a59d05SAnirudh Venkataramanan * 119964a59d05SAnirudh Venkataramanan * Between 4 and 56 we can assume that our current interrupt delay 120064a59d05SAnirudh Venkataramanan * is only slightly too low. As such we should increase it by a small 120164a59d05SAnirudh Venkataramanan * fixed amount. 120264a59d05SAnirudh Venkataramanan */ 120364a59d05SAnirudh Venkataramanan if (packets < 56) { 120464a59d05SAnirudh Venkataramanan itr = rc->target_itr + ICE_ITR_ADAPTIVE_MIN_INC; 120564a59d05SAnirudh Venkataramanan if ((itr & ICE_ITR_MASK) > ICE_ITR_ADAPTIVE_MAX_USECS) { 120664a59d05SAnirudh Venkataramanan itr &= ICE_ITR_ADAPTIVE_LATENCY; 120764a59d05SAnirudh Venkataramanan itr += ICE_ITR_ADAPTIVE_MAX_USECS; 120864a59d05SAnirudh Venkataramanan } 120964a59d05SAnirudh Venkataramanan goto clear_counts; 121064a59d05SAnirudh Venkataramanan } 121164a59d05SAnirudh Venkataramanan 121264a59d05SAnirudh Venkataramanan if (packets <= 256) { 121364a59d05SAnirudh Venkataramanan itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr); 121464a59d05SAnirudh Venkataramanan itr &= ICE_ITR_MASK; 121564a59d05SAnirudh Venkataramanan 121664a59d05SAnirudh Venkataramanan /* Between 56 and 112 is our "goldilocks" zone where we are 121764a59d05SAnirudh Venkataramanan * working out "just right". Just report that our current 121864a59d05SAnirudh Venkataramanan * ITR is good for us. 121964a59d05SAnirudh Venkataramanan */ 122064a59d05SAnirudh Venkataramanan if (packets <= 112) 122164a59d05SAnirudh Venkataramanan goto clear_counts; 122264a59d05SAnirudh Venkataramanan 122364a59d05SAnirudh Venkataramanan /* If packet count is 128 or greater we are likely looking 122464a59d05SAnirudh Venkataramanan * at a slight overrun of the delay we want. Try halving 122564a59d05SAnirudh Venkataramanan * our delay to see if that will cut the number of packets 122664a59d05SAnirudh Venkataramanan * in half per interrupt. 122764a59d05SAnirudh Venkataramanan */ 122864a59d05SAnirudh Venkataramanan itr >>= 1; 122964a59d05SAnirudh Venkataramanan itr &= ICE_ITR_MASK; 123064a59d05SAnirudh Venkataramanan if (itr < ICE_ITR_ADAPTIVE_MIN_USECS) 123164a59d05SAnirudh Venkataramanan itr = ICE_ITR_ADAPTIVE_MIN_USECS; 123264a59d05SAnirudh Venkataramanan 123364a59d05SAnirudh Venkataramanan goto clear_counts; 123464a59d05SAnirudh Venkataramanan } 123564a59d05SAnirudh Venkataramanan 123664a59d05SAnirudh Venkataramanan /* The paths below assume we are dealing with a bulk ITR since 123764a59d05SAnirudh Venkataramanan * number of packets is greater than 256. We are just going to have 123864a59d05SAnirudh Venkataramanan * to compute a value and try to bring the count under control, 123964a59d05SAnirudh Venkataramanan * though for smaller packet sizes there isn't much we can do as 124064a59d05SAnirudh Venkataramanan * NAPI polling will likely be kicking in sooner rather than later. 124164a59d05SAnirudh Venkataramanan */ 124264a59d05SAnirudh Venkataramanan itr = ICE_ITR_ADAPTIVE_BULK; 124364a59d05SAnirudh Venkataramanan 124464a59d05SAnirudh Venkataramanan adjust_by_size: 124564a59d05SAnirudh Venkataramanan /* If packet counts are 256 or greater we can assume we have a gross 124664a59d05SAnirudh Venkataramanan * overestimation of what the rate should be. Instead of trying to fine 124764a59d05SAnirudh Venkataramanan * tune it just use the formula below to try and dial in an exact value 124864a59d05SAnirudh Venkataramanan * gives the current packet size of the frame. 124964a59d05SAnirudh Venkataramanan */ 125064a59d05SAnirudh Venkataramanan avg_wire_size = bytes / packets; 125164a59d05SAnirudh Venkataramanan 125264a59d05SAnirudh Venkataramanan /* The following is a crude approximation of: 125364a59d05SAnirudh Venkataramanan * wmem_default / (size + overhead) = desired_pkts_per_int 125464a59d05SAnirudh Venkataramanan * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate 125564a59d05SAnirudh Venkataramanan * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value 125664a59d05SAnirudh Venkataramanan * 125764a59d05SAnirudh Venkataramanan * Assuming wmem_default is 212992 and overhead is 640 bytes per 125864a59d05SAnirudh Venkataramanan * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the 125964a59d05SAnirudh Venkataramanan * formula down to 126064a59d05SAnirudh Venkataramanan * 126164a59d05SAnirudh Venkataramanan * (170 * (size + 24)) / (size + 640) = ITR 126264a59d05SAnirudh Venkataramanan * 126364a59d05SAnirudh Venkataramanan * We first do some math on the packet size and then finally bitshift 126464a59d05SAnirudh Venkataramanan * by 8 after rounding up. We also have to account for PCIe link speed 126564a59d05SAnirudh Venkataramanan * difference as ITR scales based on this. 126664a59d05SAnirudh Venkataramanan */ 126764a59d05SAnirudh Venkataramanan if (avg_wire_size <= 60) { 126864a59d05SAnirudh Venkataramanan /* Start at 250k ints/sec */ 126964a59d05SAnirudh Venkataramanan avg_wire_size = 4096; 127064a59d05SAnirudh Venkataramanan } else if (avg_wire_size <= 380) { 127164a59d05SAnirudh Venkataramanan /* 250K ints/sec to 60K ints/sec */ 127264a59d05SAnirudh Venkataramanan avg_wire_size *= 40; 127364a59d05SAnirudh Venkataramanan avg_wire_size += 1696; 127464a59d05SAnirudh Venkataramanan } else if (avg_wire_size <= 1084) { 127564a59d05SAnirudh Venkataramanan /* 60K ints/sec to 36K ints/sec */ 127664a59d05SAnirudh Venkataramanan avg_wire_size *= 15; 127764a59d05SAnirudh Venkataramanan avg_wire_size += 11452; 127864a59d05SAnirudh Venkataramanan } else if (avg_wire_size <= 1980) { 127964a59d05SAnirudh Venkataramanan /* 36K ints/sec to 30K ints/sec */ 128064a59d05SAnirudh Venkataramanan avg_wire_size *= 5; 128164a59d05SAnirudh Venkataramanan avg_wire_size += 22420; 128264a59d05SAnirudh Venkataramanan } else { 128364a59d05SAnirudh Venkataramanan /* plateau at a limit of 30K ints/sec */ 128464a59d05SAnirudh Venkataramanan avg_wire_size = 32256; 128564a59d05SAnirudh Venkataramanan } 128664a59d05SAnirudh Venkataramanan 128764a59d05SAnirudh Venkataramanan /* If we are in low latency mode halve our delay which doubles the 128864a59d05SAnirudh Venkataramanan * rate to somewhere between 100K to 16K ints/sec 128964a59d05SAnirudh Venkataramanan */ 129064a59d05SAnirudh Venkataramanan if (itr & ICE_ITR_ADAPTIVE_LATENCY) 129164a59d05SAnirudh Venkataramanan avg_wire_size >>= 1; 129264a59d05SAnirudh Venkataramanan 129364a59d05SAnirudh Venkataramanan /* Resultant value is 256 times larger than it needs to be. This 129464a59d05SAnirudh Venkataramanan * gives us room to adjust the value as needed to either increase 129564a59d05SAnirudh Venkataramanan * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc. 129664a59d05SAnirudh Venkataramanan * 129764a59d05SAnirudh Venkataramanan * Use addition as we have already recorded the new latency flag 129864a59d05SAnirudh Venkataramanan * for the ITR value. 129964a59d05SAnirudh Venkataramanan */ 130064a59d05SAnirudh Venkataramanan itr += DIV_ROUND_UP(avg_wire_size, 130164a59d05SAnirudh Venkataramanan ice_itr_divisor(q_vector->vsi->port_info)) * 130264a59d05SAnirudh Venkataramanan ICE_ITR_ADAPTIVE_MIN_INC; 130364a59d05SAnirudh Venkataramanan 130464a59d05SAnirudh Venkataramanan if ((itr & ICE_ITR_MASK) > ICE_ITR_ADAPTIVE_MAX_USECS) { 130564a59d05SAnirudh Venkataramanan itr &= ICE_ITR_ADAPTIVE_LATENCY; 130664a59d05SAnirudh Venkataramanan itr += ICE_ITR_ADAPTIVE_MAX_USECS; 130764a59d05SAnirudh Venkataramanan } 130864a59d05SAnirudh Venkataramanan 130964a59d05SAnirudh Venkataramanan clear_counts: 131064a59d05SAnirudh Venkataramanan /* write back value */ 131164a59d05SAnirudh Venkataramanan rc->target_itr = itr; 131264a59d05SAnirudh Venkataramanan 131364a59d05SAnirudh Venkataramanan /* next update should occur within next jiffy */ 131464a59d05SAnirudh Venkataramanan rc->next_update = next_update + 1; 131564a59d05SAnirudh Venkataramanan 131664a59d05SAnirudh Venkataramanan rc->total_bytes = 0; 131764a59d05SAnirudh Venkataramanan rc->total_pkts = 0; 131864a59d05SAnirudh Venkataramanan } 131964a59d05SAnirudh Venkataramanan 13202b245cb2SAnirudh Venkataramanan /** 132163f545edSBrett Creeley * ice_buildreg_itr - build value for writing to the GLINT_DYN_CTL register 132263f545edSBrett Creeley * @itr_idx: interrupt throttling index 132364a59d05SAnirudh Venkataramanan * @itr: interrupt throttling value in usecs 132463f545edSBrett Creeley */ 13258244dd2dSBrett Creeley static u32 ice_buildreg_itr(u16 itr_idx, u16 itr) 132663f545edSBrett Creeley { 132764a59d05SAnirudh Venkataramanan /* The itr value is reported in microseconds, and the register value is 132864a59d05SAnirudh Venkataramanan * recorded in 2 microsecond units. For this reason we only need to 132964a59d05SAnirudh Venkataramanan * shift by the GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S to apply this 133064a59d05SAnirudh Venkataramanan * granularity as a shift instead of division. The mask makes sure the 133164a59d05SAnirudh Venkataramanan * ITR value is never odd so we don't accidentally write into the field 133264a59d05SAnirudh Venkataramanan * prior to the ITR field. 133364a59d05SAnirudh Venkataramanan */ 133464a59d05SAnirudh Venkataramanan itr &= ICE_ITR_MASK; 133564a59d05SAnirudh Venkataramanan 133663f545edSBrett Creeley return GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M | 133763f545edSBrett Creeley (itr_idx << GLINT_DYN_CTL_ITR_INDX_S) | 133864a59d05SAnirudh Venkataramanan (itr << (GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S)); 133963f545edSBrett Creeley } 134063f545edSBrett Creeley 134164a59d05SAnirudh Venkataramanan /* The act of updating the ITR will cause it to immediately trigger. In order 134264a59d05SAnirudh Venkataramanan * to prevent this from throwing off adaptive update statistics we defer the 134364a59d05SAnirudh Venkataramanan * update so that it can only happen so often. So after either Tx or Rx are 134464a59d05SAnirudh Venkataramanan * updated we make the adaptive scheme wait until either the ITR completely 134564a59d05SAnirudh Venkataramanan * expires via the next_update expiration or we have been through at least 134664a59d05SAnirudh Venkataramanan * 3 interrupts. 134764a59d05SAnirudh Venkataramanan */ 134864a59d05SAnirudh Venkataramanan #define ITR_COUNTDOWN_START 3 134964a59d05SAnirudh Venkataramanan 135063f545edSBrett Creeley /** 135163f545edSBrett Creeley * ice_update_ena_itr - Update ITR and re-enable MSIX interrupt 135263f545edSBrett Creeley * @vsi: the VSI associated with the q_vector 135363f545edSBrett Creeley * @q_vector: q_vector for which ITR is being updated and interrupt enabled 135463f545edSBrett Creeley */ 135563f545edSBrett Creeley static void 135663f545edSBrett Creeley ice_update_ena_itr(struct ice_vsi *vsi, struct ice_q_vector *q_vector) 135763f545edSBrett Creeley { 135864a59d05SAnirudh Venkataramanan struct ice_ring_container *tx = &q_vector->tx; 135964a59d05SAnirudh Venkataramanan struct ice_ring_container *rx = &q_vector->rx; 136063f545edSBrett Creeley u32 itr_val; 136163f545edSBrett Creeley 136264a59d05SAnirudh Venkataramanan /* This will do nothing if dynamic updates are not enabled */ 136364a59d05SAnirudh Venkataramanan ice_update_itr(q_vector, tx); 136464a59d05SAnirudh Venkataramanan ice_update_itr(q_vector, rx); 136564a59d05SAnirudh Venkataramanan 136663f545edSBrett Creeley /* This block of logic allows us to get away with only updating 136763f545edSBrett Creeley * one ITR value with each interrupt. The idea is to perform a 136863f545edSBrett Creeley * pseudo-lazy update with the following criteria. 136963f545edSBrett Creeley * 137063f545edSBrett Creeley * 1. Rx is given higher priority than Tx if both are in same state 137163f545edSBrett Creeley * 2. If we must reduce an ITR that is given highest priority. 137263f545edSBrett Creeley * 3. We then give priority to increasing ITR based on amount. 137363f545edSBrett Creeley */ 137464a59d05SAnirudh Venkataramanan if (rx->target_itr < rx->current_itr) { 137563f545edSBrett Creeley /* Rx ITR needs to be reduced, this is highest priority */ 137664a59d05SAnirudh Venkataramanan itr_val = ice_buildreg_itr(rx->itr_idx, rx->target_itr); 137764a59d05SAnirudh Venkataramanan rx->current_itr = rx->target_itr; 137864a59d05SAnirudh Venkataramanan q_vector->itr_countdown = ITR_COUNTDOWN_START; 137964a59d05SAnirudh Venkataramanan } else if ((tx->target_itr < tx->current_itr) || 138064a59d05SAnirudh Venkataramanan ((rx->target_itr - rx->current_itr) < 138164a59d05SAnirudh Venkataramanan (tx->target_itr - tx->current_itr))) { 138263f545edSBrett Creeley /* Tx ITR needs to be reduced, this is second priority 138363f545edSBrett Creeley * Tx ITR needs to be increased more than Rx, fourth priority 138463f545edSBrett Creeley */ 138564a59d05SAnirudh Venkataramanan itr_val = ice_buildreg_itr(tx->itr_idx, tx->target_itr); 138664a59d05SAnirudh Venkataramanan tx->current_itr = tx->target_itr; 138764a59d05SAnirudh Venkataramanan q_vector->itr_countdown = ITR_COUNTDOWN_START; 138864a59d05SAnirudh Venkataramanan } else if (rx->current_itr != rx->target_itr) { 138963f545edSBrett Creeley /* Rx ITR needs to be increased, third priority */ 139064a59d05SAnirudh Venkataramanan itr_val = ice_buildreg_itr(rx->itr_idx, rx->target_itr); 139164a59d05SAnirudh Venkataramanan rx->current_itr = rx->target_itr; 139264a59d05SAnirudh Venkataramanan q_vector->itr_countdown = ITR_COUNTDOWN_START; 139363f545edSBrett Creeley } else { 139463f545edSBrett Creeley /* Still have to re-enable the interrupts */ 139563f545edSBrett Creeley itr_val = ice_buildreg_itr(ICE_ITR_NONE, 0); 139664a59d05SAnirudh Venkataramanan if (q_vector->itr_countdown) 139764a59d05SAnirudh Venkataramanan q_vector->itr_countdown--; 139863f545edSBrett Creeley } 139963f545edSBrett Creeley 140064a59d05SAnirudh Venkataramanan if (!test_bit(__ICE_DOWN, vsi->state)) 140164a59d05SAnirudh Venkataramanan wr32(&vsi->back->hw, 140264a59d05SAnirudh Venkataramanan GLINT_DYN_CTL(vsi->hw_base_vector + q_vector->v_idx), 140364a59d05SAnirudh Venkataramanan itr_val); 140463f545edSBrett Creeley } 140563f545edSBrett Creeley 140663f545edSBrett Creeley /** 14072b245cb2SAnirudh Venkataramanan * ice_napi_poll - NAPI polling Rx/Tx cleanup routine 14082b245cb2SAnirudh Venkataramanan * @napi: napi struct with our devices info in it 14092b245cb2SAnirudh Venkataramanan * @budget: amount of work driver is allowed to do this pass, in packets 14102b245cb2SAnirudh Venkataramanan * 14112b245cb2SAnirudh Venkataramanan * This function will clean all queues associated with a q_vector. 14122b245cb2SAnirudh Venkataramanan * 14132b245cb2SAnirudh Venkataramanan * Returns the amount of work done 14142b245cb2SAnirudh Venkataramanan */ 14152b245cb2SAnirudh Venkataramanan int ice_napi_poll(struct napi_struct *napi, int budget) 14162b245cb2SAnirudh Venkataramanan { 14172b245cb2SAnirudh Venkataramanan struct ice_q_vector *q_vector = 14182b245cb2SAnirudh Venkataramanan container_of(napi, struct ice_q_vector, napi); 14192b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = q_vector->vsi; 14202b245cb2SAnirudh Venkataramanan struct ice_pf *pf = vsi->back; 14212b245cb2SAnirudh Venkataramanan bool clean_complete = true; 14222b245cb2SAnirudh Venkataramanan int budget_per_ring = 0; 14232b245cb2SAnirudh Venkataramanan struct ice_ring *ring; 14242b245cb2SAnirudh Venkataramanan int work_done = 0; 14252b245cb2SAnirudh Venkataramanan 14262b245cb2SAnirudh Venkataramanan /* Since the actual Tx work is minimal, we can give the Tx a larger 14272b245cb2SAnirudh Venkataramanan * budget and be more aggressive about cleaning up the Tx descriptors. 14282b245cb2SAnirudh Venkataramanan */ 14292b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->tx) 14302b245cb2SAnirudh Venkataramanan if (!ice_clean_tx_irq(vsi, ring, budget)) 14312b245cb2SAnirudh Venkataramanan clean_complete = false; 14322b245cb2SAnirudh Venkataramanan 14332b245cb2SAnirudh Venkataramanan /* Handle case where we are called by netpoll with a budget of 0 */ 14342b245cb2SAnirudh Venkataramanan if (budget <= 0) 14352b245cb2SAnirudh Venkataramanan return budget; 14362b245cb2SAnirudh Venkataramanan 14372b245cb2SAnirudh Venkataramanan /* We attempt to distribute budget to each Rx queue fairly, but don't 14382b245cb2SAnirudh Venkataramanan * allow the budget to go below 1 because that would exit polling early. 14392b245cb2SAnirudh Venkataramanan */ 14402b245cb2SAnirudh Venkataramanan if (q_vector->num_ring_rx) 14412b245cb2SAnirudh Venkataramanan budget_per_ring = max(budget / q_vector->num_ring_rx, 1); 14422b245cb2SAnirudh Venkataramanan 14432b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->rx) { 14442b245cb2SAnirudh Venkataramanan int cleaned; 14452b245cb2SAnirudh Venkataramanan 14462b245cb2SAnirudh Venkataramanan cleaned = ice_clean_rx_irq(ring, budget_per_ring); 14472b245cb2SAnirudh Venkataramanan work_done += cleaned; 14482b245cb2SAnirudh Venkataramanan /* if we clean as many as budgeted, we must not be done */ 14492b245cb2SAnirudh Venkataramanan if (cleaned >= budget_per_ring) 14502b245cb2SAnirudh Venkataramanan clean_complete = false; 14512b245cb2SAnirudh Venkataramanan } 14522b245cb2SAnirudh Venkataramanan 14532b245cb2SAnirudh Venkataramanan /* If work not completed, return budget and polling will return */ 14542b245cb2SAnirudh Venkataramanan if (!clean_complete) 14552b245cb2SAnirudh Venkataramanan return budget; 14562b245cb2SAnirudh Venkataramanan 14570bcd952fSJesse Brandeburg /* Exit the polling mode, but don't re-enable interrupts if stack might 14580bcd952fSJesse Brandeburg * poll us due to busy-polling 14590bcd952fSJesse Brandeburg */ 14600bcd952fSJesse Brandeburg if (likely(napi_complete_done(napi, work_done))) 14612b245cb2SAnirudh Venkataramanan if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 146263f545edSBrett Creeley ice_update_ena_itr(vsi, q_vector); 1463e0c9fd9bSDave Ertman 146432a64994SBruce Allan return min_t(int, work_done, budget - 1); 14652b245cb2SAnirudh Venkataramanan } 14662b245cb2SAnirudh Venkataramanan 14672b245cb2SAnirudh Venkataramanan /* helper function for building cmd/type/offset */ 14682b245cb2SAnirudh Venkataramanan static __le64 14692b245cb2SAnirudh Venkataramanan build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) 14702b245cb2SAnirudh Venkataramanan { 14712b245cb2SAnirudh Venkataramanan return cpu_to_le64(ICE_TX_DESC_DTYPE_DATA | 14722b245cb2SAnirudh Venkataramanan (td_cmd << ICE_TXD_QW1_CMD_S) | 14732b245cb2SAnirudh Venkataramanan (td_offset << ICE_TXD_QW1_OFFSET_S) | 14742b245cb2SAnirudh Venkataramanan ((u64)size << ICE_TXD_QW1_TX_BUF_SZ_S) | 14752b245cb2SAnirudh Venkataramanan (td_tag << ICE_TXD_QW1_L2TAG1_S)); 14762b245cb2SAnirudh Venkataramanan } 14772b245cb2SAnirudh Venkataramanan 14782b245cb2SAnirudh Venkataramanan /** 1479d337f2afSAnirudh Venkataramanan * __ice_maybe_stop_tx - 2nd level check for Tx stop conditions 14802b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 14812b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 14822b245cb2SAnirudh Venkataramanan * 14832b245cb2SAnirudh Venkataramanan * Returns -EBUSY if a stop is needed, else 0 14842b245cb2SAnirudh Venkataramanan */ 14852b245cb2SAnirudh Venkataramanan static int __ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 14862b245cb2SAnirudh Venkataramanan { 14872b245cb2SAnirudh Venkataramanan netif_stop_subqueue(tx_ring->netdev, tx_ring->q_index); 14882b245cb2SAnirudh Venkataramanan /* Memory barrier before checking head and tail */ 14892b245cb2SAnirudh Venkataramanan smp_mb(); 14902b245cb2SAnirudh Venkataramanan 14912b245cb2SAnirudh Venkataramanan /* Check again in a case another CPU has just made room available. */ 14922b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) < size)) 14932b245cb2SAnirudh Venkataramanan return -EBUSY; 14942b245cb2SAnirudh Venkataramanan 14952b245cb2SAnirudh Venkataramanan /* A reprieve! - use start_subqueue because it doesn't call schedule */ 14962b245cb2SAnirudh Venkataramanan netif_start_subqueue(tx_ring->netdev, tx_ring->q_index); 14972b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 14982b245cb2SAnirudh Venkataramanan return 0; 14992b245cb2SAnirudh Venkataramanan } 15002b245cb2SAnirudh Venkataramanan 15012b245cb2SAnirudh Venkataramanan /** 1502d337f2afSAnirudh Venkataramanan * ice_maybe_stop_tx - 1st level check for Tx stop conditions 15032b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 15042b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 15052b245cb2SAnirudh Venkataramanan * 15062b245cb2SAnirudh Venkataramanan * Returns 0 if stop is not needed 15072b245cb2SAnirudh Venkataramanan */ 15082b245cb2SAnirudh Venkataramanan static int ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 15092b245cb2SAnirudh Venkataramanan { 15102b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) >= size)) 15112b245cb2SAnirudh Venkataramanan return 0; 1512d337f2afSAnirudh Venkataramanan 15132b245cb2SAnirudh Venkataramanan return __ice_maybe_stop_tx(tx_ring, size); 15142b245cb2SAnirudh Venkataramanan } 15152b245cb2SAnirudh Venkataramanan 15162b245cb2SAnirudh Venkataramanan /** 15172b245cb2SAnirudh Venkataramanan * ice_tx_map - Build the Tx descriptor 15182b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 15192b245cb2SAnirudh Venkataramanan * @first: first buffer info buffer to use 1520d76a60baSAnirudh Venkataramanan * @off: pointer to struct that holds offload parameters 15212b245cb2SAnirudh Venkataramanan * 15222b245cb2SAnirudh Venkataramanan * This function loops over the skb data pointed to by *first 15232b245cb2SAnirudh Venkataramanan * and gets a physical address for each memory location and programs 15242b245cb2SAnirudh Venkataramanan * it and the length into the transmit descriptor. 15252b245cb2SAnirudh Venkataramanan */ 1526d76a60baSAnirudh Venkataramanan static void 1527d76a60baSAnirudh Venkataramanan ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first, 1528d76a60baSAnirudh Venkataramanan struct ice_tx_offload_params *off) 15292b245cb2SAnirudh Venkataramanan { 1530d76a60baSAnirudh Venkataramanan u64 td_offset, td_tag, td_cmd; 15312b245cb2SAnirudh Venkataramanan u16 i = tx_ring->next_to_use; 15322b245cb2SAnirudh Venkataramanan struct skb_frag_struct *frag; 15332b245cb2SAnirudh Venkataramanan unsigned int data_len, size; 15342b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 15352b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 15362b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 15372b245cb2SAnirudh Venkataramanan dma_addr_t dma; 15382b245cb2SAnirudh Venkataramanan 1539d76a60baSAnirudh Venkataramanan td_tag = off->td_l2tag1; 1540d76a60baSAnirudh Venkataramanan td_cmd = off->td_cmd; 1541d76a60baSAnirudh Venkataramanan td_offset = off->td_offset; 15422b245cb2SAnirudh Venkataramanan skb = first->skb; 15432b245cb2SAnirudh Venkataramanan 15442b245cb2SAnirudh Venkataramanan data_len = skb->data_len; 15452b245cb2SAnirudh Venkataramanan size = skb_headlen(skb); 15462b245cb2SAnirudh Venkataramanan 15472b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 15482b245cb2SAnirudh Venkataramanan 1549d76a60baSAnirudh Venkataramanan if (first->tx_flags & ICE_TX_FLAGS_HW_VLAN) { 1550d76a60baSAnirudh Venkataramanan td_cmd |= (u64)ICE_TX_DESC_CMD_IL2TAG1; 1551d76a60baSAnirudh Venkataramanan td_tag = (first->tx_flags & ICE_TX_FLAGS_VLAN_M) >> 1552d76a60baSAnirudh Venkataramanan ICE_TX_FLAGS_VLAN_S; 1553d76a60baSAnirudh Venkataramanan } 1554d76a60baSAnirudh Venkataramanan 15552b245cb2SAnirudh Venkataramanan dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 15562b245cb2SAnirudh Venkataramanan 15572b245cb2SAnirudh Venkataramanan tx_buf = first; 15582b245cb2SAnirudh Venkataramanan 15592b245cb2SAnirudh Venkataramanan for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 15602b245cb2SAnirudh Venkataramanan unsigned int max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 15612b245cb2SAnirudh Venkataramanan 15622b245cb2SAnirudh Venkataramanan if (dma_mapping_error(tx_ring->dev, dma)) 15632b245cb2SAnirudh Venkataramanan goto dma_error; 15642b245cb2SAnirudh Venkataramanan 15652b245cb2SAnirudh Venkataramanan /* record length, and DMA address */ 15662b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, size); 15672b245cb2SAnirudh Venkataramanan dma_unmap_addr_set(tx_buf, dma, dma); 15682b245cb2SAnirudh Venkataramanan 15692b245cb2SAnirudh Venkataramanan /* align size to end of page */ 15702b245cb2SAnirudh Venkataramanan max_data += -dma & (ICE_MAX_READ_REQ_SIZE - 1); 15712b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 15722b245cb2SAnirudh Venkataramanan 15732b245cb2SAnirudh Venkataramanan /* account for data chunks larger than the hardware 15742b245cb2SAnirudh Venkataramanan * can handle 15752b245cb2SAnirudh Venkataramanan */ 15762b245cb2SAnirudh Venkataramanan while (unlikely(size > ICE_MAX_DATA_PER_TXD)) { 15772b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 15782b245cb2SAnirudh Venkataramanan build_ctob(td_cmd, td_offset, max_data, td_tag); 15792b245cb2SAnirudh Venkataramanan 15802b245cb2SAnirudh Venkataramanan tx_desc++; 15812b245cb2SAnirudh Venkataramanan i++; 15822b245cb2SAnirudh Venkataramanan 15832b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 15842b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 15852b245cb2SAnirudh Venkataramanan i = 0; 15862b245cb2SAnirudh Venkataramanan } 15872b245cb2SAnirudh Venkataramanan 15882b245cb2SAnirudh Venkataramanan dma += max_data; 15892b245cb2SAnirudh Venkataramanan size -= max_data; 15902b245cb2SAnirudh Venkataramanan 15912b245cb2SAnirudh Venkataramanan max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 15922b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 15932b245cb2SAnirudh Venkataramanan } 15942b245cb2SAnirudh Venkataramanan 15952b245cb2SAnirudh Venkataramanan if (likely(!data_len)) 15962b245cb2SAnirudh Venkataramanan break; 15972b245cb2SAnirudh Venkataramanan 15982b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 15992b245cb2SAnirudh Venkataramanan size, td_tag); 16002b245cb2SAnirudh Venkataramanan 16012b245cb2SAnirudh Venkataramanan tx_desc++; 16022b245cb2SAnirudh Venkataramanan i++; 16032b245cb2SAnirudh Venkataramanan 16042b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 16052b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 16062b245cb2SAnirudh Venkataramanan i = 0; 16072b245cb2SAnirudh Venkataramanan } 16082b245cb2SAnirudh Venkataramanan 16092b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag); 16102b245cb2SAnirudh Venkataramanan data_len -= size; 16112b245cb2SAnirudh Venkataramanan 16122b245cb2SAnirudh Venkataramanan dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 16132b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 16142b245cb2SAnirudh Venkataramanan 16152b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 16162b245cb2SAnirudh Venkataramanan } 16172b245cb2SAnirudh Venkataramanan 16182b245cb2SAnirudh Venkataramanan /* record bytecount for BQL */ 16192b245cb2SAnirudh Venkataramanan netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 16202b245cb2SAnirudh Venkataramanan 16212b245cb2SAnirudh Venkataramanan /* record SW timestamp if HW timestamp is not available */ 16222b245cb2SAnirudh Venkataramanan skb_tx_timestamp(first->skb); 16232b245cb2SAnirudh Venkataramanan 16242b245cb2SAnirudh Venkataramanan i++; 16252b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) 16262b245cb2SAnirudh Venkataramanan i = 0; 16272b245cb2SAnirudh Venkataramanan 16282b245cb2SAnirudh Venkataramanan /* write last descriptor with RS and EOP bits */ 16292b245cb2SAnirudh Venkataramanan td_cmd |= (u64)(ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS); 16302b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 16312b245cb2SAnirudh Venkataramanan build_ctob(td_cmd, td_offset, size, td_tag); 16322b245cb2SAnirudh Venkataramanan 16332b245cb2SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w know there 16342b245cb2SAnirudh Venkataramanan * are new descriptors to fetch. 16352b245cb2SAnirudh Venkataramanan * 16362b245cb2SAnirudh Venkataramanan * We also use this memory barrier to make certain all of the 16372b245cb2SAnirudh Venkataramanan * status bits have been updated before next_to_watch is written. 16382b245cb2SAnirudh Venkataramanan */ 16392b245cb2SAnirudh Venkataramanan wmb(); 16402b245cb2SAnirudh Venkataramanan 16412b245cb2SAnirudh Venkataramanan /* set next_to_watch value indicating a packet is present */ 16422b245cb2SAnirudh Venkataramanan first->next_to_watch = tx_desc; 16432b245cb2SAnirudh Venkataramanan 16442b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 16452b245cb2SAnirudh Venkataramanan 16462b245cb2SAnirudh Venkataramanan ice_maybe_stop_tx(tx_ring, DESC_NEEDED); 16472b245cb2SAnirudh Venkataramanan 16482b245cb2SAnirudh Venkataramanan /* notify HW of packet */ 16496b16f9eeSFlorian Westphal if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 16502b245cb2SAnirudh Venkataramanan writel(i, tx_ring->tail); 16512b245cb2SAnirudh Venkataramanan 16522b245cb2SAnirudh Venkataramanan /* we need this if more than one processor can write to our tail 16532b245cb2SAnirudh Venkataramanan * at a time, it synchronizes IO on IA64/Altix systems 16542b245cb2SAnirudh Venkataramanan */ 16552b245cb2SAnirudh Venkataramanan mmiowb(); 16562b245cb2SAnirudh Venkataramanan } 16572b245cb2SAnirudh Venkataramanan 16582b245cb2SAnirudh Venkataramanan return; 16592b245cb2SAnirudh Venkataramanan 16602b245cb2SAnirudh Venkataramanan dma_error: 16612b245cb2SAnirudh Venkataramanan /* clear dma mappings for failed tx_buf map */ 16622b245cb2SAnirudh Venkataramanan for (;;) { 16632b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 16642b245cb2SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, tx_buf); 16652b245cb2SAnirudh Venkataramanan if (tx_buf == first) 16662b245cb2SAnirudh Venkataramanan break; 16672b245cb2SAnirudh Venkataramanan if (i == 0) 16682b245cb2SAnirudh Venkataramanan i = tx_ring->count; 16692b245cb2SAnirudh Venkataramanan i--; 16702b245cb2SAnirudh Venkataramanan } 16712b245cb2SAnirudh Venkataramanan 16722b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 16732b245cb2SAnirudh Venkataramanan } 16742b245cb2SAnirudh Venkataramanan 16752b245cb2SAnirudh Venkataramanan /** 1676d76a60baSAnirudh Venkataramanan * ice_tx_csum - Enable Tx checksum offloads 1677d76a60baSAnirudh Venkataramanan * @first: pointer to the first descriptor 1678d76a60baSAnirudh Venkataramanan * @off: pointer to struct that holds offload parameters 1679d76a60baSAnirudh Venkataramanan * 1680d76a60baSAnirudh Venkataramanan * Returns 0 or error (negative) if checksum offload can't happen, 1 otherwise. 1681d76a60baSAnirudh Venkataramanan */ 1682d76a60baSAnirudh Venkataramanan static 1683d76a60baSAnirudh Venkataramanan int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) 1684d76a60baSAnirudh Venkataramanan { 1685d76a60baSAnirudh Venkataramanan u32 l4_len = 0, l3_len = 0, l2_len = 0; 1686d76a60baSAnirudh Venkataramanan struct sk_buff *skb = first->skb; 1687d76a60baSAnirudh Venkataramanan union { 1688d76a60baSAnirudh Venkataramanan struct iphdr *v4; 1689d76a60baSAnirudh Venkataramanan struct ipv6hdr *v6; 1690d76a60baSAnirudh Venkataramanan unsigned char *hdr; 1691d76a60baSAnirudh Venkataramanan } ip; 1692d76a60baSAnirudh Venkataramanan union { 1693d76a60baSAnirudh Venkataramanan struct tcphdr *tcp; 1694d76a60baSAnirudh Venkataramanan unsigned char *hdr; 1695d76a60baSAnirudh Venkataramanan } l4; 1696d76a60baSAnirudh Venkataramanan __be16 frag_off, protocol; 1697d76a60baSAnirudh Venkataramanan unsigned char *exthdr; 1698d76a60baSAnirudh Venkataramanan u32 offset, cmd = 0; 1699d76a60baSAnirudh Venkataramanan u8 l4_proto = 0; 1700d76a60baSAnirudh Venkataramanan 1701d76a60baSAnirudh Venkataramanan if (skb->ip_summed != CHECKSUM_PARTIAL) 1702d76a60baSAnirudh Venkataramanan return 0; 1703d76a60baSAnirudh Venkataramanan 1704d76a60baSAnirudh Venkataramanan ip.hdr = skb_network_header(skb); 1705d76a60baSAnirudh Venkataramanan l4.hdr = skb_transport_header(skb); 1706d76a60baSAnirudh Venkataramanan 1707d76a60baSAnirudh Venkataramanan /* compute outer L2 header size */ 1708d76a60baSAnirudh Venkataramanan l2_len = ip.hdr - skb->data; 1709d76a60baSAnirudh Venkataramanan offset = (l2_len / 2) << ICE_TX_DESC_LEN_MACLEN_S; 1710d76a60baSAnirudh Venkataramanan 1711d76a60baSAnirudh Venkataramanan if (skb->encapsulation) 1712d76a60baSAnirudh Venkataramanan return -1; 1713d76a60baSAnirudh Venkataramanan 1714d76a60baSAnirudh Venkataramanan /* Enable IP checksum offloads */ 1715d76a60baSAnirudh Venkataramanan protocol = vlan_get_protocol(skb); 1716d76a60baSAnirudh Venkataramanan if (protocol == htons(ETH_P_IP)) { 1717d76a60baSAnirudh Venkataramanan l4_proto = ip.v4->protocol; 1718d76a60baSAnirudh Venkataramanan /* the stack computes the IP header already, the only time we 1719d76a60baSAnirudh Venkataramanan * need the hardware to recompute it is in the case of TSO. 1720d76a60baSAnirudh Venkataramanan */ 1721d76a60baSAnirudh Venkataramanan if (first->tx_flags & ICE_TX_FLAGS_TSO) 1722d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 1723d76a60baSAnirudh Venkataramanan else 1724d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 1725d76a60baSAnirudh Venkataramanan 1726d76a60baSAnirudh Venkataramanan } else if (protocol == htons(ETH_P_IPV6)) { 1727d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 1728d76a60baSAnirudh Venkataramanan exthdr = ip.hdr + sizeof(*ip.v6); 1729d76a60baSAnirudh Venkataramanan l4_proto = ip.v6->nexthdr; 1730d76a60baSAnirudh Venkataramanan if (l4.hdr != exthdr) 1731d76a60baSAnirudh Venkataramanan ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto, 1732d76a60baSAnirudh Venkataramanan &frag_off); 1733d76a60baSAnirudh Venkataramanan } else { 1734d76a60baSAnirudh Venkataramanan return -1; 1735d76a60baSAnirudh Venkataramanan } 1736d76a60baSAnirudh Venkataramanan 1737d76a60baSAnirudh Venkataramanan /* compute inner L3 header size */ 1738d76a60baSAnirudh Venkataramanan l3_len = l4.hdr - ip.hdr; 1739d76a60baSAnirudh Venkataramanan offset |= (l3_len / 4) << ICE_TX_DESC_LEN_IPLEN_S; 1740d76a60baSAnirudh Venkataramanan 1741d76a60baSAnirudh Venkataramanan /* Enable L4 checksum offloads */ 1742d76a60baSAnirudh Venkataramanan switch (l4_proto) { 1743d76a60baSAnirudh Venkataramanan case IPPROTO_TCP: 1744d76a60baSAnirudh Venkataramanan /* enable checksum offloads */ 1745d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 1746d76a60baSAnirudh Venkataramanan l4_len = l4.tcp->doff; 1747d76a60baSAnirudh Venkataramanan offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S; 1748d76a60baSAnirudh Venkataramanan break; 1749d76a60baSAnirudh Venkataramanan case IPPROTO_UDP: 1750d76a60baSAnirudh Venkataramanan /* enable UDP checksum offload */ 1751d76a60baSAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 1752d76a60baSAnirudh Venkataramanan l4_len = (sizeof(struct udphdr) >> 2); 1753d76a60baSAnirudh Venkataramanan offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S; 1754d76a60baSAnirudh Venkataramanan break; 1755d76a60baSAnirudh Venkataramanan case IPPROTO_SCTP: 1756cf909e19SAnirudh Venkataramanan /* enable SCTP checksum offload */ 1757cf909e19SAnirudh Venkataramanan cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 1758cf909e19SAnirudh Venkataramanan l4_len = sizeof(struct sctphdr) >> 2; 1759cf909e19SAnirudh Venkataramanan offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S; 1760cf909e19SAnirudh Venkataramanan break; 1761cf909e19SAnirudh Venkataramanan 1762d76a60baSAnirudh Venkataramanan default: 1763d76a60baSAnirudh Venkataramanan if (first->tx_flags & ICE_TX_FLAGS_TSO) 1764d76a60baSAnirudh Venkataramanan return -1; 1765d76a60baSAnirudh Venkataramanan skb_checksum_help(skb); 1766d76a60baSAnirudh Venkataramanan return 0; 1767d76a60baSAnirudh Venkataramanan } 1768d76a60baSAnirudh Venkataramanan 1769d76a60baSAnirudh Venkataramanan off->td_cmd |= cmd; 1770d76a60baSAnirudh Venkataramanan off->td_offset |= offset; 1771d76a60baSAnirudh Venkataramanan return 1; 1772d76a60baSAnirudh Venkataramanan } 1773d76a60baSAnirudh Venkataramanan 1774d76a60baSAnirudh Venkataramanan /** 1775d76a60baSAnirudh Venkataramanan * ice_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW 1776d76a60baSAnirudh Venkataramanan * @tx_ring: ring to send buffer on 1777d76a60baSAnirudh Venkataramanan * @first: pointer to struct ice_tx_buf 1778d76a60baSAnirudh Venkataramanan * 1779d76a60baSAnirudh Venkataramanan * Checks the skb and set up correspondingly several generic transmit flags 1780d76a60baSAnirudh Venkataramanan * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 1781d76a60baSAnirudh Venkataramanan * 1782d76a60baSAnirudh Venkataramanan * Returns error code indicate the frame should be dropped upon error and the 1783d76a60baSAnirudh Venkataramanan * otherwise returns 0 to indicate the flags has been set properly. 1784d76a60baSAnirudh Venkataramanan */ 1785d76a60baSAnirudh Venkataramanan static int 1786d76a60baSAnirudh Venkataramanan ice_tx_prepare_vlan_flags(struct ice_ring *tx_ring, struct ice_tx_buf *first) 1787d76a60baSAnirudh Venkataramanan { 1788d76a60baSAnirudh Venkataramanan struct sk_buff *skb = first->skb; 1789d76a60baSAnirudh Venkataramanan __be16 protocol = skb->protocol; 1790d76a60baSAnirudh Venkataramanan 1791d76a60baSAnirudh Venkataramanan if (protocol == htons(ETH_P_8021Q) && 1792d76a60baSAnirudh Venkataramanan !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { 1793d76a60baSAnirudh Venkataramanan /* when HW VLAN acceleration is turned off by the user the 1794d76a60baSAnirudh Venkataramanan * stack sets the protocol to 8021q so that the driver 1795d76a60baSAnirudh Venkataramanan * can take any steps required to support the SW only 1796d76a60baSAnirudh Venkataramanan * VLAN handling. In our case the driver doesn't need 1797d76a60baSAnirudh Venkataramanan * to take any further steps so just set the protocol 1798d76a60baSAnirudh Venkataramanan * to the encapsulated ethertype. 1799d76a60baSAnirudh Venkataramanan */ 1800d76a60baSAnirudh Venkataramanan skb->protocol = vlan_get_protocol(skb); 1801d76a60baSAnirudh Venkataramanan goto out; 1802d76a60baSAnirudh Venkataramanan } 1803d76a60baSAnirudh Venkataramanan 1804d76a60baSAnirudh Venkataramanan /* if we have a HW VLAN tag being added, default to the HW one */ 1805d76a60baSAnirudh Venkataramanan if (skb_vlan_tag_present(skb)) { 1806d76a60baSAnirudh Venkataramanan first->tx_flags |= skb_vlan_tag_get(skb) << ICE_TX_FLAGS_VLAN_S; 1807d76a60baSAnirudh Venkataramanan first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; 1808d76a60baSAnirudh Venkataramanan } else if (protocol == htons(ETH_P_8021Q)) { 1809d76a60baSAnirudh Venkataramanan struct vlan_hdr *vhdr, _vhdr; 1810d76a60baSAnirudh Venkataramanan 1811d76a60baSAnirudh Venkataramanan /* for SW VLAN, check the next protocol and store the tag */ 1812d76a60baSAnirudh Venkataramanan vhdr = (struct vlan_hdr *)skb_header_pointer(skb, ETH_HLEN, 1813d76a60baSAnirudh Venkataramanan sizeof(_vhdr), 1814d76a60baSAnirudh Venkataramanan &_vhdr); 1815d76a60baSAnirudh Venkataramanan if (!vhdr) 1816d76a60baSAnirudh Venkataramanan return -EINVAL; 1817d76a60baSAnirudh Venkataramanan 1818d76a60baSAnirudh Venkataramanan first->tx_flags |= ntohs(vhdr->h_vlan_TCI) << 1819d76a60baSAnirudh Venkataramanan ICE_TX_FLAGS_VLAN_S; 1820d76a60baSAnirudh Venkataramanan first->tx_flags |= ICE_TX_FLAGS_SW_VLAN; 1821d76a60baSAnirudh Venkataramanan } 1822d76a60baSAnirudh Venkataramanan 1823d76a60baSAnirudh Venkataramanan out: 1824d76a60baSAnirudh Venkataramanan return 0; 1825d76a60baSAnirudh Venkataramanan } 1826d76a60baSAnirudh Venkataramanan 1827d76a60baSAnirudh Venkataramanan /** 1828d76a60baSAnirudh Venkataramanan * ice_tso - computes mss and TSO length to prepare for TSO 1829d76a60baSAnirudh Venkataramanan * @first: pointer to struct ice_tx_buf 1830d76a60baSAnirudh Venkataramanan * @off: pointer to struct that holds offload parameters 1831d76a60baSAnirudh Venkataramanan * 1832d76a60baSAnirudh Venkataramanan * Returns 0 or error (negative) if TSO can't happen, 1 otherwise. 1833d76a60baSAnirudh Venkataramanan */ 1834d76a60baSAnirudh Venkataramanan static 1835d76a60baSAnirudh Venkataramanan int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off) 1836d76a60baSAnirudh Venkataramanan { 1837d76a60baSAnirudh Venkataramanan struct sk_buff *skb = first->skb; 1838d76a60baSAnirudh Venkataramanan union { 1839d76a60baSAnirudh Venkataramanan struct iphdr *v4; 1840d76a60baSAnirudh Venkataramanan struct ipv6hdr *v6; 1841d76a60baSAnirudh Venkataramanan unsigned char *hdr; 1842d76a60baSAnirudh Venkataramanan } ip; 1843d76a60baSAnirudh Venkataramanan union { 1844d76a60baSAnirudh Venkataramanan struct tcphdr *tcp; 1845d76a60baSAnirudh Venkataramanan unsigned char *hdr; 1846d76a60baSAnirudh Venkataramanan } l4; 1847d76a60baSAnirudh Venkataramanan u64 cd_mss, cd_tso_len; 1848d76a60baSAnirudh Venkataramanan u32 paylen, l4_start; 1849d76a60baSAnirudh Venkataramanan int err; 1850d76a60baSAnirudh Venkataramanan 1851d76a60baSAnirudh Venkataramanan if (skb->ip_summed != CHECKSUM_PARTIAL) 1852d76a60baSAnirudh Venkataramanan return 0; 1853d76a60baSAnirudh Venkataramanan 1854d76a60baSAnirudh Venkataramanan if (!skb_is_gso(skb)) 1855d76a60baSAnirudh Venkataramanan return 0; 1856d76a60baSAnirudh Venkataramanan 1857d76a60baSAnirudh Venkataramanan err = skb_cow_head(skb, 0); 1858d76a60baSAnirudh Venkataramanan if (err < 0) 1859d76a60baSAnirudh Venkataramanan return err; 1860d76a60baSAnirudh Venkataramanan 1861d76a60baSAnirudh Venkataramanan ip.hdr = skb_network_header(skb); 1862d76a60baSAnirudh Venkataramanan l4.hdr = skb_transport_header(skb); 1863d76a60baSAnirudh Venkataramanan 1864d76a60baSAnirudh Venkataramanan /* initialize outer IP header fields */ 1865d76a60baSAnirudh Venkataramanan if (ip.v4->version == 4) { 1866d76a60baSAnirudh Venkataramanan ip.v4->tot_len = 0; 1867d76a60baSAnirudh Venkataramanan ip.v4->check = 0; 1868d76a60baSAnirudh Venkataramanan } else { 1869d76a60baSAnirudh Venkataramanan ip.v6->payload_len = 0; 1870d76a60baSAnirudh Venkataramanan } 1871d76a60baSAnirudh Venkataramanan 1872d76a60baSAnirudh Venkataramanan /* determine offset of transport header */ 1873d76a60baSAnirudh Venkataramanan l4_start = l4.hdr - skb->data; 1874d76a60baSAnirudh Venkataramanan 1875d76a60baSAnirudh Venkataramanan /* remove payload length from checksum */ 1876d76a60baSAnirudh Venkataramanan paylen = skb->len - l4_start; 1877d76a60baSAnirudh Venkataramanan csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); 1878d76a60baSAnirudh Venkataramanan 1879d76a60baSAnirudh Venkataramanan /* compute length of segmentation header */ 1880d76a60baSAnirudh Venkataramanan off->header_len = (l4.tcp->doff * 4) + l4_start; 1881d76a60baSAnirudh Venkataramanan 1882d76a60baSAnirudh Venkataramanan /* update gso_segs and bytecount */ 1883d76a60baSAnirudh Venkataramanan first->gso_segs = skb_shinfo(skb)->gso_segs; 1884d944b469SBrett Creeley first->bytecount += (first->gso_segs - 1) * off->header_len; 1885d76a60baSAnirudh Venkataramanan 1886d76a60baSAnirudh Venkataramanan cd_tso_len = skb->len - off->header_len; 1887d76a60baSAnirudh Venkataramanan cd_mss = skb_shinfo(skb)->gso_size; 1888d76a60baSAnirudh Venkataramanan 1889d76a60baSAnirudh Venkataramanan /* record cdesc_qw1 with TSO parameters */ 1890d76a60baSAnirudh Venkataramanan off->cd_qw1 |= ICE_TX_DESC_DTYPE_CTX | 1891d76a60baSAnirudh Venkataramanan (ICE_TX_CTX_DESC_TSO << ICE_TXD_CTX_QW1_CMD_S) | 1892d76a60baSAnirudh Venkataramanan (cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) | 1893d76a60baSAnirudh Venkataramanan (cd_mss << ICE_TXD_CTX_QW1_MSS_S); 1894d76a60baSAnirudh Venkataramanan first->tx_flags |= ICE_TX_FLAGS_TSO; 1895d76a60baSAnirudh Venkataramanan return 1; 1896d76a60baSAnirudh Venkataramanan } 1897d76a60baSAnirudh Venkataramanan 1898d76a60baSAnirudh Venkataramanan /** 18992b245cb2SAnirudh Venkataramanan * ice_txd_use_count - estimate the number of descriptors needed for Tx 19002b245cb2SAnirudh Venkataramanan * @size: transmit request size in bytes 19012b245cb2SAnirudh Venkataramanan * 19022b245cb2SAnirudh Venkataramanan * Due to hardware alignment restrictions (4K alignment), we need to 19032b245cb2SAnirudh Venkataramanan * assume that we can have no more than 12K of data per descriptor, even 19042b245cb2SAnirudh Venkataramanan * though each descriptor can take up to 16K - 1 bytes of aligned memory. 19052b245cb2SAnirudh Venkataramanan * Thus, we need to divide by 12K. But division is slow! Instead, 19062b245cb2SAnirudh Venkataramanan * we decompose the operation into shifts and one relatively cheap 19072b245cb2SAnirudh Venkataramanan * multiply operation. 19082b245cb2SAnirudh Venkataramanan * 19092b245cb2SAnirudh Venkataramanan * To divide by 12K, we first divide by 4K, then divide by 3: 19102b245cb2SAnirudh Venkataramanan * To divide by 4K, shift right by 12 bits 19112b245cb2SAnirudh Venkataramanan * To divide by 3, multiply by 85, then divide by 256 19122b245cb2SAnirudh Venkataramanan * (Divide by 256 is done by shifting right by 8 bits) 19132b245cb2SAnirudh Venkataramanan * Finally, we add one to round up. Because 256 isn't an exact multiple of 19142b245cb2SAnirudh Venkataramanan * 3, we'll underestimate near each multiple of 12K. This is actually more 19152b245cb2SAnirudh Venkataramanan * accurate as we have 4K - 1 of wiggle room that we can fit into the last 19162b245cb2SAnirudh Venkataramanan * segment. For our purposes this is accurate out to 1M which is orders of 19172b245cb2SAnirudh Venkataramanan * magnitude greater than our largest possible GSO size. 19182b245cb2SAnirudh Venkataramanan * 19192b245cb2SAnirudh Venkataramanan * This would then be implemented as: 1920c585ea42SBrett Creeley * return (((size >> 12) * 85) >> 8) + ICE_DESCS_FOR_SKB_DATA_PTR; 19212b245cb2SAnirudh Venkataramanan * 19222b245cb2SAnirudh Venkataramanan * Since multiplication and division are commutative, we can reorder 19232b245cb2SAnirudh Venkataramanan * operations into: 1924c585ea42SBrett Creeley * return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR; 19252b245cb2SAnirudh Venkataramanan */ 19262b245cb2SAnirudh Venkataramanan static unsigned int ice_txd_use_count(unsigned int size) 19272b245cb2SAnirudh Venkataramanan { 1928c585ea42SBrett Creeley return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR; 19292b245cb2SAnirudh Venkataramanan } 19302b245cb2SAnirudh Venkataramanan 19312b245cb2SAnirudh Venkataramanan /** 1932d337f2afSAnirudh Venkataramanan * ice_xmit_desc_count - calculate number of Tx descriptors needed 19332b245cb2SAnirudh Venkataramanan * @skb: send buffer 19342b245cb2SAnirudh Venkataramanan * 19352b245cb2SAnirudh Venkataramanan * Returns number of data descriptors needed for this skb. 19362b245cb2SAnirudh Venkataramanan */ 19372b245cb2SAnirudh Venkataramanan static unsigned int ice_xmit_desc_count(struct sk_buff *skb) 19382b245cb2SAnirudh Venkataramanan { 19392b245cb2SAnirudh Venkataramanan const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 19402b245cb2SAnirudh Venkataramanan unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 19412b245cb2SAnirudh Venkataramanan unsigned int count = 0, size = skb_headlen(skb); 19422b245cb2SAnirudh Venkataramanan 19432b245cb2SAnirudh Venkataramanan for (;;) { 19442b245cb2SAnirudh Venkataramanan count += ice_txd_use_count(size); 19452b245cb2SAnirudh Venkataramanan 19462b245cb2SAnirudh Venkataramanan if (!nr_frags--) 19472b245cb2SAnirudh Venkataramanan break; 19482b245cb2SAnirudh Venkataramanan 19492b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag++); 19502b245cb2SAnirudh Venkataramanan } 19512b245cb2SAnirudh Venkataramanan 19522b245cb2SAnirudh Venkataramanan return count; 19532b245cb2SAnirudh Venkataramanan } 19542b245cb2SAnirudh Venkataramanan 19552b245cb2SAnirudh Venkataramanan /** 19562b245cb2SAnirudh Venkataramanan * __ice_chk_linearize - Check if there are more than 8 buffers per packet 19572b245cb2SAnirudh Venkataramanan * @skb: send buffer 19582b245cb2SAnirudh Venkataramanan * 19592b245cb2SAnirudh Venkataramanan * Note: This HW can't DMA more than 8 buffers to build a packet on the wire 19602b245cb2SAnirudh Venkataramanan * and so we need to figure out the cases where we need to linearize the skb. 19612b245cb2SAnirudh Venkataramanan * 19622b245cb2SAnirudh Venkataramanan * For TSO we need to count the TSO header and segment payload separately. 19632b245cb2SAnirudh Venkataramanan * As such we need to check cases where we have 7 fragments or more as we 19642b245cb2SAnirudh Venkataramanan * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 19652b245cb2SAnirudh Venkataramanan * the segment payload in the first descriptor, and another 7 for the 19662b245cb2SAnirudh Venkataramanan * fragments. 19672b245cb2SAnirudh Venkataramanan */ 19682b245cb2SAnirudh Venkataramanan static bool __ice_chk_linearize(struct sk_buff *skb) 19692b245cb2SAnirudh Venkataramanan { 19702b245cb2SAnirudh Venkataramanan const struct skb_frag_struct *frag, *stale; 19712b245cb2SAnirudh Venkataramanan int nr_frags, sum; 19722b245cb2SAnirudh Venkataramanan 19732b245cb2SAnirudh Venkataramanan /* no need to check if number of frags is less than 7 */ 19742b245cb2SAnirudh Venkataramanan nr_frags = skb_shinfo(skb)->nr_frags; 19752b245cb2SAnirudh Venkataramanan if (nr_frags < (ICE_MAX_BUF_TXD - 1)) 19762b245cb2SAnirudh Venkataramanan return false; 19772b245cb2SAnirudh Venkataramanan 19782b245cb2SAnirudh Venkataramanan /* We need to walk through the list and validate that each group 19792b245cb2SAnirudh Venkataramanan * of 6 fragments totals at least gso_size. 19802b245cb2SAnirudh Venkataramanan */ 19812b245cb2SAnirudh Venkataramanan nr_frags -= ICE_MAX_BUF_TXD - 2; 19822b245cb2SAnirudh Venkataramanan frag = &skb_shinfo(skb)->frags[0]; 19832b245cb2SAnirudh Venkataramanan 19842b245cb2SAnirudh Venkataramanan /* Initialize size to the negative value of gso_size minus 1. We 19852b245cb2SAnirudh Venkataramanan * use this as the worst case scenerio in which the frag ahead 19862b245cb2SAnirudh Venkataramanan * of us only provides one byte which is why we are limited to 6 19872b245cb2SAnirudh Venkataramanan * descriptors for a single transmit as the header and previous 19882b245cb2SAnirudh Venkataramanan * fragment are already consuming 2 descriptors. 19892b245cb2SAnirudh Venkataramanan */ 19902b245cb2SAnirudh Venkataramanan sum = 1 - skb_shinfo(skb)->gso_size; 19912b245cb2SAnirudh Venkataramanan 19922b245cb2SAnirudh Venkataramanan /* Add size of frags 0 through 4 to create our initial sum */ 19932b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 19942b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 19952b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 19962b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 19972b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 19982b245cb2SAnirudh Venkataramanan 19992b245cb2SAnirudh Venkataramanan /* Walk through fragments adding latest fragment, testing it, and 20002b245cb2SAnirudh Venkataramanan * then removing stale fragments from the sum. 20012b245cb2SAnirudh Venkataramanan */ 20022b245cb2SAnirudh Venkataramanan stale = &skb_shinfo(skb)->frags[0]; 20032b245cb2SAnirudh Venkataramanan for (;;) { 20042b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 20052b245cb2SAnirudh Venkataramanan 20062b245cb2SAnirudh Venkataramanan /* if sum is negative we failed to make sufficient progress */ 20072b245cb2SAnirudh Venkataramanan if (sum < 0) 20082b245cb2SAnirudh Venkataramanan return true; 20092b245cb2SAnirudh Venkataramanan 20102b245cb2SAnirudh Venkataramanan if (!nr_frags--) 20112b245cb2SAnirudh Venkataramanan break; 20122b245cb2SAnirudh Venkataramanan 20132b245cb2SAnirudh Venkataramanan sum -= skb_frag_size(stale++); 20142b245cb2SAnirudh Venkataramanan } 20152b245cb2SAnirudh Venkataramanan 20162b245cb2SAnirudh Venkataramanan return false; 20172b245cb2SAnirudh Venkataramanan } 20182b245cb2SAnirudh Venkataramanan 20192b245cb2SAnirudh Venkataramanan /** 20202b245cb2SAnirudh Venkataramanan * ice_chk_linearize - Check if there are more than 8 fragments per packet 20212b245cb2SAnirudh Venkataramanan * @skb: send buffer 20222b245cb2SAnirudh Venkataramanan * @count: number of buffers used 20232b245cb2SAnirudh Venkataramanan * 20242b245cb2SAnirudh Venkataramanan * Note: Our HW can't scatter-gather more than 8 fragments to build 20252b245cb2SAnirudh Venkataramanan * a packet on the wire and so we need to figure out the cases where we 20262b245cb2SAnirudh Venkataramanan * need to linearize the skb. 20272b245cb2SAnirudh Venkataramanan */ 20282b245cb2SAnirudh Venkataramanan static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count) 20292b245cb2SAnirudh Venkataramanan { 20302b245cb2SAnirudh Venkataramanan /* Both TSO and single send will work if count is less than 8 */ 20312b245cb2SAnirudh Venkataramanan if (likely(count < ICE_MAX_BUF_TXD)) 20322b245cb2SAnirudh Venkataramanan return false; 20332b245cb2SAnirudh Venkataramanan 20342b245cb2SAnirudh Venkataramanan if (skb_is_gso(skb)) 20352b245cb2SAnirudh Venkataramanan return __ice_chk_linearize(skb); 20362b245cb2SAnirudh Venkataramanan 20372b245cb2SAnirudh Venkataramanan /* we can support up to 8 data buffers for a single send */ 20382b245cb2SAnirudh Venkataramanan return count != ICE_MAX_BUF_TXD; 20392b245cb2SAnirudh Venkataramanan } 20402b245cb2SAnirudh Venkataramanan 20412b245cb2SAnirudh Venkataramanan /** 20422b245cb2SAnirudh Venkataramanan * ice_xmit_frame_ring - Sends buffer on Tx ring 20432b245cb2SAnirudh Venkataramanan * @skb: send buffer 20442b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 20452b245cb2SAnirudh Venkataramanan * 20462b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 20472b245cb2SAnirudh Venkataramanan */ 20482b245cb2SAnirudh Venkataramanan static netdev_tx_t 20492b245cb2SAnirudh Venkataramanan ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) 20502b245cb2SAnirudh Venkataramanan { 2051d76a60baSAnirudh Venkataramanan struct ice_tx_offload_params offload = { 0 }; 20522b245cb2SAnirudh Venkataramanan struct ice_tx_buf *first; 20532b245cb2SAnirudh Venkataramanan unsigned int count; 2054d76a60baSAnirudh Venkataramanan int tso, csum; 20552b245cb2SAnirudh Venkataramanan 20562b245cb2SAnirudh Venkataramanan count = ice_xmit_desc_count(skb); 20572b245cb2SAnirudh Venkataramanan if (ice_chk_linearize(skb, count)) { 20582b245cb2SAnirudh Venkataramanan if (__skb_linearize(skb)) 20592b245cb2SAnirudh Venkataramanan goto out_drop; 20602b245cb2SAnirudh Venkataramanan count = ice_txd_use_count(skb->len); 20612b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_linearize++; 20622b245cb2SAnirudh Venkataramanan } 20632b245cb2SAnirudh Venkataramanan 20642b245cb2SAnirudh Venkataramanan /* need: 1 descriptor per page * PAGE_SIZE/ICE_MAX_DATA_PER_TXD, 20652b245cb2SAnirudh Venkataramanan * + 1 desc for skb_head_len/ICE_MAX_DATA_PER_TXD, 20662b245cb2SAnirudh Venkataramanan * + 4 desc gap to avoid the cache line where head is, 20672b245cb2SAnirudh Venkataramanan * + 1 desc for context descriptor, 20682b245cb2SAnirudh Venkataramanan * otherwise try next time 20692b245cb2SAnirudh Venkataramanan */ 2070c585ea42SBrett Creeley if (ice_maybe_stop_tx(tx_ring, count + ICE_DESCS_PER_CACHE_LINE + 2071c585ea42SBrett Creeley ICE_DESCS_FOR_CTX_DESC)) { 20722b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_busy++; 20732b245cb2SAnirudh Venkataramanan return NETDEV_TX_BUSY; 20742b245cb2SAnirudh Venkataramanan } 20752b245cb2SAnirudh Venkataramanan 2076d76a60baSAnirudh Venkataramanan offload.tx_ring = tx_ring; 2077d76a60baSAnirudh Venkataramanan 20782b245cb2SAnirudh Venkataramanan /* record the location of the first descriptor for this packet */ 20792b245cb2SAnirudh Venkataramanan first = &tx_ring->tx_buf[tx_ring->next_to_use]; 20802b245cb2SAnirudh Venkataramanan first->skb = skb; 20812b245cb2SAnirudh Venkataramanan first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 20822b245cb2SAnirudh Venkataramanan first->gso_segs = 1; 2083d76a60baSAnirudh Venkataramanan first->tx_flags = 0; 20842b245cb2SAnirudh Venkataramanan 2085d76a60baSAnirudh Venkataramanan /* prepare the VLAN tagging flags for Tx */ 2086d76a60baSAnirudh Venkataramanan if (ice_tx_prepare_vlan_flags(tx_ring, first)) 2087d76a60baSAnirudh Venkataramanan goto out_drop; 2088d76a60baSAnirudh Venkataramanan 2089d76a60baSAnirudh Venkataramanan /* set up TSO offload */ 2090d76a60baSAnirudh Venkataramanan tso = ice_tso(first, &offload); 2091d76a60baSAnirudh Venkataramanan if (tso < 0) 2092d76a60baSAnirudh Venkataramanan goto out_drop; 2093d76a60baSAnirudh Venkataramanan 2094d76a60baSAnirudh Venkataramanan /* always set up Tx checksum offload */ 2095d76a60baSAnirudh Venkataramanan csum = ice_tx_csum(first, &offload); 2096d76a60baSAnirudh Venkataramanan if (csum < 0) 2097d76a60baSAnirudh Venkataramanan goto out_drop; 2098d76a60baSAnirudh Venkataramanan 2099d76a60baSAnirudh Venkataramanan if (tso || offload.cd_tunnel_params) { 2100d76a60baSAnirudh Venkataramanan struct ice_tx_ctx_desc *cdesc; 2101d76a60baSAnirudh Venkataramanan int i = tx_ring->next_to_use; 2102d76a60baSAnirudh Venkataramanan 2103d76a60baSAnirudh Venkataramanan /* grab the next descriptor */ 2104d76a60baSAnirudh Venkataramanan cdesc = ICE_TX_CTX_DESC(tx_ring, i); 2105d76a60baSAnirudh Venkataramanan i++; 2106d76a60baSAnirudh Venkataramanan tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2107d76a60baSAnirudh Venkataramanan 2108d76a60baSAnirudh Venkataramanan /* setup context descriptor */ 2109d76a60baSAnirudh Venkataramanan cdesc->tunneling_params = cpu_to_le32(offload.cd_tunnel_params); 2110d76a60baSAnirudh Venkataramanan cdesc->l2tag2 = cpu_to_le16(offload.cd_l2tag2); 2111d76a60baSAnirudh Venkataramanan cdesc->rsvd = cpu_to_le16(0); 2112d76a60baSAnirudh Venkataramanan cdesc->qw1 = cpu_to_le64(offload.cd_qw1); 2113d76a60baSAnirudh Venkataramanan } 2114d76a60baSAnirudh Venkataramanan 2115d76a60baSAnirudh Venkataramanan ice_tx_map(tx_ring, first, &offload); 21162b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 21172b245cb2SAnirudh Venkataramanan 21182b245cb2SAnirudh Venkataramanan out_drop: 21192b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 21202b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 21212b245cb2SAnirudh Venkataramanan } 21222b245cb2SAnirudh Venkataramanan 21232b245cb2SAnirudh Venkataramanan /** 21242b245cb2SAnirudh Venkataramanan * ice_start_xmit - Selects the correct VSI and Tx queue to send buffer 21252b245cb2SAnirudh Venkataramanan * @skb: send buffer 21262b245cb2SAnirudh Venkataramanan * @netdev: network interface device structure 21272b245cb2SAnirudh Venkataramanan * 21282b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 21292b245cb2SAnirudh Venkataramanan */ 21302b245cb2SAnirudh Venkataramanan netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev) 21312b245cb2SAnirudh Venkataramanan { 21322b245cb2SAnirudh Venkataramanan struct ice_netdev_priv *np = netdev_priv(netdev); 21332b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = np->vsi; 21342b245cb2SAnirudh Venkataramanan struct ice_ring *tx_ring; 21352b245cb2SAnirudh Venkataramanan 21362b245cb2SAnirudh Venkataramanan tx_ring = vsi->tx_rings[skb->queue_mapping]; 21372b245cb2SAnirudh Venkataramanan 21382b245cb2SAnirudh Venkataramanan /* hardware can't handle really short frames, hardware padding works 21392b245cb2SAnirudh Venkataramanan * beyond this point 21402b245cb2SAnirudh Venkataramanan */ 21412b245cb2SAnirudh Venkataramanan if (skb_put_padto(skb, ICE_MIN_TX_LEN)) 21422b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 21432b245cb2SAnirudh Venkataramanan 21442b245cb2SAnirudh Venkataramanan return ice_xmit_frame_ring(skb, tx_ring); 21452b245cb2SAnirudh Venkataramanan } 2146