165e0ace2SJie Deng /* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver 265e0ace2SJie Deng * 365e0ace2SJie Deng * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com) 465e0ace2SJie Deng * 5ea8c1c64SJie Deng * This program is dual-licensed; you may select either version 2 of 6ea8c1c64SJie Deng * the GNU General Public License ("GPL") or BSD license ("BSD"). 765e0ace2SJie Deng * 865e0ace2SJie Deng * This Synopsys DWC XLGMAC software driver and associated documentation 965e0ace2SJie Deng * (hereinafter the "Software") is an unsupported proprietary work of 1065e0ace2SJie Deng * Synopsys, Inc. unless otherwise expressly agreed to in writing between 1165e0ace2SJie Deng * Synopsys and you. The Software IS NOT an item of Licensed Software or a 1265e0ace2SJie Deng * Licensed Product under any End User Software License Agreement or 1365e0ace2SJie Deng * Agreement for Licensed Products with Synopsys or any supplement thereto. 1465e0ace2SJie Deng * Synopsys is a registered trademark of Synopsys, Inc. Other names included 1565e0ace2SJie Deng * in the SOFTWARE may be the trademarks of their respective owners. 1665e0ace2SJie Deng */ 1765e0ace2SJie Deng 1865e0ace2SJie Deng #include "dwc-xlgmac.h" 1965e0ace2SJie Deng #include "dwc-xlgmac-reg.h" 2065e0ace2SJie Deng 2165e0ace2SJie Deng static void xlgmac_unmap_desc_data(struct xlgmac_pdata *pdata, 2265e0ace2SJie Deng struct xlgmac_desc_data *desc_data) 2365e0ace2SJie Deng { 2465e0ace2SJie Deng if (desc_data->skb_dma) { 2565e0ace2SJie Deng if (desc_data->mapped_as_page) { 2665e0ace2SJie Deng dma_unmap_page(pdata->dev, desc_data->skb_dma, 2765e0ace2SJie Deng desc_data->skb_dma_len, DMA_TO_DEVICE); 2865e0ace2SJie Deng } else { 2965e0ace2SJie Deng dma_unmap_single(pdata->dev, desc_data->skb_dma, 3065e0ace2SJie Deng desc_data->skb_dma_len, DMA_TO_DEVICE); 3165e0ace2SJie Deng } 3265e0ace2SJie Deng desc_data->skb_dma = 0; 3365e0ace2SJie Deng desc_data->skb_dma_len = 0; 3465e0ace2SJie Deng } 3565e0ace2SJie Deng 3665e0ace2SJie Deng if (desc_data->skb) { 3765e0ace2SJie Deng dev_kfree_skb_any(desc_data->skb); 3865e0ace2SJie Deng desc_data->skb = NULL; 3965e0ace2SJie Deng } 4065e0ace2SJie Deng 4165e0ace2SJie Deng if (desc_data->rx.hdr.pa.pages) 4265e0ace2SJie Deng put_page(desc_data->rx.hdr.pa.pages); 4365e0ace2SJie Deng 4465e0ace2SJie Deng if (desc_data->rx.hdr.pa_unmap.pages) { 4565e0ace2SJie Deng dma_unmap_page(pdata->dev, desc_data->rx.hdr.pa_unmap.pages_dma, 4665e0ace2SJie Deng desc_data->rx.hdr.pa_unmap.pages_len, 4765e0ace2SJie Deng DMA_FROM_DEVICE); 4865e0ace2SJie Deng put_page(desc_data->rx.hdr.pa_unmap.pages); 4965e0ace2SJie Deng } 5065e0ace2SJie Deng 5165e0ace2SJie Deng if (desc_data->rx.buf.pa.pages) 5265e0ace2SJie Deng put_page(desc_data->rx.buf.pa.pages); 5365e0ace2SJie Deng 5465e0ace2SJie Deng if (desc_data->rx.buf.pa_unmap.pages) { 5565e0ace2SJie Deng dma_unmap_page(pdata->dev, desc_data->rx.buf.pa_unmap.pages_dma, 5665e0ace2SJie Deng desc_data->rx.buf.pa_unmap.pages_len, 5765e0ace2SJie Deng DMA_FROM_DEVICE); 5865e0ace2SJie Deng put_page(desc_data->rx.buf.pa_unmap.pages); 5965e0ace2SJie Deng } 6065e0ace2SJie Deng 6165e0ace2SJie Deng memset(&desc_data->tx, 0, sizeof(desc_data->tx)); 6265e0ace2SJie Deng memset(&desc_data->rx, 0, sizeof(desc_data->rx)); 6365e0ace2SJie Deng 6465e0ace2SJie Deng desc_data->mapped_as_page = 0; 6565e0ace2SJie Deng 6665e0ace2SJie Deng if (desc_data->state_saved) { 6765e0ace2SJie Deng desc_data->state_saved = 0; 6865e0ace2SJie Deng desc_data->state.skb = NULL; 6965e0ace2SJie Deng desc_data->state.len = 0; 7065e0ace2SJie Deng desc_data->state.error = 0; 7165e0ace2SJie Deng } 7265e0ace2SJie Deng } 7365e0ace2SJie Deng 7465e0ace2SJie Deng static void xlgmac_free_ring(struct xlgmac_pdata *pdata, 7565e0ace2SJie Deng struct xlgmac_ring *ring) 7665e0ace2SJie Deng { 7765e0ace2SJie Deng struct xlgmac_desc_data *desc_data; 7865e0ace2SJie Deng unsigned int i; 7965e0ace2SJie Deng 8065e0ace2SJie Deng if (!ring) 8165e0ace2SJie Deng return; 8265e0ace2SJie Deng 8365e0ace2SJie Deng if (ring->desc_data_head) { 8465e0ace2SJie Deng for (i = 0; i < ring->dma_desc_count; i++) { 8565e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, i); 8665e0ace2SJie Deng xlgmac_unmap_desc_data(pdata, desc_data); 8765e0ace2SJie Deng } 8865e0ace2SJie Deng 8965e0ace2SJie Deng kfree(ring->desc_data_head); 9065e0ace2SJie Deng ring->desc_data_head = NULL; 9165e0ace2SJie Deng } 9265e0ace2SJie Deng 9365e0ace2SJie Deng if (ring->rx_hdr_pa.pages) { 9465e0ace2SJie Deng dma_unmap_page(pdata->dev, ring->rx_hdr_pa.pages_dma, 9565e0ace2SJie Deng ring->rx_hdr_pa.pages_len, DMA_FROM_DEVICE); 9665e0ace2SJie Deng put_page(ring->rx_hdr_pa.pages); 9765e0ace2SJie Deng 9865e0ace2SJie Deng ring->rx_hdr_pa.pages = NULL; 9965e0ace2SJie Deng ring->rx_hdr_pa.pages_len = 0; 10065e0ace2SJie Deng ring->rx_hdr_pa.pages_offset = 0; 10165e0ace2SJie Deng ring->rx_hdr_pa.pages_dma = 0; 10265e0ace2SJie Deng } 10365e0ace2SJie Deng 10465e0ace2SJie Deng if (ring->rx_buf_pa.pages) { 10565e0ace2SJie Deng dma_unmap_page(pdata->dev, ring->rx_buf_pa.pages_dma, 10665e0ace2SJie Deng ring->rx_buf_pa.pages_len, DMA_FROM_DEVICE); 10765e0ace2SJie Deng put_page(ring->rx_buf_pa.pages); 10865e0ace2SJie Deng 10965e0ace2SJie Deng ring->rx_buf_pa.pages = NULL; 11065e0ace2SJie Deng ring->rx_buf_pa.pages_len = 0; 11165e0ace2SJie Deng ring->rx_buf_pa.pages_offset = 0; 11265e0ace2SJie Deng ring->rx_buf_pa.pages_dma = 0; 11365e0ace2SJie Deng } 11465e0ace2SJie Deng 11565e0ace2SJie Deng if (ring->dma_desc_head) { 11665e0ace2SJie Deng dma_free_coherent(pdata->dev, 11765e0ace2SJie Deng (sizeof(struct xlgmac_dma_desc) * 11865e0ace2SJie Deng ring->dma_desc_count), 11965e0ace2SJie Deng ring->dma_desc_head, 12065e0ace2SJie Deng ring->dma_desc_head_addr); 12165e0ace2SJie Deng ring->dma_desc_head = NULL; 12265e0ace2SJie Deng } 12365e0ace2SJie Deng } 12465e0ace2SJie Deng 12565e0ace2SJie Deng static int xlgmac_init_ring(struct xlgmac_pdata *pdata, 12665e0ace2SJie Deng struct xlgmac_ring *ring, 12765e0ace2SJie Deng unsigned int dma_desc_count) 12865e0ace2SJie Deng { 12965e0ace2SJie Deng if (!ring) 13065e0ace2SJie Deng return 0; 13165e0ace2SJie Deng 13265e0ace2SJie Deng /* Descriptors */ 13365e0ace2SJie Deng ring->dma_desc_count = dma_desc_count; 13465e0ace2SJie Deng ring->dma_desc_head = dma_alloc_coherent(pdata->dev, 13565e0ace2SJie Deng (sizeof(struct xlgmac_dma_desc) * 13665e0ace2SJie Deng dma_desc_count), 13765e0ace2SJie Deng &ring->dma_desc_head_addr, 13865e0ace2SJie Deng GFP_KERNEL); 13965e0ace2SJie Deng if (!ring->dma_desc_head) 14065e0ace2SJie Deng return -ENOMEM; 14165e0ace2SJie Deng 14265e0ace2SJie Deng /* Array of descriptor data */ 14365e0ace2SJie Deng ring->desc_data_head = kcalloc(dma_desc_count, 14465e0ace2SJie Deng sizeof(struct xlgmac_desc_data), 14565e0ace2SJie Deng GFP_KERNEL); 14665e0ace2SJie Deng if (!ring->desc_data_head) 14765e0ace2SJie Deng return -ENOMEM; 14865e0ace2SJie Deng 14965e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, 15065e0ace2SJie Deng "dma_desc_head=%p, dma_desc_head_addr=%pad, desc_data_head=%p\n", 15165e0ace2SJie Deng ring->dma_desc_head, 15265e0ace2SJie Deng &ring->dma_desc_head_addr, 15365e0ace2SJie Deng ring->desc_data_head); 15465e0ace2SJie Deng 15565e0ace2SJie Deng return 0; 15665e0ace2SJie Deng } 15765e0ace2SJie Deng 15865e0ace2SJie Deng static void xlgmac_free_rings(struct xlgmac_pdata *pdata) 15965e0ace2SJie Deng { 16065e0ace2SJie Deng struct xlgmac_channel *channel; 16165e0ace2SJie Deng unsigned int i; 16265e0ace2SJie Deng 16365e0ace2SJie Deng if (!pdata->channel_head) 16465e0ace2SJie Deng return; 16565e0ace2SJie Deng 16665e0ace2SJie Deng channel = pdata->channel_head; 16765e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) { 16865e0ace2SJie Deng xlgmac_free_ring(pdata, channel->tx_ring); 16965e0ace2SJie Deng xlgmac_free_ring(pdata, channel->rx_ring); 17065e0ace2SJie Deng } 17165e0ace2SJie Deng } 17265e0ace2SJie Deng 17365e0ace2SJie Deng static int xlgmac_alloc_rings(struct xlgmac_pdata *pdata) 17465e0ace2SJie Deng { 17565e0ace2SJie Deng struct xlgmac_channel *channel; 17665e0ace2SJie Deng unsigned int i; 17765e0ace2SJie Deng int ret; 17865e0ace2SJie Deng 17965e0ace2SJie Deng channel = pdata->channel_head; 18065e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) { 18165e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, "%s - Tx ring:\n", 18265e0ace2SJie Deng channel->name); 18365e0ace2SJie Deng 18465e0ace2SJie Deng ret = xlgmac_init_ring(pdata, channel->tx_ring, 18565e0ace2SJie Deng pdata->tx_desc_count); 18665e0ace2SJie Deng 18765e0ace2SJie Deng if (ret) { 18865e0ace2SJie Deng netdev_alert(pdata->netdev, 18965e0ace2SJie Deng "error initializing Tx ring"); 19065e0ace2SJie Deng goto err_init_ring; 19165e0ace2SJie Deng } 19265e0ace2SJie Deng 19365e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, "%s - Rx ring:\n", 19465e0ace2SJie Deng channel->name); 19565e0ace2SJie Deng 19665e0ace2SJie Deng ret = xlgmac_init_ring(pdata, channel->rx_ring, 19765e0ace2SJie Deng pdata->rx_desc_count); 19865e0ace2SJie Deng if (ret) { 19965e0ace2SJie Deng netdev_alert(pdata->netdev, 20065e0ace2SJie Deng "error initializing Rx ring\n"); 20165e0ace2SJie Deng goto err_init_ring; 20265e0ace2SJie Deng } 20365e0ace2SJie Deng } 20465e0ace2SJie Deng 20565e0ace2SJie Deng return 0; 20665e0ace2SJie Deng 20765e0ace2SJie Deng err_init_ring: 20865e0ace2SJie Deng xlgmac_free_rings(pdata); 20965e0ace2SJie Deng 21065e0ace2SJie Deng return ret; 21165e0ace2SJie Deng } 21265e0ace2SJie Deng 21365e0ace2SJie Deng static void xlgmac_free_channels(struct xlgmac_pdata *pdata) 21465e0ace2SJie Deng { 21565e0ace2SJie Deng if (!pdata->channel_head) 21665e0ace2SJie Deng return; 21765e0ace2SJie Deng 21865e0ace2SJie Deng kfree(pdata->channel_head->tx_ring); 21965e0ace2SJie Deng pdata->channel_head->tx_ring = NULL; 22065e0ace2SJie Deng 22165e0ace2SJie Deng kfree(pdata->channel_head->rx_ring); 22265e0ace2SJie Deng pdata->channel_head->rx_ring = NULL; 22365e0ace2SJie Deng 22465e0ace2SJie Deng kfree(pdata->channel_head); 22565e0ace2SJie Deng 22665e0ace2SJie Deng pdata->channel_head = NULL; 22765e0ace2SJie Deng pdata->channel_count = 0; 22865e0ace2SJie Deng } 22965e0ace2SJie Deng 23065e0ace2SJie Deng static int xlgmac_alloc_channels(struct xlgmac_pdata *pdata) 23165e0ace2SJie Deng { 23265e0ace2SJie Deng struct xlgmac_channel *channel_head, *channel; 23365e0ace2SJie Deng struct xlgmac_ring *tx_ring, *rx_ring; 23465e0ace2SJie Deng int ret = -ENOMEM; 23565e0ace2SJie Deng unsigned int i; 23665e0ace2SJie Deng 23765e0ace2SJie Deng channel_head = kcalloc(pdata->channel_count, 23865e0ace2SJie Deng sizeof(struct xlgmac_channel), GFP_KERNEL); 23965e0ace2SJie Deng if (!channel_head) 24065e0ace2SJie Deng return ret; 24165e0ace2SJie Deng 24265e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, 24365e0ace2SJie Deng "channel_head=%p\n", channel_head); 24465e0ace2SJie Deng 24565e0ace2SJie Deng tx_ring = kcalloc(pdata->tx_ring_count, sizeof(struct xlgmac_ring), 24665e0ace2SJie Deng GFP_KERNEL); 24765e0ace2SJie Deng if (!tx_ring) 24865e0ace2SJie Deng goto err_tx_ring; 24965e0ace2SJie Deng 25065e0ace2SJie Deng rx_ring = kcalloc(pdata->rx_ring_count, sizeof(struct xlgmac_ring), 25165e0ace2SJie Deng GFP_KERNEL); 25265e0ace2SJie Deng if (!rx_ring) 25365e0ace2SJie Deng goto err_rx_ring; 25465e0ace2SJie Deng 25565e0ace2SJie Deng for (i = 0, channel = channel_head; i < pdata->channel_count; 25665e0ace2SJie Deng i++, channel++) { 25765e0ace2SJie Deng snprintf(channel->name, sizeof(channel->name), "channel-%u", i); 25865e0ace2SJie Deng channel->pdata = pdata; 25965e0ace2SJie Deng channel->queue_index = i; 26065e0ace2SJie Deng channel->dma_regs = pdata->mac_regs + DMA_CH_BASE + 26165e0ace2SJie Deng (DMA_CH_INC * i); 26265e0ace2SJie Deng 26365e0ace2SJie Deng if (pdata->per_channel_irq) { 26465e0ace2SJie Deng /* Get the per DMA interrupt */ 26565e0ace2SJie Deng ret = pdata->channel_irq[i]; 26665e0ace2SJie Deng if (ret < 0) { 26765e0ace2SJie Deng netdev_err(pdata->netdev, 26865e0ace2SJie Deng "get_irq %u failed\n", 26965e0ace2SJie Deng i + 1); 27065e0ace2SJie Deng goto err_irq; 27165e0ace2SJie Deng } 27265e0ace2SJie Deng channel->dma_irq = ret; 27365e0ace2SJie Deng } 27465e0ace2SJie Deng 27565e0ace2SJie Deng if (i < pdata->tx_ring_count) 27665e0ace2SJie Deng channel->tx_ring = tx_ring++; 27765e0ace2SJie Deng 27865e0ace2SJie Deng if (i < pdata->rx_ring_count) 27965e0ace2SJie Deng channel->rx_ring = rx_ring++; 28065e0ace2SJie Deng 28165e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, 28265e0ace2SJie Deng "%s: dma_regs=%p, tx_ring=%p, rx_ring=%p\n", 28365e0ace2SJie Deng channel->name, channel->dma_regs, 28465e0ace2SJie Deng channel->tx_ring, channel->rx_ring); 28565e0ace2SJie Deng } 28665e0ace2SJie Deng 28765e0ace2SJie Deng pdata->channel_head = channel_head; 28865e0ace2SJie Deng 28965e0ace2SJie Deng return 0; 29065e0ace2SJie Deng 29165e0ace2SJie Deng err_irq: 29265e0ace2SJie Deng kfree(rx_ring); 29365e0ace2SJie Deng 29465e0ace2SJie Deng err_rx_ring: 29565e0ace2SJie Deng kfree(tx_ring); 29665e0ace2SJie Deng 29765e0ace2SJie Deng err_tx_ring: 29865e0ace2SJie Deng kfree(channel_head); 29965e0ace2SJie Deng 30065e0ace2SJie Deng return ret; 30165e0ace2SJie Deng } 30265e0ace2SJie Deng 30365e0ace2SJie Deng static void xlgmac_free_channels_and_rings(struct xlgmac_pdata *pdata) 30465e0ace2SJie Deng { 30565e0ace2SJie Deng xlgmac_free_rings(pdata); 30665e0ace2SJie Deng 30765e0ace2SJie Deng xlgmac_free_channels(pdata); 30865e0ace2SJie Deng } 30965e0ace2SJie Deng 31065e0ace2SJie Deng static int xlgmac_alloc_channels_and_rings(struct xlgmac_pdata *pdata) 31165e0ace2SJie Deng { 31265e0ace2SJie Deng int ret; 31365e0ace2SJie Deng 31465e0ace2SJie Deng ret = xlgmac_alloc_channels(pdata); 31565e0ace2SJie Deng if (ret) 31665e0ace2SJie Deng goto err_alloc; 31765e0ace2SJie Deng 31865e0ace2SJie Deng ret = xlgmac_alloc_rings(pdata); 31965e0ace2SJie Deng if (ret) 32065e0ace2SJie Deng goto err_alloc; 32165e0ace2SJie Deng 32265e0ace2SJie Deng return 0; 32365e0ace2SJie Deng 32465e0ace2SJie Deng err_alloc: 32565e0ace2SJie Deng xlgmac_free_channels_and_rings(pdata); 32665e0ace2SJie Deng 32765e0ace2SJie Deng return ret; 32865e0ace2SJie Deng } 32965e0ace2SJie Deng 33065e0ace2SJie Deng static int xlgmac_alloc_pages(struct xlgmac_pdata *pdata, 33165e0ace2SJie Deng struct xlgmac_page_alloc *pa, 33265e0ace2SJie Deng gfp_t gfp, int order) 33365e0ace2SJie Deng { 33465e0ace2SJie Deng struct page *pages = NULL; 33565e0ace2SJie Deng dma_addr_t pages_dma; 33665e0ace2SJie Deng 33765e0ace2SJie Deng /* Try to obtain pages, decreasing order if necessary */ 338453f85d4SMel Gorman gfp |= __GFP_COMP | __GFP_NOWARN; 33965e0ace2SJie Deng while (order >= 0) { 34065e0ace2SJie Deng pages = alloc_pages(gfp, order); 34165e0ace2SJie Deng if (pages) 34265e0ace2SJie Deng break; 34365e0ace2SJie Deng 34465e0ace2SJie Deng order--; 34565e0ace2SJie Deng } 34665e0ace2SJie Deng if (!pages) 34765e0ace2SJie Deng return -ENOMEM; 34865e0ace2SJie Deng 34965e0ace2SJie Deng /* Map the pages */ 35065e0ace2SJie Deng pages_dma = dma_map_page(pdata->dev, pages, 0, 35165e0ace2SJie Deng PAGE_SIZE << order, DMA_FROM_DEVICE); 35231c7ba9eSDan Carpenter if (dma_mapping_error(pdata->dev, pages_dma)) { 35365e0ace2SJie Deng put_page(pages); 35431c7ba9eSDan Carpenter return -ENOMEM; 35565e0ace2SJie Deng } 35665e0ace2SJie Deng 35765e0ace2SJie Deng pa->pages = pages; 35865e0ace2SJie Deng pa->pages_len = PAGE_SIZE << order; 35965e0ace2SJie Deng pa->pages_offset = 0; 36065e0ace2SJie Deng pa->pages_dma = pages_dma; 36165e0ace2SJie Deng 36265e0ace2SJie Deng return 0; 36365e0ace2SJie Deng } 36465e0ace2SJie Deng 36565e0ace2SJie Deng static void xlgmac_set_buffer_data(struct xlgmac_buffer_data *bd, 36665e0ace2SJie Deng struct xlgmac_page_alloc *pa, 36765e0ace2SJie Deng unsigned int len) 36865e0ace2SJie Deng { 36965e0ace2SJie Deng get_page(pa->pages); 37065e0ace2SJie Deng bd->pa = *pa; 37165e0ace2SJie Deng 37265e0ace2SJie Deng bd->dma_base = pa->pages_dma; 37365e0ace2SJie Deng bd->dma_off = pa->pages_offset; 37465e0ace2SJie Deng bd->dma_len = len; 37565e0ace2SJie Deng 37665e0ace2SJie Deng pa->pages_offset += len; 37765e0ace2SJie Deng if ((pa->pages_offset + len) > pa->pages_len) { 37865e0ace2SJie Deng /* This data descriptor is responsible for unmapping page(s) */ 37965e0ace2SJie Deng bd->pa_unmap = *pa; 38065e0ace2SJie Deng 38165e0ace2SJie Deng /* Get a new allocation next time */ 38265e0ace2SJie Deng pa->pages = NULL; 38365e0ace2SJie Deng pa->pages_len = 0; 38465e0ace2SJie Deng pa->pages_offset = 0; 38565e0ace2SJie Deng pa->pages_dma = 0; 38665e0ace2SJie Deng } 38765e0ace2SJie Deng } 38865e0ace2SJie Deng 38965e0ace2SJie Deng static int xlgmac_map_rx_buffer(struct xlgmac_pdata *pdata, 39065e0ace2SJie Deng struct xlgmac_ring *ring, 39165e0ace2SJie Deng struct xlgmac_desc_data *desc_data) 39265e0ace2SJie Deng { 39365e0ace2SJie Deng int order, ret; 39465e0ace2SJie Deng 39565e0ace2SJie Deng if (!ring->rx_hdr_pa.pages) { 39665e0ace2SJie Deng ret = xlgmac_alloc_pages(pdata, &ring->rx_hdr_pa, 39765e0ace2SJie Deng GFP_ATOMIC, 0); 39865e0ace2SJie Deng if (ret) 39965e0ace2SJie Deng return ret; 40065e0ace2SJie Deng } 40165e0ace2SJie Deng 40265e0ace2SJie Deng if (!ring->rx_buf_pa.pages) { 40365e0ace2SJie Deng order = max_t(int, PAGE_ALLOC_COSTLY_ORDER - 1, 0); 40465e0ace2SJie Deng ret = xlgmac_alloc_pages(pdata, &ring->rx_buf_pa, 40565e0ace2SJie Deng GFP_ATOMIC, order); 40665e0ace2SJie Deng if (ret) 40765e0ace2SJie Deng return ret; 40865e0ace2SJie Deng } 40965e0ace2SJie Deng 41065e0ace2SJie Deng /* Set up the header page info */ 41165e0ace2SJie Deng xlgmac_set_buffer_data(&desc_data->rx.hdr, &ring->rx_hdr_pa, 41265e0ace2SJie Deng XLGMAC_SKB_ALLOC_SIZE); 41365e0ace2SJie Deng 41465e0ace2SJie Deng /* Set up the buffer page info */ 41565e0ace2SJie Deng xlgmac_set_buffer_data(&desc_data->rx.buf, &ring->rx_buf_pa, 41665e0ace2SJie Deng pdata->rx_buf_size); 41765e0ace2SJie Deng 41865e0ace2SJie Deng return 0; 41965e0ace2SJie Deng } 42065e0ace2SJie Deng 42165e0ace2SJie Deng static void xlgmac_tx_desc_init(struct xlgmac_pdata *pdata) 42265e0ace2SJie Deng { 42365e0ace2SJie Deng struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops; 42465e0ace2SJie Deng struct xlgmac_desc_data *desc_data; 42565e0ace2SJie Deng struct xlgmac_dma_desc *dma_desc; 42665e0ace2SJie Deng struct xlgmac_channel *channel; 42765e0ace2SJie Deng struct xlgmac_ring *ring; 42865e0ace2SJie Deng dma_addr_t dma_desc_addr; 42965e0ace2SJie Deng unsigned int i, j; 43065e0ace2SJie Deng 43165e0ace2SJie Deng channel = pdata->channel_head; 43265e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) { 43365e0ace2SJie Deng ring = channel->tx_ring; 43465e0ace2SJie Deng if (!ring) 43565e0ace2SJie Deng break; 43665e0ace2SJie Deng 43765e0ace2SJie Deng dma_desc = ring->dma_desc_head; 43865e0ace2SJie Deng dma_desc_addr = ring->dma_desc_head_addr; 43965e0ace2SJie Deng 44065e0ace2SJie Deng for (j = 0; j < ring->dma_desc_count; j++) { 44165e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, j); 44265e0ace2SJie Deng 44365e0ace2SJie Deng desc_data->dma_desc = dma_desc; 44465e0ace2SJie Deng desc_data->dma_desc_addr = dma_desc_addr; 44565e0ace2SJie Deng 44665e0ace2SJie Deng dma_desc++; 44765e0ace2SJie Deng dma_desc_addr += sizeof(struct xlgmac_dma_desc); 44865e0ace2SJie Deng } 44965e0ace2SJie Deng 45065e0ace2SJie Deng ring->cur = 0; 45165e0ace2SJie Deng ring->dirty = 0; 45265e0ace2SJie Deng memset(&ring->tx, 0, sizeof(ring->tx)); 45365e0ace2SJie Deng 45465e0ace2SJie Deng hw_ops->tx_desc_init(channel); 45565e0ace2SJie Deng } 45665e0ace2SJie Deng } 45765e0ace2SJie Deng 45865e0ace2SJie Deng static void xlgmac_rx_desc_init(struct xlgmac_pdata *pdata) 45965e0ace2SJie Deng { 46065e0ace2SJie Deng struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops; 46165e0ace2SJie Deng struct xlgmac_desc_data *desc_data; 46265e0ace2SJie Deng struct xlgmac_dma_desc *dma_desc; 46365e0ace2SJie Deng struct xlgmac_channel *channel; 46465e0ace2SJie Deng struct xlgmac_ring *ring; 46565e0ace2SJie Deng dma_addr_t dma_desc_addr; 46665e0ace2SJie Deng unsigned int i, j; 46765e0ace2SJie Deng 46865e0ace2SJie Deng channel = pdata->channel_head; 46965e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) { 47065e0ace2SJie Deng ring = channel->rx_ring; 47165e0ace2SJie Deng if (!ring) 47265e0ace2SJie Deng break; 47365e0ace2SJie Deng 47465e0ace2SJie Deng dma_desc = ring->dma_desc_head; 47565e0ace2SJie Deng dma_desc_addr = ring->dma_desc_head_addr; 47665e0ace2SJie Deng 47765e0ace2SJie Deng for (j = 0; j < ring->dma_desc_count; j++) { 47865e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, j); 47965e0ace2SJie Deng 48065e0ace2SJie Deng desc_data->dma_desc = dma_desc; 48165e0ace2SJie Deng desc_data->dma_desc_addr = dma_desc_addr; 48265e0ace2SJie Deng 48365e0ace2SJie Deng if (xlgmac_map_rx_buffer(pdata, ring, desc_data)) 48465e0ace2SJie Deng break; 48565e0ace2SJie Deng 48665e0ace2SJie Deng dma_desc++; 48765e0ace2SJie Deng dma_desc_addr += sizeof(struct xlgmac_dma_desc); 48865e0ace2SJie Deng } 48965e0ace2SJie Deng 49065e0ace2SJie Deng ring->cur = 0; 49165e0ace2SJie Deng ring->dirty = 0; 49265e0ace2SJie Deng 49365e0ace2SJie Deng hw_ops->rx_desc_init(channel); 49465e0ace2SJie Deng } 49565e0ace2SJie Deng } 49665e0ace2SJie Deng 49765e0ace2SJie Deng static int xlgmac_map_tx_skb(struct xlgmac_channel *channel, 49865e0ace2SJie Deng struct sk_buff *skb) 49965e0ace2SJie Deng { 50065e0ace2SJie Deng struct xlgmac_pdata *pdata = channel->pdata; 50165e0ace2SJie Deng struct xlgmac_ring *ring = channel->tx_ring; 50265e0ace2SJie Deng unsigned int start_index, cur_index; 50365e0ace2SJie Deng struct xlgmac_desc_data *desc_data; 50465e0ace2SJie Deng unsigned int offset, datalen, len; 50565e0ace2SJie Deng struct xlgmac_pkt_info *pkt_info; 50665e0ace2SJie Deng struct skb_frag_struct *frag; 50765e0ace2SJie Deng unsigned int tso, vlan; 50865e0ace2SJie Deng dma_addr_t skb_dma; 50965e0ace2SJie Deng unsigned int i; 51065e0ace2SJie Deng 51165e0ace2SJie Deng offset = 0; 51265e0ace2SJie Deng start_index = ring->cur; 51365e0ace2SJie Deng cur_index = ring->cur; 51465e0ace2SJie Deng 51565e0ace2SJie Deng pkt_info = &ring->pkt_info; 51665e0ace2SJie Deng pkt_info->desc_count = 0; 51765e0ace2SJie Deng pkt_info->length = 0; 51865e0ace2SJie Deng 51965e0ace2SJie Deng tso = XLGMAC_GET_REG_BITS(pkt_info->attributes, 52065e0ace2SJie Deng TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS, 52165e0ace2SJie Deng TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN); 52265e0ace2SJie Deng vlan = XLGMAC_GET_REG_BITS(pkt_info->attributes, 52365e0ace2SJie Deng TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS, 52465e0ace2SJie Deng TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN); 52565e0ace2SJie Deng 52665e0ace2SJie Deng /* Save space for a context descriptor if needed */ 52765e0ace2SJie Deng if ((tso && (pkt_info->mss != ring->tx.cur_mss)) || 52865e0ace2SJie Deng (vlan && (pkt_info->vlan_ctag != ring->tx.cur_vlan_ctag))) 52965e0ace2SJie Deng cur_index++; 53065e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index); 53165e0ace2SJie Deng 53265e0ace2SJie Deng if (tso) { 53365e0ace2SJie Deng /* Map the TSO header */ 53465e0ace2SJie Deng skb_dma = dma_map_single(pdata->dev, skb->data, 53565e0ace2SJie Deng pkt_info->header_len, DMA_TO_DEVICE); 53665e0ace2SJie Deng if (dma_mapping_error(pdata->dev, skb_dma)) { 53765e0ace2SJie Deng netdev_alert(pdata->netdev, "dma_map_single failed\n"); 53865e0ace2SJie Deng goto err_out; 53965e0ace2SJie Deng } 54065e0ace2SJie Deng desc_data->skb_dma = skb_dma; 54165e0ace2SJie Deng desc_data->skb_dma_len = pkt_info->header_len; 54265e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev, 54365e0ace2SJie Deng "skb header: index=%u, dma=%pad, len=%u\n", 54465e0ace2SJie Deng cur_index, &skb_dma, pkt_info->header_len); 54565e0ace2SJie Deng 54665e0ace2SJie Deng offset = pkt_info->header_len; 54765e0ace2SJie Deng 54865e0ace2SJie Deng pkt_info->length += pkt_info->header_len; 54965e0ace2SJie Deng 55065e0ace2SJie Deng cur_index++; 55165e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index); 55265e0ace2SJie Deng } 55365e0ace2SJie Deng 55465e0ace2SJie Deng /* Map the (remainder of the) packet */ 55565e0ace2SJie Deng for (datalen = skb_headlen(skb) - offset; datalen; ) { 55665e0ace2SJie Deng len = min_t(unsigned int, datalen, XLGMAC_TX_MAX_BUF_SIZE); 55765e0ace2SJie Deng 55865e0ace2SJie Deng skb_dma = dma_map_single(pdata->dev, skb->data + offset, len, 55965e0ace2SJie Deng DMA_TO_DEVICE); 56065e0ace2SJie Deng if (dma_mapping_error(pdata->dev, skb_dma)) { 56165e0ace2SJie Deng netdev_alert(pdata->netdev, "dma_map_single failed\n"); 56265e0ace2SJie Deng goto err_out; 56365e0ace2SJie Deng } 56465e0ace2SJie Deng desc_data->skb_dma = skb_dma; 56565e0ace2SJie Deng desc_data->skb_dma_len = len; 56665e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev, 56765e0ace2SJie Deng "skb data: index=%u, dma=%pad, len=%u\n", 56865e0ace2SJie Deng cur_index, &skb_dma, len); 56965e0ace2SJie Deng 57065e0ace2SJie Deng datalen -= len; 57165e0ace2SJie Deng offset += len; 57265e0ace2SJie Deng 57365e0ace2SJie Deng pkt_info->length += len; 57465e0ace2SJie Deng 57565e0ace2SJie Deng cur_index++; 57665e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index); 57765e0ace2SJie Deng } 57865e0ace2SJie Deng 57965e0ace2SJie Deng for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 58065e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev, 58165e0ace2SJie Deng "mapping frag %u\n", i); 58265e0ace2SJie Deng 58365e0ace2SJie Deng frag = &skb_shinfo(skb)->frags[i]; 58465e0ace2SJie Deng offset = 0; 58565e0ace2SJie Deng 58665e0ace2SJie Deng for (datalen = skb_frag_size(frag); datalen; ) { 58765e0ace2SJie Deng len = min_t(unsigned int, datalen, 58865e0ace2SJie Deng XLGMAC_TX_MAX_BUF_SIZE); 58965e0ace2SJie Deng 59065e0ace2SJie Deng skb_dma = skb_frag_dma_map(pdata->dev, frag, offset, 59165e0ace2SJie Deng len, DMA_TO_DEVICE); 59265e0ace2SJie Deng if (dma_mapping_error(pdata->dev, skb_dma)) { 59365e0ace2SJie Deng netdev_alert(pdata->netdev, 59465e0ace2SJie Deng "skb_frag_dma_map failed\n"); 59565e0ace2SJie Deng goto err_out; 59665e0ace2SJie Deng } 59765e0ace2SJie Deng desc_data->skb_dma = skb_dma; 59865e0ace2SJie Deng desc_data->skb_dma_len = len; 59965e0ace2SJie Deng desc_data->mapped_as_page = 1; 60065e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev, 60165e0ace2SJie Deng "skb frag: index=%u, dma=%pad, len=%u\n", 60265e0ace2SJie Deng cur_index, &skb_dma, len); 60365e0ace2SJie Deng 60465e0ace2SJie Deng datalen -= len; 60565e0ace2SJie Deng offset += len; 60665e0ace2SJie Deng 60765e0ace2SJie Deng pkt_info->length += len; 60865e0ace2SJie Deng 60965e0ace2SJie Deng cur_index++; 61065e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index); 61165e0ace2SJie Deng } 61265e0ace2SJie Deng } 61365e0ace2SJie Deng 61465e0ace2SJie Deng /* Save the skb address in the last entry. We always have some data 61565e0ace2SJie Deng * that has been mapped so desc_data is always advanced past the last 61665e0ace2SJie Deng * piece of mapped data - use the entry pointed to by cur_index - 1. 61765e0ace2SJie Deng */ 61865e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index - 1); 61965e0ace2SJie Deng desc_data->skb = skb; 62065e0ace2SJie Deng 62165e0ace2SJie Deng /* Save the number of descriptor entries used */ 62265e0ace2SJie Deng pkt_info->desc_count = cur_index - start_index; 62365e0ace2SJie Deng 62465e0ace2SJie Deng return pkt_info->desc_count; 62565e0ace2SJie Deng 62665e0ace2SJie Deng err_out: 62765e0ace2SJie Deng while (start_index < cur_index) { 62865e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, start_index++); 62965e0ace2SJie Deng xlgmac_unmap_desc_data(pdata, desc_data); 63065e0ace2SJie Deng } 63165e0ace2SJie Deng 63265e0ace2SJie Deng return 0; 63365e0ace2SJie Deng } 63465e0ace2SJie Deng 63565e0ace2SJie Deng void xlgmac_init_desc_ops(struct xlgmac_desc_ops *desc_ops) 63665e0ace2SJie Deng { 63765e0ace2SJie Deng desc_ops->alloc_channles_and_rings = xlgmac_alloc_channels_and_rings; 63865e0ace2SJie Deng desc_ops->free_channels_and_rings = xlgmac_free_channels_and_rings; 63965e0ace2SJie Deng desc_ops->map_tx_skb = xlgmac_map_tx_skb; 64065e0ace2SJie Deng desc_ops->map_rx_buffer = xlgmac_map_rx_buffer; 64165e0ace2SJie Deng desc_ops->unmap_desc_data = xlgmac_unmap_desc_data; 64265e0ace2SJie Deng desc_ops->tx_desc_init = xlgmac_tx_desc_init; 64365e0ace2SJie Deng desc_ops->rx_desc_init = xlgmac_rx_desc_init; 64465e0ace2SJie Deng } 645