165e0ace2SJie Deng /* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
265e0ace2SJie Deng *
365e0ace2SJie Deng * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
465e0ace2SJie Deng *
5ea8c1c64SJie Deng * This program is dual-licensed; you may select either version 2 of
6ea8c1c64SJie Deng * the GNU General Public License ("GPL") or BSD license ("BSD").
765e0ace2SJie Deng *
865e0ace2SJie Deng * This Synopsys DWC XLGMAC software driver and associated documentation
965e0ace2SJie Deng * (hereinafter the "Software") is an unsupported proprietary work of
1065e0ace2SJie Deng * Synopsys, Inc. unless otherwise expressly agreed to in writing between
1165e0ace2SJie Deng * Synopsys and you. The Software IS NOT an item of Licensed Software or a
1265e0ace2SJie Deng * Licensed Product under any End User Software License Agreement or
1365e0ace2SJie Deng * Agreement for Licensed Products with Synopsys or any supplement thereto.
1465e0ace2SJie Deng * Synopsys is a registered trademark of Synopsys, Inc. Other names included
1565e0ace2SJie Deng * in the SOFTWARE may be the trademarks of their respective owners.
1665e0ace2SJie Deng */
1765e0ace2SJie Deng
1865e0ace2SJie Deng #include "dwc-xlgmac.h"
1965e0ace2SJie Deng #include "dwc-xlgmac-reg.h"
2065e0ace2SJie Deng
xlgmac_unmap_desc_data(struct xlgmac_pdata * pdata,struct xlgmac_desc_data * desc_data)2165e0ace2SJie Deng static void xlgmac_unmap_desc_data(struct xlgmac_pdata *pdata,
2265e0ace2SJie Deng struct xlgmac_desc_data *desc_data)
2365e0ace2SJie Deng {
2465e0ace2SJie Deng if (desc_data->skb_dma) {
2565e0ace2SJie Deng if (desc_data->mapped_as_page) {
2665e0ace2SJie Deng dma_unmap_page(pdata->dev, desc_data->skb_dma,
2765e0ace2SJie Deng desc_data->skb_dma_len, DMA_TO_DEVICE);
2865e0ace2SJie Deng } else {
2965e0ace2SJie Deng dma_unmap_single(pdata->dev, desc_data->skb_dma,
3065e0ace2SJie Deng desc_data->skb_dma_len, DMA_TO_DEVICE);
3165e0ace2SJie Deng }
3265e0ace2SJie Deng desc_data->skb_dma = 0;
3365e0ace2SJie Deng desc_data->skb_dma_len = 0;
3465e0ace2SJie Deng }
3565e0ace2SJie Deng
3665e0ace2SJie Deng if (desc_data->skb) {
3765e0ace2SJie Deng dev_kfree_skb_any(desc_data->skb);
3865e0ace2SJie Deng desc_data->skb = NULL;
3965e0ace2SJie Deng }
4065e0ace2SJie Deng
4165e0ace2SJie Deng if (desc_data->rx.hdr.pa.pages)
4265e0ace2SJie Deng put_page(desc_data->rx.hdr.pa.pages);
4365e0ace2SJie Deng
4465e0ace2SJie Deng if (desc_data->rx.hdr.pa_unmap.pages) {
4565e0ace2SJie Deng dma_unmap_page(pdata->dev, desc_data->rx.hdr.pa_unmap.pages_dma,
4665e0ace2SJie Deng desc_data->rx.hdr.pa_unmap.pages_len,
4765e0ace2SJie Deng DMA_FROM_DEVICE);
4865e0ace2SJie Deng put_page(desc_data->rx.hdr.pa_unmap.pages);
4965e0ace2SJie Deng }
5065e0ace2SJie Deng
5165e0ace2SJie Deng if (desc_data->rx.buf.pa.pages)
5265e0ace2SJie Deng put_page(desc_data->rx.buf.pa.pages);
5365e0ace2SJie Deng
5465e0ace2SJie Deng if (desc_data->rx.buf.pa_unmap.pages) {
5565e0ace2SJie Deng dma_unmap_page(pdata->dev, desc_data->rx.buf.pa_unmap.pages_dma,
5665e0ace2SJie Deng desc_data->rx.buf.pa_unmap.pages_len,
5765e0ace2SJie Deng DMA_FROM_DEVICE);
5865e0ace2SJie Deng put_page(desc_data->rx.buf.pa_unmap.pages);
5965e0ace2SJie Deng }
6065e0ace2SJie Deng
6165e0ace2SJie Deng memset(&desc_data->tx, 0, sizeof(desc_data->tx));
6265e0ace2SJie Deng memset(&desc_data->rx, 0, sizeof(desc_data->rx));
6365e0ace2SJie Deng
6465e0ace2SJie Deng desc_data->mapped_as_page = 0;
6565e0ace2SJie Deng
6665e0ace2SJie Deng if (desc_data->state_saved) {
6765e0ace2SJie Deng desc_data->state_saved = 0;
6865e0ace2SJie Deng desc_data->state.skb = NULL;
6965e0ace2SJie Deng desc_data->state.len = 0;
7065e0ace2SJie Deng desc_data->state.error = 0;
7165e0ace2SJie Deng }
7265e0ace2SJie Deng }
7365e0ace2SJie Deng
xlgmac_free_ring(struct xlgmac_pdata * pdata,struct xlgmac_ring * ring)7465e0ace2SJie Deng static void xlgmac_free_ring(struct xlgmac_pdata *pdata,
7565e0ace2SJie Deng struct xlgmac_ring *ring)
7665e0ace2SJie Deng {
7765e0ace2SJie Deng struct xlgmac_desc_data *desc_data;
7865e0ace2SJie Deng unsigned int i;
7965e0ace2SJie Deng
8065e0ace2SJie Deng if (!ring)
8165e0ace2SJie Deng return;
8265e0ace2SJie Deng
8365e0ace2SJie Deng if (ring->desc_data_head) {
8465e0ace2SJie Deng for (i = 0; i < ring->dma_desc_count; i++) {
8565e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, i);
8665e0ace2SJie Deng xlgmac_unmap_desc_data(pdata, desc_data);
8765e0ace2SJie Deng }
8865e0ace2SJie Deng
8965e0ace2SJie Deng kfree(ring->desc_data_head);
9065e0ace2SJie Deng ring->desc_data_head = NULL;
9165e0ace2SJie Deng }
9265e0ace2SJie Deng
9365e0ace2SJie Deng if (ring->rx_hdr_pa.pages) {
9465e0ace2SJie Deng dma_unmap_page(pdata->dev, ring->rx_hdr_pa.pages_dma,
9565e0ace2SJie Deng ring->rx_hdr_pa.pages_len, DMA_FROM_DEVICE);
9665e0ace2SJie Deng put_page(ring->rx_hdr_pa.pages);
9765e0ace2SJie Deng
9865e0ace2SJie Deng ring->rx_hdr_pa.pages = NULL;
9965e0ace2SJie Deng ring->rx_hdr_pa.pages_len = 0;
10065e0ace2SJie Deng ring->rx_hdr_pa.pages_offset = 0;
10165e0ace2SJie Deng ring->rx_hdr_pa.pages_dma = 0;
10265e0ace2SJie Deng }
10365e0ace2SJie Deng
10465e0ace2SJie Deng if (ring->rx_buf_pa.pages) {
10565e0ace2SJie Deng dma_unmap_page(pdata->dev, ring->rx_buf_pa.pages_dma,
10665e0ace2SJie Deng ring->rx_buf_pa.pages_len, DMA_FROM_DEVICE);
10765e0ace2SJie Deng put_page(ring->rx_buf_pa.pages);
10865e0ace2SJie Deng
10965e0ace2SJie Deng ring->rx_buf_pa.pages = NULL;
11065e0ace2SJie Deng ring->rx_buf_pa.pages_len = 0;
11165e0ace2SJie Deng ring->rx_buf_pa.pages_offset = 0;
11265e0ace2SJie Deng ring->rx_buf_pa.pages_dma = 0;
11365e0ace2SJie Deng }
11465e0ace2SJie Deng
11565e0ace2SJie Deng if (ring->dma_desc_head) {
11665e0ace2SJie Deng dma_free_coherent(pdata->dev,
11765e0ace2SJie Deng (sizeof(struct xlgmac_dma_desc) *
11865e0ace2SJie Deng ring->dma_desc_count),
11965e0ace2SJie Deng ring->dma_desc_head,
12065e0ace2SJie Deng ring->dma_desc_head_addr);
12165e0ace2SJie Deng ring->dma_desc_head = NULL;
12265e0ace2SJie Deng }
12365e0ace2SJie Deng }
12465e0ace2SJie Deng
xlgmac_init_ring(struct xlgmac_pdata * pdata,struct xlgmac_ring * ring,unsigned int dma_desc_count)12565e0ace2SJie Deng static int xlgmac_init_ring(struct xlgmac_pdata *pdata,
12665e0ace2SJie Deng struct xlgmac_ring *ring,
12765e0ace2SJie Deng unsigned int dma_desc_count)
12865e0ace2SJie Deng {
12965e0ace2SJie Deng if (!ring)
13065e0ace2SJie Deng return 0;
13165e0ace2SJie Deng
13265e0ace2SJie Deng /* Descriptors */
13365e0ace2SJie Deng ring->dma_desc_count = dma_desc_count;
13465e0ace2SJie Deng ring->dma_desc_head = dma_alloc_coherent(pdata->dev,
13565e0ace2SJie Deng (sizeof(struct xlgmac_dma_desc) *
13665e0ace2SJie Deng dma_desc_count),
13765e0ace2SJie Deng &ring->dma_desc_head_addr,
13865e0ace2SJie Deng GFP_KERNEL);
13965e0ace2SJie Deng if (!ring->dma_desc_head)
14065e0ace2SJie Deng return -ENOMEM;
14165e0ace2SJie Deng
14265e0ace2SJie Deng /* Array of descriptor data */
14365e0ace2SJie Deng ring->desc_data_head = kcalloc(dma_desc_count,
14465e0ace2SJie Deng sizeof(struct xlgmac_desc_data),
14565e0ace2SJie Deng GFP_KERNEL);
14665e0ace2SJie Deng if (!ring->desc_data_head)
14765e0ace2SJie Deng return -ENOMEM;
14865e0ace2SJie Deng
14965e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev,
15065e0ace2SJie Deng "dma_desc_head=%p, dma_desc_head_addr=%pad, desc_data_head=%p\n",
15165e0ace2SJie Deng ring->dma_desc_head,
15265e0ace2SJie Deng &ring->dma_desc_head_addr,
15365e0ace2SJie Deng ring->desc_data_head);
15465e0ace2SJie Deng
15565e0ace2SJie Deng return 0;
15665e0ace2SJie Deng }
15765e0ace2SJie Deng
xlgmac_free_rings(struct xlgmac_pdata * pdata)15865e0ace2SJie Deng static void xlgmac_free_rings(struct xlgmac_pdata *pdata)
15965e0ace2SJie Deng {
16065e0ace2SJie Deng struct xlgmac_channel *channel;
16165e0ace2SJie Deng unsigned int i;
16265e0ace2SJie Deng
16365e0ace2SJie Deng if (!pdata->channel_head)
16465e0ace2SJie Deng return;
16565e0ace2SJie Deng
16665e0ace2SJie Deng channel = pdata->channel_head;
16765e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) {
16865e0ace2SJie Deng xlgmac_free_ring(pdata, channel->tx_ring);
16965e0ace2SJie Deng xlgmac_free_ring(pdata, channel->rx_ring);
17065e0ace2SJie Deng }
17165e0ace2SJie Deng }
17265e0ace2SJie Deng
xlgmac_alloc_rings(struct xlgmac_pdata * pdata)17365e0ace2SJie Deng static int xlgmac_alloc_rings(struct xlgmac_pdata *pdata)
17465e0ace2SJie Deng {
17565e0ace2SJie Deng struct xlgmac_channel *channel;
17665e0ace2SJie Deng unsigned int i;
17765e0ace2SJie Deng int ret;
17865e0ace2SJie Deng
17965e0ace2SJie Deng channel = pdata->channel_head;
18065e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) {
18165e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, "%s - Tx ring:\n",
18265e0ace2SJie Deng channel->name);
18365e0ace2SJie Deng
18465e0ace2SJie Deng ret = xlgmac_init_ring(pdata, channel->tx_ring,
18565e0ace2SJie Deng pdata->tx_desc_count);
18665e0ace2SJie Deng
18765e0ace2SJie Deng if (ret) {
18865e0ace2SJie Deng netdev_alert(pdata->netdev,
18965e0ace2SJie Deng "error initializing Tx ring");
19065e0ace2SJie Deng goto err_init_ring;
19165e0ace2SJie Deng }
19265e0ace2SJie Deng
19365e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev, "%s - Rx ring:\n",
19465e0ace2SJie Deng channel->name);
19565e0ace2SJie Deng
19665e0ace2SJie Deng ret = xlgmac_init_ring(pdata, channel->rx_ring,
19765e0ace2SJie Deng pdata->rx_desc_count);
19865e0ace2SJie Deng if (ret) {
19965e0ace2SJie Deng netdev_alert(pdata->netdev,
20065e0ace2SJie Deng "error initializing Rx ring\n");
20165e0ace2SJie Deng goto err_init_ring;
20265e0ace2SJie Deng }
20365e0ace2SJie Deng }
20465e0ace2SJie Deng
20565e0ace2SJie Deng return 0;
20665e0ace2SJie Deng
20765e0ace2SJie Deng err_init_ring:
20865e0ace2SJie Deng xlgmac_free_rings(pdata);
20965e0ace2SJie Deng
21065e0ace2SJie Deng return ret;
21165e0ace2SJie Deng }
21265e0ace2SJie Deng
xlgmac_free_channels(struct xlgmac_pdata * pdata)21365e0ace2SJie Deng static void xlgmac_free_channels(struct xlgmac_pdata *pdata)
21465e0ace2SJie Deng {
21565e0ace2SJie Deng if (!pdata->channel_head)
21665e0ace2SJie Deng return;
21765e0ace2SJie Deng
21865e0ace2SJie Deng kfree(pdata->channel_head->tx_ring);
21965e0ace2SJie Deng pdata->channel_head->tx_ring = NULL;
22065e0ace2SJie Deng
22165e0ace2SJie Deng kfree(pdata->channel_head->rx_ring);
22265e0ace2SJie Deng pdata->channel_head->rx_ring = NULL;
22365e0ace2SJie Deng
22465e0ace2SJie Deng kfree(pdata->channel_head);
22565e0ace2SJie Deng
22665e0ace2SJie Deng pdata->channel_head = NULL;
22765e0ace2SJie Deng pdata->channel_count = 0;
22865e0ace2SJie Deng }
22965e0ace2SJie Deng
xlgmac_alloc_channels(struct xlgmac_pdata * pdata)23065e0ace2SJie Deng static int xlgmac_alloc_channels(struct xlgmac_pdata *pdata)
23165e0ace2SJie Deng {
23265e0ace2SJie Deng struct xlgmac_channel *channel_head, *channel;
23365e0ace2SJie Deng struct xlgmac_ring *tx_ring, *rx_ring;
23465e0ace2SJie Deng int ret = -ENOMEM;
23565e0ace2SJie Deng unsigned int i;
23665e0ace2SJie Deng
23765e0ace2SJie Deng channel_head = kcalloc(pdata->channel_count,
23865e0ace2SJie Deng sizeof(struct xlgmac_channel), GFP_KERNEL);
23965e0ace2SJie Deng if (!channel_head)
24065e0ace2SJie Deng return ret;
24165e0ace2SJie Deng
24265e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev,
24365e0ace2SJie Deng "channel_head=%p\n", channel_head);
24465e0ace2SJie Deng
24565e0ace2SJie Deng tx_ring = kcalloc(pdata->tx_ring_count, sizeof(struct xlgmac_ring),
24665e0ace2SJie Deng GFP_KERNEL);
24765e0ace2SJie Deng if (!tx_ring)
24865e0ace2SJie Deng goto err_tx_ring;
24965e0ace2SJie Deng
25065e0ace2SJie Deng rx_ring = kcalloc(pdata->rx_ring_count, sizeof(struct xlgmac_ring),
25165e0ace2SJie Deng GFP_KERNEL);
25265e0ace2SJie Deng if (!rx_ring)
25365e0ace2SJie Deng goto err_rx_ring;
25465e0ace2SJie Deng
25565e0ace2SJie Deng for (i = 0, channel = channel_head; i < pdata->channel_count;
25665e0ace2SJie Deng i++, channel++) {
25765e0ace2SJie Deng snprintf(channel->name, sizeof(channel->name), "channel-%u", i);
25865e0ace2SJie Deng channel->pdata = pdata;
25965e0ace2SJie Deng channel->queue_index = i;
26065e0ace2SJie Deng channel->dma_regs = pdata->mac_regs + DMA_CH_BASE +
26165e0ace2SJie Deng (DMA_CH_INC * i);
26265e0ace2SJie Deng
26365e0ace2SJie Deng if (pdata->per_channel_irq) {
26465e0ace2SJie Deng /* Get the per DMA interrupt */
26565e0ace2SJie Deng ret = pdata->channel_irq[i];
26665e0ace2SJie Deng if (ret < 0) {
26765e0ace2SJie Deng netdev_err(pdata->netdev,
26865e0ace2SJie Deng "get_irq %u failed\n",
26965e0ace2SJie Deng i + 1);
27065e0ace2SJie Deng goto err_irq;
27165e0ace2SJie Deng }
27265e0ace2SJie Deng channel->dma_irq = ret;
27365e0ace2SJie Deng }
27465e0ace2SJie Deng
27565e0ace2SJie Deng if (i < pdata->tx_ring_count)
27665e0ace2SJie Deng channel->tx_ring = tx_ring++;
27765e0ace2SJie Deng
27865e0ace2SJie Deng if (i < pdata->rx_ring_count)
27965e0ace2SJie Deng channel->rx_ring = rx_ring++;
28065e0ace2SJie Deng
28165e0ace2SJie Deng netif_dbg(pdata, drv, pdata->netdev,
28265e0ace2SJie Deng "%s: dma_regs=%p, tx_ring=%p, rx_ring=%p\n",
28365e0ace2SJie Deng channel->name, channel->dma_regs,
28465e0ace2SJie Deng channel->tx_ring, channel->rx_ring);
28565e0ace2SJie Deng }
28665e0ace2SJie Deng
28765e0ace2SJie Deng pdata->channel_head = channel_head;
28865e0ace2SJie Deng
28965e0ace2SJie Deng return 0;
29065e0ace2SJie Deng
29165e0ace2SJie Deng err_irq:
29265e0ace2SJie Deng kfree(rx_ring);
29365e0ace2SJie Deng
29465e0ace2SJie Deng err_rx_ring:
29565e0ace2SJie Deng kfree(tx_ring);
29665e0ace2SJie Deng
29765e0ace2SJie Deng err_tx_ring:
29865e0ace2SJie Deng kfree(channel_head);
29965e0ace2SJie Deng
30065e0ace2SJie Deng return ret;
30165e0ace2SJie Deng }
30265e0ace2SJie Deng
xlgmac_free_channels_and_rings(struct xlgmac_pdata * pdata)30365e0ace2SJie Deng static void xlgmac_free_channels_and_rings(struct xlgmac_pdata *pdata)
30465e0ace2SJie Deng {
30565e0ace2SJie Deng xlgmac_free_rings(pdata);
30665e0ace2SJie Deng
30765e0ace2SJie Deng xlgmac_free_channels(pdata);
30865e0ace2SJie Deng }
30965e0ace2SJie Deng
xlgmac_alloc_channels_and_rings(struct xlgmac_pdata * pdata)31065e0ace2SJie Deng static int xlgmac_alloc_channels_and_rings(struct xlgmac_pdata *pdata)
31165e0ace2SJie Deng {
31265e0ace2SJie Deng int ret;
31365e0ace2SJie Deng
31465e0ace2SJie Deng ret = xlgmac_alloc_channels(pdata);
31565e0ace2SJie Deng if (ret)
31665e0ace2SJie Deng goto err_alloc;
31765e0ace2SJie Deng
31865e0ace2SJie Deng ret = xlgmac_alloc_rings(pdata);
31965e0ace2SJie Deng if (ret)
32065e0ace2SJie Deng goto err_alloc;
32165e0ace2SJie Deng
32265e0ace2SJie Deng return 0;
32365e0ace2SJie Deng
32465e0ace2SJie Deng err_alloc:
32565e0ace2SJie Deng xlgmac_free_channels_and_rings(pdata);
32665e0ace2SJie Deng
32765e0ace2SJie Deng return ret;
32865e0ace2SJie Deng }
32965e0ace2SJie Deng
xlgmac_alloc_pages(struct xlgmac_pdata * pdata,struct xlgmac_page_alloc * pa,gfp_t gfp,int order)33065e0ace2SJie Deng static int xlgmac_alloc_pages(struct xlgmac_pdata *pdata,
33165e0ace2SJie Deng struct xlgmac_page_alloc *pa,
33265e0ace2SJie Deng gfp_t gfp, int order)
33365e0ace2SJie Deng {
33465e0ace2SJie Deng struct page *pages = NULL;
33565e0ace2SJie Deng dma_addr_t pages_dma;
33665e0ace2SJie Deng
33765e0ace2SJie Deng /* Try to obtain pages, decreasing order if necessary */
338453f85d4SMel Gorman gfp |= __GFP_COMP | __GFP_NOWARN;
33965e0ace2SJie Deng while (order >= 0) {
34065e0ace2SJie Deng pages = alloc_pages(gfp, order);
34165e0ace2SJie Deng if (pages)
34265e0ace2SJie Deng break;
34365e0ace2SJie Deng
34465e0ace2SJie Deng order--;
34565e0ace2SJie Deng }
34665e0ace2SJie Deng if (!pages)
34765e0ace2SJie Deng return -ENOMEM;
34865e0ace2SJie Deng
34965e0ace2SJie Deng /* Map the pages */
35065e0ace2SJie Deng pages_dma = dma_map_page(pdata->dev, pages, 0,
35165e0ace2SJie Deng PAGE_SIZE << order, DMA_FROM_DEVICE);
35231c7ba9eSDan Carpenter if (dma_mapping_error(pdata->dev, pages_dma)) {
35365e0ace2SJie Deng put_page(pages);
35431c7ba9eSDan Carpenter return -ENOMEM;
35565e0ace2SJie Deng }
35665e0ace2SJie Deng
35765e0ace2SJie Deng pa->pages = pages;
35865e0ace2SJie Deng pa->pages_len = PAGE_SIZE << order;
35965e0ace2SJie Deng pa->pages_offset = 0;
36065e0ace2SJie Deng pa->pages_dma = pages_dma;
36165e0ace2SJie Deng
36265e0ace2SJie Deng return 0;
36365e0ace2SJie Deng }
36465e0ace2SJie Deng
xlgmac_set_buffer_data(struct xlgmac_buffer_data * bd,struct xlgmac_page_alloc * pa,unsigned int len)36565e0ace2SJie Deng static void xlgmac_set_buffer_data(struct xlgmac_buffer_data *bd,
36665e0ace2SJie Deng struct xlgmac_page_alloc *pa,
36765e0ace2SJie Deng unsigned int len)
36865e0ace2SJie Deng {
36965e0ace2SJie Deng get_page(pa->pages);
37065e0ace2SJie Deng bd->pa = *pa;
37165e0ace2SJie Deng
37265e0ace2SJie Deng bd->dma_base = pa->pages_dma;
37365e0ace2SJie Deng bd->dma_off = pa->pages_offset;
37465e0ace2SJie Deng bd->dma_len = len;
37565e0ace2SJie Deng
37665e0ace2SJie Deng pa->pages_offset += len;
37765e0ace2SJie Deng if ((pa->pages_offset + len) > pa->pages_len) {
37865e0ace2SJie Deng /* This data descriptor is responsible for unmapping page(s) */
37965e0ace2SJie Deng bd->pa_unmap = *pa;
38065e0ace2SJie Deng
38165e0ace2SJie Deng /* Get a new allocation next time */
38265e0ace2SJie Deng pa->pages = NULL;
38365e0ace2SJie Deng pa->pages_len = 0;
38465e0ace2SJie Deng pa->pages_offset = 0;
38565e0ace2SJie Deng pa->pages_dma = 0;
38665e0ace2SJie Deng }
38765e0ace2SJie Deng }
38865e0ace2SJie Deng
xlgmac_map_rx_buffer(struct xlgmac_pdata * pdata,struct xlgmac_ring * ring,struct xlgmac_desc_data * desc_data)38965e0ace2SJie Deng static int xlgmac_map_rx_buffer(struct xlgmac_pdata *pdata,
39065e0ace2SJie Deng struct xlgmac_ring *ring,
39165e0ace2SJie Deng struct xlgmac_desc_data *desc_data)
39265e0ace2SJie Deng {
39365e0ace2SJie Deng int order, ret;
39465e0ace2SJie Deng
39565e0ace2SJie Deng if (!ring->rx_hdr_pa.pages) {
39665e0ace2SJie Deng ret = xlgmac_alloc_pages(pdata, &ring->rx_hdr_pa,
39765e0ace2SJie Deng GFP_ATOMIC, 0);
39865e0ace2SJie Deng if (ret)
39965e0ace2SJie Deng return ret;
40065e0ace2SJie Deng }
40165e0ace2SJie Deng
40265e0ace2SJie Deng if (!ring->rx_buf_pa.pages) {
40365e0ace2SJie Deng order = max_t(int, PAGE_ALLOC_COSTLY_ORDER - 1, 0);
40465e0ace2SJie Deng ret = xlgmac_alloc_pages(pdata, &ring->rx_buf_pa,
40565e0ace2SJie Deng GFP_ATOMIC, order);
40665e0ace2SJie Deng if (ret)
40765e0ace2SJie Deng return ret;
40865e0ace2SJie Deng }
40965e0ace2SJie Deng
41065e0ace2SJie Deng /* Set up the header page info */
41165e0ace2SJie Deng xlgmac_set_buffer_data(&desc_data->rx.hdr, &ring->rx_hdr_pa,
41265e0ace2SJie Deng XLGMAC_SKB_ALLOC_SIZE);
41365e0ace2SJie Deng
41465e0ace2SJie Deng /* Set up the buffer page info */
41565e0ace2SJie Deng xlgmac_set_buffer_data(&desc_data->rx.buf, &ring->rx_buf_pa,
41665e0ace2SJie Deng pdata->rx_buf_size);
41765e0ace2SJie Deng
41865e0ace2SJie Deng return 0;
41965e0ace2SJie Deng }
42065e0ace2SJie Deng
xlgmac_tx_desc_init(struct xlgmac_pdata * pdata)42165e0ace2SJie Deng static void xlgmac_tx_desc_init(struct xlgmac_pdata *pdata)
42265e0ace2SJie Deng {
42365e0ace2SJie Deng struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
42465e0ace2SJie Deng struct xlgmac_desc_data *desc_data;
42565e0ace2SJie Deng struct xlgmac_dma_desc *dma_desc;
42665e0ace2SJie Deng struct xlgmac_channel *channel;
42765e0ace2SJie Deng struct xlgmac_ring *ring;
42865e0ace2SJie Deng dma_addr_t dma_desc_addr;
42965e0ace2SJie Deng unsigned int i, j;
43065e0ace2SJie Deng
43165e0ace2SJie Deng channel = pdata->channel_head;
43265e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) {
43365e0ace2SJie Deng ring = channel->tx_ring;
43465e0ace2SJie Deng if (!ring)
43565e0ace2SJie Deng break;
43665e0ace2SJie Deng
43765e0ace2SJie Deng dma_desc = ring->dma_desc_head;
43865e0ace2SJie Deng dma_desc_addr = ring->dma_desc_head_addr;
43965e0ace2SJie Deng
44065e0ace2SJie Deng for (j = 0; j < ring->dma_desc_count; j++) {
44165e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, j);
44265e0ace2SJie Deng
44365e0ace2SJie Deng desc_data->dma_desc = dma_desc;
44465e0ace2SJie Deng desc_data->dma_desc_addr = dma_desc_addr;
44565e0ace2SJie Deng
44665e0ace2SJie Deng dma_desc++;
44765e0ace2SJie Deng dma_desc_addr += sizeof(struct xlgmac_dma_desc);
44865e0ace2SJie Deng }
44965e0ace2SJie Deng
45065e0ace2SJie Deng ring->cur = 0;
45165e0ace2SJie Deng ring->dirty = 0;
45265e0ace2SJie Deng memset(&ring->tx, 0, sizeof(ring->tx));
45365e0ace2SJie Deng
45465e0ace2SJie Deng hw_ops->tx_desc_init(channel);
45565e0ace2SJie Deng }
45665e0ace2SJie Deng }
45765e0ace2SJie Deng
xlgmac_rx_desc_init(struct xlgmac_pdata * pdata)45865e0ace2SJie Deng static void xlgmac_rx_desc_init(struct xlgmac_pdata *pdata)
45965e0ace2SJie Deng {
46065e0ace2SJie Deng struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
46165e0ace2SJie Deng struct xlgmac_desc_data *desc_data;
46265e0ace2SJie Deng struct xlgmac_dma_desc *dma_desc;
46365e0ace2SJie Deng struct xlgmac_channel *channel;
46465e0ace2SJie Deng struct xlgmac_ring *ring;
46565e0ace2SJie Deng dma_addr_t dma_desc_addr;
46665e0ace2SJie Deng unsigned int i, j;
46765e0ace2SJie Deng
46865e0ace2SJie Deng channel = pdata->channel_head;
46965e0ace2SJie Deng for (i = 0; i < pdata->channel_count; i++, channel++) {
47065e0ace2SJie Deng ring = channel->rx_ring;
47165e0ace2SJie Deng if (!ring)
47265e0ace2SJie Deng break;
47365e0ace2SJie Deng
47465e0ace2SJie Deng dma_desc = ring->dma_desc_head;
47565e0ace2SJie Deng dma_desc_addr = ring->dma_desc_head_addr;
47665e0ace2SJie Deng
47765e0ace2SJie Deng for (j = 0; j < ring->dma_desc_count; j++) {
47865e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, j);
47965e0ace2SJie Deng
48065e0ace2SJie Deng desc_data->dma_desc = dma_desc;
48165e0ace2SJie Deng desc_data->dma_desc_addr = dma_desc_addr;
48265e0ace2SJie Deng
48365e0ace2SJie Deng if (xlgmac_map_rx_buffer(pdata, ring, desc_data))
48465e0ace2SJie Deng break;
48565e0ace2SJie Deng
48665e0ace2SJie Deng dma_desc++;
48765e0ace2SJie Deng dma_desc_addr += sizeof(struct xlgmac_dma_desc);
48865e0ace2SJie Deng }
48965e0ace2SJie Deng
49065e0ace2SJie Deng ring->cur = 0;
49165e0ace2SJie Deng ring->dirty = 0;
49265e0ace2SJie Deng
49365e0ace2SJie Deng hw_ops->rx_desc_init(channel);
49465e0ace2SJie Deng }
49565e0ace2SJie Deng }
49665e0ace2SJie Deng
xlgmac_map_tx_skb(struct xlgmac_channel * channel,struct sk_buff * skb)49765e0ace2SJie Deng static int xlgmac_map_tx_skb(struct xlgmac_channel *channel,
49865e0ace2SJie Deng struct sk_buff *skb)
49965e0ace2SJie Deng {
50065e0ace2SJie Deng struct xlgmac_pdata *pdata = channel->pdata;
50165e0ace2SJie Deng struct xlgmac_ring *ring = channel->tx_ring;
50265e0ace2SJie Deng unsigned int start_index, cur_index;
50365e0ace2SJie Deng struct xlgmac_desc_data *desc_data;
50465e0ace2SJie Deng unsigned int offset, datalen, len;
50565e0ace2SJie Deng struct xlgmac_pkt_info *pkt_info;
506d7840976SMatthew Wilcox (Oracle) skb_frag_t *frag;
50765e0ace2SJie Deng unsigned int tso, vlan;
50865e0ace2SJie Deng dma_addr_t skb_dma;
50965e0ace2SJie Deng unsigned int i;
51065e0ace2SJie Deng
51165e0ace2SJie Deng offset = 0;
51265e0ace2SJie Deng start_index = ring->cur;
51365e0ace2SJie Deng cur_index = ring->cur;
51465e0ace2SJie Deng
51565e0ace2SJie Deng pkt_info = &ring->pkt_info;
51665e0ace2SJie Deng pkt_info->desc_count = 0;
51765e0ace2SJie Deng pkt_info->length = 0;
51865e0ace2SJie Deng
51965e0ace2SJie Deng tso = XLGMAC_GET_REG_BITS(pkt_info->attributes,
52065e0ace2SJie Deng TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
52165e0ace2SJie Deng TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN);
52265e0ace2SJie Deng vlan = XLGMAC_GET_REG_BITS(pkt_info->attributes,
52365e0ace2SJie Deng TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
52465e0ace2SJie Deng TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN);
52565e0ace2SJie Deng
52665e0ace2SJie Deng /* Save space for a context descriptor if needed */
52765e0ace2SJie Deng if ((tso && (pkt_info->mss != ring->tx.cur_mss)) ||
52865e0ace2SJie Deng (vlan && (pkt_info->vlan_ctag != ring->tx.cur_vlan_ctag)))
52965e0ace2SJie Deng cur_index++;
53065e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
53165e0ace2SJie Deng
53265e0ace2SJie Deng if (tso) {
53365e0ace2SJie Deng /* Map the TSO header */
53465e0ace2SJie Deng skb_dma = dma_map_single(pdata->dev, skb->data,
53565e0ace2SJie Deng pkt_info->header_len, DMA_TO_DEVICE);
53665e0ace2SJie Deng if (dma_mapping_error(pdata->dev, skb_dma)) {
53765e0ace2SJie Deng netdev_alert(pdata->netdev, "dma_map_single failed\n");
53865e0ace2SJie Deng goto err_out;
53965e0ace2SJie Deng }
54065e0ace2SJie Deng desc_data->skb_dma = skb_dma;
54165e0ace2SJie Deng desc_data->skb_dma_len = pkt_info->header_len;
54265e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev,
54365e0ace2SJie Deng "skb header: index=%u, dma=%pad, len=%u\n",
54465e0ace2SJie Deng cur_index, &skb_dma, pkt_info->header_len);
54565e0ace2SJie Deng
54665e0ace2SJie Deng offset = pkt_info->header_len;
54765e0ace2SJie Deng
54865e0ace2SJie Deng pkt_info->length += pkt_info->header_len;
54965e0ace2SJie Deng
55065e0ace2SJie Deng cur_index++;
55165e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
55265e0ace2SJie Deng }
55365e0ace2SJie Deng
55465e0ace2SJie Deng /* Map the (remainder of the) packet */
55565e0ace2SJie Deng for (datalen = skb_headlen(skb) - offset; datalen; ) {
55665e0ace2SJie Deng len = min_t(unsigned int, datalen, XLGMAC_TX_MAX_BUF_SIZE);
55765e0ace2SJie Deng
55865e0ace2SJie Deng skb_dma = dma_map_single(pdata->dev, skb->data + offset, len,
55965e0ace2SJie Deng DMA_TO_DEVICE);
56065e0ace2SJie Deng if (dma_mapping_error(pdata->dev, skb_dma)) {
56165e0ace2SJie Deng netdev_alert(pdata->netdev, "dma_map_single failed\n");
56265e0ace2SJie Deng goto err_out;
56365e0ace2SJie Deng }
56465e0ace2SJie Deng desc_data->skb_dma = skb_dma;
56565e0ace2SJie Deng desc_data->skb_dma_len = len;
56665e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev,
56765e0ace2SJie Deng "skb data: index=%u, dma=%pad, len=%u\n",
56865e0ace2SJie Deng cur_index, &skb_dma, len);
56965e0ace2SJie Deng
57065e0ace2SJie Deng datalen -= len;
57165e0ace2SJie Deng offset += len;
57265e0ace2SJie Deng
57365e0ace2SJie Deng pkt_info->length += len;
57465e0ace2SJie Deng
57565e0ace2SJie Deng cur_index++;
57665e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
57765e0ace2SJie Deng }
57865e0ace2SJie Deng
57965e0ace2SJie Deng for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
58065e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev,
58165e0ace2SJie Deng "mapping frag %u\n", i);
58265e0ace2SJie Deng
58365e0ace2SJie Deng frag = &skb_shinfo(skb)->frags[i];
58465e0ace2SJie Deng offset = 0;
58565e0ace2SJie Deng
58665e0ace2SJie Deng for (datalen = skb_frag_size(frag); datalen; ) {
58765e0ace2SJie Deng len = min_t(unsigned int, datalen,
58865e0ace2SJie Deng XLGMAC_TX_MAX_BUF_SIZE);
58965e0ace2SJie Deng
59065e0ace2SJie Deng skb_dma = skb_frag_dma_map(pdata->dev, frag, offset,
59165e0ace2SJie Deng len, DMA_TO_DEVICE);
59265e0ace2SJie Deng if (dma_mapping_error(pdata->dev, skb_dma)) {
59365e0ace2SJie Deng netdev_alert(pdata->netdev,
59465e0ace2SJie Deng "skb_frag_dma_map failed\n");
59565e0ace2SJie Deng goto err_out;
59665e0ace2SJie Deng }
59765e0ace2SJie Deng desc_data->skb_dma = skb_dma;
59865e0ace2SJie Deng desc_data->skb_dma_len = len;
59965e0ace2SJie Deng desc_data->mapped_as_page = 1;
60065e0ace2SJie Deng netif_dbg(pdata, tx_queued, pdata->netdev,
60165e0ace2SJie Deng "skb frag: index=%u, dma=%pad, len=%u\n",
60265e0ace2SJie Deng cur_index, &skb_dma, len);
60365e0ace2SJie Deng
60465e0ace2SJie Deng datalen -= len;
60565e0ace2SJie Deng offset += len;
60665e0ace2SJie Deng
60765e0ace2SJie Deng pkt_info->length += len;
60865e0ace2SJie Deng
60965e0ace2SJie Deng cur_index++;
61065e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);
61165e0ace2SJie Deng }
61265e0ace2SJie Deng }
61365e0ace2SJie Deng
61465e0ace2SJie Deng /* Save the skb address in the last entry. We always have some data
61565e0ace2SJie Deng * that has been mapped so desc_data is always advanced past the last
61665e0ace2SJie Deng * piece of mapped data - use the entry pointed to by cur_index - 1.
61765e0ace2SJie Deng */
61865e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index - 1);
61965e0ace2SJie Deng desc_data->skb = skb;
62065e0ace2SJie Deng
62165e0ace2SJie Deng /* Save the number of descriptor entries used */
62265e0ace2SJie Deng pkt_info->desc_count = cur_index - start_index;
62365e0ace2SJie Deng
62465e0ace2SJie Deng return pkt_info->desc_count;
62565e0ace2SJie Deng
62665e0ace2SJie Deng err_out:
62765e0ace2SJie Deng while (start_index < cur_index) {
62865e0ace2SJie Deng desc_data = XLGMAC_GET_DESC_DATA(ring, start_index++);
62965e0ace2SJie Deng xlgmac_unmap_desc_data(pdata, desc_data);
63065e0ace2SJie Deng }
63165e0ace2SJie Deng
63265e0ace2SJie Deng return 0;
63365e0ace2SJie Deng }
63465e0ace2SJie Deng
xlgmac_init_desc_ops(struct xlgmac_desc_ops * desc_ops)63565e0ace2SJie Deng void xlgmac_init_desc_ops(struct xlgmac_desc_ops *desc_ops)
63665e0ace2SJie Deng {
637*a455fcd7SColin Ian King desc_ops->alloc_channels_and_rings = xlgmac_alloc_channels_and_rings;
63865e0ace2SJie Deng desc_ops->free_channels_and_rings = xlgmac_free_channels_and_rings;
63965e0ace2SJie Deng desc_ops->map_tx_skb = xlgmac_map_tx_skb;
64065e0ace2SJie Deng desc_ops->map_rx_buffer = xlgmac_map_rx_buffer;
64165e0ace2SJie Deng desc_ops->unmap_desc_data = xlgmac_unmap_desc_data;
64265e0ace2SJie Deng desc_ops->tx_desc_init = xlgmac_tx_desc_init;
64365e0ace2SJie Deng desc_ops->rx_desc_init = xlgmac_rx_desc_init;
64465e0ace2SJie Deng }
645