132d31c79SAmireddy Mallikarjuna reddy // SPDX-License-Identifier: GPL-2.0
232d31c79SAmireddy Mallikarjuna reddy /*
332d31c79SAmireddy Mallikarjuna reddy * Lightning Mountain centralized DMA controller driver
432d31c79SAmireddy Mallikarjuna reddy *
532d31c79SAmireddy Mallikarjuna reddy * Copyright (c) 2016 - 2020 Intel Corporation.
632d31c79SAmireddy Mallikarjuna reddy */
732d31c79SAmireddy Mallikarjuna reddy
832d31c79SAmireddy Mallikarjuna reddy #include <linux/bitfield.h>
932d31c79SAmireddy Mallikarjuna reddy #include <linux/clk.h>
1032d31c79SAmireddy Mallikarjuna reddy #include <linux/dma-mapping.h>
1132d31c79SAmireddy Mallikarjuna reddy #include <linux/dmapool.h>
1232d31c79SAmireddy Mallikarjuna reddy #include <linux/err.h>
1332d31c79SAmireddy Mallikarjuna reddy #include <linux/export.h>
1432d31c79SAmireddy Mallikarjuna reddy #include <linux/init.h>
1532d31c79SAmireddy Mallikarjuna reddy #include <linux/interrupt.h>
1632d31c79SAmireddy Mallikarjuna reddy #include <linux/iopoll.h>
1732d31c79SAmireddy Mallikarjuna reddy #include <linux/of_dma.h>
1832d31c79SAmireddy Mallikarjuna reddy #include <linux/of_irq.h>
1932d31c79SAmireddy Mallikarjuna reddy #include <linux/platform_device.h>
2032d31c79SAmireddy Mallikarjuna reddy #include <linux/reset.h>
2132d31c79SAmireddy Mallikarjuna reddy
2232d31c79SAmireddy Mallikarjuna reddy #include "../dmaengine.h"
2332d31c79SAmireddy Mallikarjuna reddy #include "../virt-dma.h"
2432d31c79SAmireddy Mallikarjuna reddy
2532d31c79SAmireddy Mallikarjuna reddy #define DRIVER_NAME "lgm-dma"
2632d31c79SAmireddy Mallikarjuna reddy
2732d31c79SAmireddy Mallikarjuna reddy #define DMA_ID 0x0008
2832d31c79SAmireddy Mallikarjuna reddy #define DMA_ID_REV GENMASK(7, 0)
2932d31c79SAmireddy Mallikarjuna reddy #define DMA_ID_PNR GENMASK(19, 16)
3032d31c79SAmireddy Mallikarjuna reddy #define DMA_ID_CHNR GENMASK(26, 20)
3132d31c79SAmireddy Mallikarjuna reddy #define DMA_ID_DW_128B BIT(27)
3232d31c79SAmireddy Mallikarjuna reddy #define DMA_ID_AW_36B BIT(28)
3332d31c79SAmireddy Mallikarjuna reddy #define DMA_VER32 0x32
3432d31c79SAmireddy Mallikarjuna reddy #define DMA_VER31 0x31
3532d31c79SAmireddy Mallikarjuna reddy #define DMA_VER22 0x0A
3632d31c79SAmireddy Mallikarjuna reddy
3732d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL 0x0010
3832d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_RST BIT(0)
3932d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_DSRAM_PATH BIT(1)
4032d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_DBURST_WR BIT(3)
4132d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_VLD_DF_ACK BIT(4)
4232d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_CH_FL BIT(6)
4332d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_DS_FOD BIT(7)
4432d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_DRB BIT(8)
4532d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_ENBE BIT(9)
4632d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_DESC_TMOUT_CNT_V31 GENMASK(27, 16)
4732d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_DESC_TMOUT_EN_V31 BIT(30)
4832d31c79SAmireddy Mallikarjuna reddy #define DMA_CTRL_PKTARB BIT(31)
4932d31c79SAmireddy Mallikarjuna reddy
5032d31c79SAmireddy Mallikarjuna reddy #define DMA_CPOLL 0x0014
5132d31c79SAmireddy Mallikarjuna reddy #define DMA_CPOLL_CNT GENMASK(15, 4)
5232d31c79SAmireddy Mallikarjuna reddy #define DMA_CPOLL_EN BIT(31)
5332d31c79SAmireddy Mallikarjuna reddy
5432d31c79SAmireddy Mallikarjuna reddy #define DMA_CS 0x0018
5532d31c79SAmireddy Mallikarjuna reddy #define DMA_CS_MASK GENMASK(5, 0)
5632d31c79SAmireddy Mallikarjuna reddy
5732d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL 0x001C
5832d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_ON BIT(0)
5932d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_RST BIT(1)
6032d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_CH_POLL_EN BIT(2)
6132d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_CH_ABC BIT(3) /* Adaptive Burst Chop */
6232d31c79SAmireddy Mallikarjuna reddy #define DMA_CDBA_MSB GENMASK(7, 4)
6332d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_DIR_TX BIT(8)
6432d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_CLASS GENMASK(11, 9)
6532d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_CLASSH GENMASK(19, 18)
6632d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_WR_NP_EN BIT(21)
6732d31c79SAmireddy Mallikarjuna reddy #define DMA_CCTRL_PDEN BIT(23)
6832d31c79SAmireddy Mallikarjuna reddy #define DMA_MAX_CLASS (SZ_32 - 1)
6932d31c79SAmireddy Mallikarjuna reddy
7032d31c79SAmireddy Mallikarjuna reddy #define DMA_CDBA 0x0020
7132d31c79SAmireddy Mallikarjuna reddy #define DMA_CDLEN 0x0024
7232d31c79SAmireddy Mallikarjuna reddy #define DMA_CIS 0x0028
7332d31c79SAmireddy Mallikarjuna reddy #define DMA_CIE 0x002C
7432d31c79SAmireddy Mallikarjuna reddy #define DMA_CI_EOP BIT(1)
7532d31c79SAmireddy Mallikarjuna reddy #define DMA_CI_DUR BIT(2)
7632d31c79SAmireddy Mallikarjuna reddy #define DMA_CI_DESCPT BIT(3)
7732d31c79SAmireddy Mallikarjuna reddy #define DMA_CI_CHOFF BIT(4)
7832d31c79SAmireddy Mallikarjuna reddy #define DMA_CI_RDERR BIT(5)
7932d31c79SAmireddy Mallikarjuna reddy #define DMA_CI_ALL \
8032d31c79SAmireddy Mallikarjuna reddy (DMA_CI_EOP | DMA_CI_DUR | DMA_CI_DESCPT | DMA_CI_CHOFF | DMA_CI_RDERR)
8132d31c79SAmireddy Mallikarjuna reddy
8232d31c79SAmireddy Mallikarjuna reddy #define DMA_PS 0x0040
8332d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL 0x0044
8432d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_RXBL16 BIT(0)
8532d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_TXBL16 BIT(1)
8632d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_RXBL GENMASK(3, 2)
8732d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_RXBL_8 3
8832d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_TXBL GENMASK(5, 4)
8932d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_TXBL_8 3
9032d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_PDEN BIT(6)
9132d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_RXBL32 BIT(7)
9232d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_RXENDI GENMASK(9, 8)
9332d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_TXENDI GENMASK(11, 10)
9432d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_TXBL32 BIT(15)
9532d31c79SAmireddy Mallikarjuna reddy #define DMA_PCTRL_MEM_FLUSH BIT(16)
9632d31c79SAmireddy Mallikarjuna reddy
9732d31c79SAmireddy Mallikarjuna reddy #define DMA_IRNEN1 0x00E8
9832d31c79SAmireddy Mallikarjuna reddy #define DMA_IRNCR1 0x00EC
9932d31c79SAmireddy Mallikarjuna reddy #define DMA_IRNEN 0x00F4
10032d31c79SAmireddy Mallikarjuna reddy #define DMA_IRNCR 0x00F8
10132d31c79SAmireddy Mallikarjuna reddy #define DMA_C_DP_TICK 0x100
10232d31c79SAmireddy Mallikarjuna reddy #define DMA_C_DP_TICK_TIKNARB GENMASK(15, 0)
10332d31c79SAmireddy Mallikarjuna reddy #define DMA_C_DP_TICK_TIKARB GENMASK(31, 16)
10432d31c79SAmireddy Mallikarjuna reddy
10532d31c79SAmireddy Mallikarjuna reddy #define DMA_C_HDRM 0x110
10632d31c79SAmireddy Mallikarjuna reddy /*
10732d31c79SAmireddy Mallikarjuna reddy * If header mode is set in DMA descriptor,
10832d31c79SAmireddy Mallikarjuna reddy * If bit 30 is disabled, HDR_LEN must be configured according to channel
10932d31c79SAmireddy Mallikarjuna reddy * requirement.
11032d31c79SAmireddy Mallikarjuna reddy * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to
11132d31c79SAmireddy Mallikarjuna reddy * be configured. It will enable check sum for switch
11232d31c79SAmireddy Mallikarjuna reddy * If header mode is not set in DMA descriptor,
11332d31c79SAmireddy Mallikarjuna reddy * This register setting doesn't matter
11432d31c79SAmireddy Mallikarjuna reddy */
11532d31c79SAmireddy Mallikarjuna reddy #define DMA_C_HDRM_HDR_SUM BIT(30)
11632d31c79SAmireddy Mallikarjuna reddy
11732d31c79SAmireddy Mallikarjuna reddy #define DMA_C_BOFF 0x120
11832d31c79SAmireddy Mallikarjuna reddy #define DMA_C_BOFF_BOF_LEN GENMASK(7, 0)
11932d31c79SAmireddy Mallikarjuna reddy #define DMA_C_BOFF_EN BIT(31)
12032d31c79SAmireddy Mallikarjuna reddy
12132d31c79SAmireddy Mallikarjuna reddy #define DMA_ORRC 0x190
12232d31c79SAmireddy Mallikarjuna reddy #define DMA_ORRC_ORRCNT GENMASK(8, 4)
12332d31c79SAmireddy Mallikarjuna reddy #define DMA_ORRC_EN BIT(31)
12432d31c79SAmireddy Mallikarjuna reddy
12532d31c79SAmireddy Mallikarjuna reddy #define DMA_C_ENDIAN 0x200
12632d31c79SAmireddy Mallikarjuna reddy #define DMA_C_END_DATAENDI GENMASK(1, 0)
12732d31c79SAmireddy Mallikarjuna reddy #define DMA_C_END_DE_EN BIT(7)
12832d31c79SAmireddy Mallikarjuna reddy #define DMA_C_END_DESENDI GENMASK(9, 8)
12932d31c79SAmireddy Mallikarjuna reddy #define DMA_C_END_DES_EN BIT(16)
13032d31c79SAmireddy Mallikarjuna reddy
13132d31c79SAmireddy Mallikarjuna reddy /* DMA controller capability */
13232d31c79SAmireddy Mallikarjuna reddy #define DMA_ADDR_36BIT BIT(0)
13332d31c79SAmireddy Mallikarjuna reddy #define DMA_DATA_128BIT BIT(1)
13432d31c79SAmireddy Mallikarjuna reddy #define DMA_CHAN_FLOW_CTL BIT(2)
13532d31c79SAmireddy Mallikarjuna reddy #define DMA_DESC_FOD BIT(3)
13632d31c79SAmireddy Mallikarjuna reddy #define DMA_DESC_IN_SRAM BIT(4)
13732d31c79SAmireddy Mallikarjuna reddy #define DMA_EN_BYTE_EN BIT(5)
13832d31c79SAmireddy Mallikarjuna reddy #define DMA_DBURST_WR BIT(6)
13932d31c79SAmireddy Mallikarjuna reddy #define DMA_VALID_DESC_FETCH_ACK BIT(7)
14032d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_DRB BIT(8)
14132d31c79SAmireddy Mallikarjuna reddy
14232d31c79SAmireddy Mallikarjuna reddy #define DMA_ORRC_MAX_CNT (SZ_32 - 1)
14332d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_POLL_CNT SZ_4
14432d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_BURST_V22 SZ_2
14532d31c79SAmireddy Mallikarjuna reddy #define DMA_BURSTL_8DW SZ_8
14632d31c79SAmireddy Mallikarjuna reddy #define DMA_BURSTL_16DW SZ_16
14732d31c79SAmireddy Mallikarjuna reddy #define DMA_BURSTL_32DW SZ_32
14832d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_BURST DMA_BURSTL_16DW
14932d31c79SAmireddy Mallikarjuna reddy #define DMA_MAX_DESC_NUM (SZ_8K - 1)
15032d31c79SAmireddy Mallikarjuna reddy #define DMA_CHAN_BOFF_MAX (SZ_256 - 1)
15132d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_ENDIAN 0
15232d31c79SAmireddy Mallikarjuna reddy
15332d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_DESC_TCNT 50
15432d31c79SAmireddy Mallikarjuna reddy #define DMA_HDR_LEN_MAX (SZ_16K - 1)
15532d31c79SAmireddy Mallikarjuna reddy
15632d31c79SAmireddy Mallikarjuna reddy /* DMA flags */
15732d31c79SAmireddy Mallikarjuna reddy #define DMA_TX_CH BIT(0)
15832d31c79SAmireddy Mallikarjuna reddy #define DMA_RX_CH BIT(1)
15932d31c79SAmireddy Mallikarjuna reddy #define DEVICE_ALLOC_DESC BIT(2)
16032d31c79SAmireddy Mallikarjuna reddy #define CHAN_IN_USE BIT(3)
16132d31c79SAmireddy Mallikarjuna reddy #define DMA_HW_DESC BIT(4)
16232d31c79SAmireddy Mallikarjuna reddy
16332d31c79SAmireddy Mallikarjuna reddy /* Descriptor fields */
16432d31c79SAmireddy Mallikarjuna reddy #define DESC_DATA_LEN GENMASK(15, 0)
16532d31c79SAmireddy Mallikarjuna reddy #define DESC_BYTE_OFF GENMASK(25, 23)
16632d31c79SAmireddy Mallikarjuna reddy #define DESC_EOP BIT(28)
16732d31c79SAmireddy Mallikarjuna reddy #define DESC_SOP BIT(29)
16832d31c79SAmireddy Mallikarjuna reddy #define DESC_C BIT(30)
16932d31c79SAmireddy Mallikarjuna reddy #define DESC_OWN BIT(31)
17032d31c79SAmireddy Mallikarjuna reddy
17132d31c79SAmireddy Mallikarjuna reddy #define DMA_CHAN_RST 1
17232d31c79SAmireddy Mallikarjuna reddy #define DMA_MAX_SIZE (BIT(16) - 1)
17332d31c79SAmireddy Mallikarjuna reddy #define MAX_LOWER_CHANS 32
17432d31c79SAmireddy Mallikarjuna reddy #define MASK_LOWER_CHANS GENMASK(4, 0)
17532d31c79SAmireddy Mallikarjuna reddy #define DMA_OWN 1
17632d31c79SAmireddy Mallikarjuna reddy #define HIGH_4_BITS GENMASK(3, 0)
17732d31c79SAmireddy Mallikarjuna reddy #define DMA_DFT_DESC_NUM 1
17832d31c79SAmireddy Mallikarjuna reddy #define DMA_PKT_DROP_DIS 0
17932d31c79SAmireddy Mallikarjuna reddy
18032d31c79SAmireddy Mallikarjuna reddy enum ldma_chan_on_off {
18132d31c79SAmireddy Mallikarjuna reddy DMA_CH_OFF = 0,
18232d31c79SAmireddy Mallikarjuna reddy DMA_CH_ON = 1,
18332d31c79SAmireddy Mallikarjuna reddy };
18432d31c79SAmireddy Mallikarjuna reddy
18532d31c79SAmireddy Mallikarjuna reddy enum {
18632d31c79SAmireddy Mallikarjuna reddy DMA_TYPE_TX = 0,
18732d31c79SAmireddy Mallikarjuna reddy DMA_TYPE_RX,
18832d31c79SAmireddy Mallikarjuna reddy DMA_TYPE_MCPY,
18932d31c79SAmireddy Mallikarjuna reddy };
19032d31c79SAmireddy Mallikarjuna reddy
19132d31c79SAmireddy Mallikarjuna reddy struct ldma_dev;
19232d31c79SAmireddy Mallikarjuna reddy struct ldma_port;
19332d31c79SAmireddy Mallikarjuna reddy
19432d31c79SAmireddy Mallikarjuna reddy struct ldma_chan {
19532d31c79SAmireddy Mallikarjuna reddy struct virt_dma_chan vchan;
19632d31c79SAmireddy Mallikarjuna reddy struct ldma_port *port; /* back pointer */
19732d31c79SAmireddy Mallikarjuna reddy char name[8]; /* Channel name */
19832d31c79SAmireddy Mallikarjuna reddy int nr; /* Channel id in hardware */
19932d31c79SAmireddy Mallikarjuna reddy u32 flags; /* central way or channel based way */
20032d31c79SAmireddy Mallikarjuna reddy enum ldma_chan_on_off onoff;
20132d31c79SAmireddy Mallikarjuna reddy dma_addr_t desc_phys;
20232d31c79SAmireddy Mallikarjuna reddy void *desc_base; /* Virtual address */
20332d31c79SAmireddy Mallikarjuna reddy u32 desc_cnt; /* Number of descriptors */
20432d31c79SAmireddy Mallikarjuna reddy int rst;
20532d31c79SAmireddy Mallikarjuna reddy u32 hdrm_len;
20632d31c79SAmireddy Mallikarjuna reddy bool hdrm_csum;
20732d31c79SAmireddy Mallikarjuna reddy u32 boff_len;
20832d31c79SAmireddy Mallikarjuna reddy u32 data_endian;
20932d31c79SAmireddy Mallikarjuna reddy u32 desc_endian;
21032d31c79SAmireddy Mallikarjuna reddy bool pden;
21132d31c79SAmireddy Mallikarjuna reddy bool desc_rx_np;
21232d31c79SAmireddy Mallikarjuna reddy bool data_endian_en;
21332d31c79SAmireddy Mallikarjuna reddy bool desc_endian_en;
21432d31c79SAmireddy Mallikarjuna reddy bool abc_en;
21532d31c79SAmireddy Mallikarjuna reddy bool desc_init;
21632d31c79SAmireddy Mallikarjuna reddy struct dma_pool *desc_pool; /* Descriptors pool */
21732d31c79SAmireddy Mallikarjuna reddy u32 desc_num;
21832d31c79SAmireddy Mallikarjuna reddy struct dw2_desc_sw *ds;
21932d31c79SAmireddy Mallikarjuna reddy struct work_struct work;
22032d31c79SAmireddy Mallikarjuna reddy struct dma_slave_config config;
22132d31c79SAmireddy Mallikarjuna reddy };
22232d31c79SAmireddy Mallikarjuna reddy
22332d31c79SAmireddy Mallikarjuna reddy struct ldma_port {
22432d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *ldev; /* back pointer */
22532d31c79SAmireddy Mallikarjuna reddy u32 portid;
22632d31c79SAmireddy Mallikarjuna reddy u32 rxbl;
22732d31c79SAmireddy Mallikarjuna reddy u32 txbl;
22832d31c79SAmireddy Mallikarjuna reddy u32 rxendi;
22932d31c79SAmireddy Mallikarjuna reddy u32 txendi;
23032d31c79SAmireddy Mallikarjuna reddy u32 pkt_drop;
23132d31c79SAmireddy Mallikarjuna reddy };
23232d31c79SAmireddy Mallikarjuna reddy
23332d31c79SAmireddy Mallikarjuna reddy /* Instance specific data */
23432d31c79SAmireddy Mallikarjuna reddy struct ldma_inst_data {
23532d31c79SAmireddy Mallikarjuna reddy bool desc_in_sram;
23632d31c79SAmireddy Mallikarjuna reddy bool chan_fc;
23732d31c79SAmireddy Mallikarjuna reddy bool desc_fod; /* Fetch On Demand */
23832d31c79SAmireddy Mallikarjuna reddy bool valid_desc_fetch_ack;
23932d31c79SAmireddy Mallikarjuna reddy u32 orrc; /* Outstanding read count */
24032d31c79SAmireddy Mallikarjuna reddy const char *name;
24132d31c79SAmireddy Mallikarjuna reddy u32 type;
24232d31c79SAmireddy Mallikarjuna reddy };
24332d31c79SAmireddy Mallikarjuna reddy
24432d31c79SAmireddy Mallikarjuna reddy struct ldma_dev {
24532d31c79SAmireddy Mallikarjuna reddy struct device *dev;
24632d31c79SAmireddy Mallikarjuna reddy void __iomem *base;
24732d31c79SAmireddy Mallikarjuna reddy struct reset_control *rst;
24832d31c79SAmireddy Mallikarjuna reddy struct clk *core_clk;
24932d31c79SAmireddy Mallikarjuna reddy struct dma_device dma_dev;
25032d31c79SAmireddy Mallikarjuna reddy u32 ver;
25132d31c79SAmireddy Mallikarjuna reddy int irq;
25232d31c79SAmireddy Mallikarjuna reddy struct ldma_port *ports;
25332d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *chans; /* channel list on this DMA or port */
25432d31c79SAmireddy Mallikarjuna reddy spinlock_t dev_lock; /* Controller register exclusive */
25532d31c79SAmireddy Mallikarjuna reddy u32 chan_nrs;
25632d31c79SAmireddy Mallikarjuna reddy u32 port_nrs;
25732d31c79SAmireddy Mallikarjuna reddy u32 channels_mask;
25832d31c79SAmireddy Mallikarjuna reddy u32 flags;
25932d31c79SAmireddy Mallikarjuna reddy u32 pollcnt;
26032d31c79SAmireddy Mallikarjuna reddy const struct ldma_inst_data *inst;
26132d31c79SAmireddy Mallikarjuna reddy struct workqueue_struct *wq;
26232d31c79SAmireddy Mallikarjuna reddy };
26332d31c79SAmireddy Mallikarjuna reddy
26432d31c79SAmireddy Mallikarjuna reddy struct dw2_desc {
26532d31c79SAmireddy Mallikarjuna reddy u32 field;
26632d31c79SAmireddy Mallikarjuna reddy u32 addr;
26732d31c79SAmireddy Mallikarjuna reddy } __packed __aligned(8);
26832d31c79SAmireddy Mallikarjuna reddy
26932d31c79SAmireddy Mallikarjuna reddy struct dw2_desc_sw {
27032d31c79SAmireddy Mallikarjuna reddy struct virt_dma_desc vdesc;
27132d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *chan;
27232d31c79SAmireddy Mallikarjuna reddy dma_addr_t desc_phys;
27332d31c79SAmireddy Mallikarjuna reddy size_t desc_cnt;
27432d31c79SAmireddy Mallikarjuna reddy size_t size;
27532d31c79SAmireddy Mallikarjuna reddy struct dw2_desc *desc_hw;
27632d31c79SAmireddy Mallikarjuna reddy };
27732d31c79SAmireddy Mallikarjuna reddy
27832d31c79SAmireddy Mallikarjuna reddy static inline void
ldma_update_bits(struct ldma_dev * d,u32 mask,u32 val,u32 ofs)27932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(struct ldma_dev *d, u32 mask, u32 val, u32 ofs)
28032d31c79SAmireddy Mallikarjuna reddy {
28132d31c79SAmireddy Mallikarjuna reddy u32 old_val, new_val;
28232d31c79SAmireddy Mallikarjuna reddy
28332d31c79SAmireddy Mallikarjuna reddy old_val = readl(d->base + ofs);
28432d31c79SAmireddy Mallikarjuna reddy new_val = (old_val & ~mask) | (val & mask);
28532d31c79SAmireddy Mallikarjuna reddy
28632d31c79SAmireddy Mallikarjuna reddy if (new_val != old_val)
28732d31c79SAmireddy Mallikarjuna reddy writel(new_val, d->base + ofs);
28832d31c79SAmireddy Mallikarjuna reddy }
28932d31c79SAmireddy Mallikarjuna reddy
to_ldma_chan(struct dma_chan * chan)29032d31c79SAmireddy Mallikarjuna reddy static inline struct ldma_chan *to_ldma_chan(struct dma_chan *chan)
29132d31c79SAmireddy Mallikarjuna reddy {
29232d31c79SAmireddy Mallikarjuna reddy return container_of(chan, struct ldma_chan, vchan.chan);
29332d31c79SAmireddy Mallikarjuna reddy }
29432d31c79SAmireddy Mallikarjuna reddy
to_ldma_dev(struct dma_device * dma_dev)29532d31c79SAmireddy Mallikarjuna reddy static inline struct ldma_dev *to_ldma_dev(struct dma_device *dma_dev)
29632d31c79SAmireddy Mallikarjuna reddy {
29732d31c79SAmireddy Mallikarjuna reddy return container_of(dma_dev, struct ldma_dev, dma_dev);
29832d31c79SAmireddy Mallikarjuna reddy }
29932d31c79SAmireddy Mallikarjuna reddy
to_lgm_dma_desc(struct virt_dma_desc * vdesc)30032d31c79SAmireddy Mallikarjuna reddy static inline struct dw2_desc_sw *to_lgm_dma_desc(struct virt_dma_desc *vdesc)
30132d31c79SAmireddy Mallikarjuna reddy {
30232d31c79SAmireddy Mallikarjuna reddy return container_of(vdesc, struct dw2_desc_sw, vdesc);
30332d31c79SAmireddy Mallikarjuna reddy }
30432d31c79SAmireddy Mallikarjuna reddy
ldma_chan_tx(struct ldma_chan * c)30532d31c79SAmireddy Mallikarjuna reddy static inline bool ldma_chan_tx(struct ldma_chan *c)
30632d31c79SAmireddy Mallikarjuna reddy {
30732d31c79SAmireddy Mallikarjuna reddy return !!(c->flags & DMA_TX_CH);
30832d31c79SAmireddy Mallikarjuna reddy }
30932d31c79SAmireddy Mallikarjuna reddy
ldma_chan_is_hw_desc(struct ldma_chan * c)31032d31c79SAmireddy Mallikarjuna reddy static inline bool ldma_chan_is_hw_desc(struct ldma_chan *c)
31132d31c79SAmireddy Mallikarjuna reddy {
31232d31c79SAmireddy Mallikarjuna reddy return !!(c->flags & DMA_HW_DESC);
31332d31c79SAmireddy Mallikarjuna reddy }
31432d31c79SAmireddy Mallikarjuna reddy
ldma_dev_reset(struct ldma_dev * d)31532d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_reset(struct ldma_dev *d)
31632d31c79SAmireddy Mallikarjuna reddy
31732d31c79SAmireddy Mallikarjuna reddy {
31832d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
31932d31c79SAmireddy Mallikarjuna reddy
32032d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
32132d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CTRL_RST, DMA_CTRL_RST, DMA_CTRL);
32232d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
32332d31c79SAmireddy Mallikarjuna reddy }
32432d31c79SAmireddy Mallikarjuna reddy
ldma_dev_pkt_arb_cfg(struct ldma_dev * d,bool enable)32532d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_pkt_arb_cfg(struct ldma_dev *d, bool enable)
32632d31c79SAmireddy Mallikarjuna reddy {
32732d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
32832d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_CTRL_PKTARB;
32932d31c79SAmireddy Mallikarjuna reddy u32 val = enable ? DMA_CTRL_PKTARB : 0;
33032d31c79SAmireddy Mallikarjuna reddy
33132d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
33232d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
33332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
33432d31c79SAmireddy Mallikarjuna reddy }
33532d31c79SAmireddy Mallikarjuna reddy
ldma_dev_sram_desc_cfg(struct ldma_dev * d,bool enable)33632d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_sram_desc_cfg(struct ldma_dev *d, bool enable)
33732d31c79SAmireddy Mallikarjuna reddy {
33832d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
33932d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_CTRL_DSRAM_PATH;
34032d31c79SAmireddy Mallikarjuna reddy u32 val = enable ? DMA_CTRL_DSRAM_PATH : 0;
34132d31c79SAmireddy Mallikarjuna reddy
34232d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
34332d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
34432d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
34532d31c79SAmireddy Mallikarjuna reddy }
34632d31c79SAmireddy Mallikarjuna reddy
ldma_dev_chan_flow_ctl_cfg(struct ldma_dev * d,bool enable)34732d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_chan_flow_ctl_cfg(struct ldma_dev *d, bool enable)
34832d31c79SAmireddy Mallikarjuna reddy {
34932d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
35032d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
35132d31c79SAmireddy Mallikarjuna reddy
35232d31c79SAmireddy Mallikarjuna reddy if (d->inst->type != DMA_TYPE_TX)
35332d31c79SAmireddy Mallikarjuna reddy return;
35432d31c79SAmireddy Mallikarjuna reddy
35532d31c79SAmireddy Mallikarjuna reddy mask = DMA_CTRL_CH_FL;
35632d31c79SAmireddy Mallikarjuna reddy val = enable ? DMA_CTRL_CH_FL : 0;
35732d31c79SAmireddy Mallikarjuna reddy
35832d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
35932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
36032d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
36132d31c79SAmireddy Mallikarjuna reddy }
36232d31c79SAmireddy Mallikarjuna reddy
ldma_dev_global_polling_enable(struct ldma_dev * d)36332d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_global_polling_enable(struct ldma_dev *d)
36432d31c79SAmireddy Mallikarjuna reddy {
36532d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
36632d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_CPOLL_EN | DMA_CPOLL_CNT;
36732d31c79SAmireddy Mallikarjuna reddy u32 val = DMA_CPOLL_EN;
36832d31c79SAmireddy Mallikarjuna reddy
36932d31c79SAmireddy Mallikarjuna reddy val |= FIELD_PREP(DMA_CPOLL_CNT, d->pollcnt);
37032d31c79SAmireddy Mallikarjuna reddy
37132d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
37232d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CPOLL);
37332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
37432d31c79SAmireddy Mallikarjuna reddy }
37532d31c79SAmireddy Mallikarjuna reddy
ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev * d,bool enable)37632d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev *d, bool enable)
37732d31c79SAmireddy Mallikarjuna reddy {
37832d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
37932d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
38032d31c79SAmireddy Mallikarjuna reddy
38132d31c79SAmireddy Mallikarjuna reddy if (d->inst->type == DMA_TYPE_MCPY)
38232d31c79SAmireddy Mallikarjuna reddy return;
38332d31c79SAmireddy Mallikarjuna reddy
38432d31c79SAmireddy Mallikarjuna reddy mask = DMA_CTRL_DS_FOD;
38532d31c79SAmireddy Mallikarjuna reddy val = enable ? DMA_CTRL_DS_FOD : 0;
38632d31c79SAmireddy Mallikarjuna reddy
38732d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
38832d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
38932d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
39032d31c79SAmireddy Mallikarjuna reddy }
39132d31c79SAmireddy Mallikarjuna reddy
ldma_dev_byte_enable_cfg(struct ldma_dev * d,bool enable)39232d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_byte_enable_cfg(struct ldma_dev *d, bool enable)
39332d31c79SAmireddy Mallikarjuna reddy {
39432d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
39532d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_CTRL_ENBE;
39632d31c79SAmireddy Mallikarjuna reddy u32 val = enable ? DMA_CTRL_ENBE : 0;
39732d31c79SAmireddy Mallikarjuna reddy
39832d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
39932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
40032d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
40132d31c79SAmireddy Mallikarjuna reddy }
40232d31c79SAmireddy Mallikarjuna reddy
ldma_dev_orrc_cfg(struct ldma_dev * d)40332d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_orrc_cfg(struct ldma_dev *d)
40432d31c79SAmireddy Mallikarjuna reddy {
40532d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
40632d31c79SAmireddy Mallikarjuna reddy u32 val = 0;
40732d31c79SAmireddy Mallikarjuna reddy u32 mask;
40832d31c79SAmireddy Mallikarjuna reddy
40932d31c79SAmireddy Mallikarjuna reddy if (d->inst->type == DMA_TYPE_RX)
41032d31c79SAmireddy Mallikarjuna reddy return;
41132d31c79SAmireddy Mallikarjuna reddy
41232d31c79SAmireddy Mallikarjuna reddy mask = DMA_ORRC_EN | DMA_ORRC_ORRCNT;
41332d31c79SAmireddy Mallikarjuna reddy if (d->inst->orrc > 0 && d->inst->orrc <= DMA_ORRC_MAX_CNT)
41432d31c79SAmireddy Mallikarjuna reddy val = DMA_ORRC_EN | FIELD_PREP(DMA_ORRC_ORRCNT, d->inst->orrc);
41532d31c79SAmireddy Mallikarjuna reddy
41632d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
41732d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_ORRC);
41832d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
41932d31c79SAmireddy Mallikarjuna reddy }
42032d31c79SAmireddy Mallikarjuna reddy
ldma_dev_df_tout_cfg(struct ldma_dev * d,bool enable,int tcnt)42132d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_df_tout_cfg(struct ldma_dev *d, bool enable, int tcnt)
42232d31c79SAmireddy Mallikarjuna reddy {
42332d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_CTRL_DESC_TMOUT_CNT_V31;
42432d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
42532d31c79SAmireddy Mallikarjuna reddy u32 val;
42632d31c79SAmireddy Mallikarjuna reddy
42732d31c79SAmireddy Mallikarjuna reddy if (enable)
42832d31c79SAmireddy Mallikarjuna reddy val = DMA_CTRL_DESC_TMOUT_EN_V31 | FIELD_PREP(DMA_CTRL_DESC_TMOUT_CNT_V31, tcnt);
42932d31c79SAmireddy Mallikarjuna reddy else
43032d31c79SAmireddy Mallikarjuna reddy val = 0;
43132d31c79SAmireddy Mallikarjuna reddy
43232d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
43332d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
43432d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
43532d31c79SAmireddy Mallikarjuna reddy }
43632d31c79SAmireddy Mallikarjuna reddy
ldma_dev_dburst_wr_cfg(struct ldma_dev * d,bool enable)43732d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_dburst_wr_cfg(struct ldma_dev *d, bool enable)
43832d31c79SAmireddy Mallikarjuna reddy {
43932d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
44032d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
44132d31c79SAmireddy Mallikarjuna reddy
44232d31c79SAmireddy Mallikarjuna reddy if (d->inst->type != DMA_TYPE_RX && d->inst->type != DMA_TYPE_MCPY)
44332d31c79SAmireddy Mallikarjuna reddy return;
44432d31c79SAmireddy Mallikarjuna reddy
44532d31c79SAmireddy Mallikarjuna reddy mask = DMA_CTRL_DBURST_WR;
44632d31c79SAmireddy Mallikarjuna reddy val = enable ? DMA_CTRL_DBURST_WR : 0;
44732d31c79SAmireddy Mallikarjuna reddy
44832d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
44932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
45032d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
45132d31c79SAmireddy Mallikarjuna reddy }
45232d31c79SAmireddy Mallikarjuna reddy
ldma_dev_vld_fetch_ack_cfg(struct ldma_dev * d,bool enable)45332d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_vld_fetch_ack_cfg(struct ldma_dev *d, bool enable)
45432d31c79SAmireddy Mallikarjuna reddy {
45532d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
45632d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
45732d31c79SAmireddy Mallikarjuna reddy
45832d31c79SAmireddy Mallikarjuna reddy if (d->inst->type != DMA_TYPE_TX)
45932d31c79SAmireddy Mallikarjuna reddy return;
46032d31c79SAmireddy Mallikarjuna reddy
46132d31c79SAmireddy Mallikarjuna reddy mask = DMA_CTRL_VLD_DF_ACK;
46232d31c79SAmireddy Mallikarjuna reddy val = enable ? DMA_CTRL_VLD_DF_ACK : 0;
46332d31c79SAmireddy Mallikarjuna reddy
46432d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
46532d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
46632d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
46732d31c79SAmireddy Mallikarjuna reddy }
46832d31c79SAmireddy Mallikarjuna reddy
ldma_dev_drb_cfg(struct ldma_dev * d,int enable)46932d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_drb_cfg(struct ldma_dev *d, int enable)
47032d31c79SAmireddy Mallikarjuna reddy {
47132d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
47232d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_CTRL_DRB;
47332d31c79SAmireddy Mallikarjuna reddy u32 val = enable ? DMA_CTRL_DRB : 0;
47432d31c79SAmireddy Mallikarjuna reddy
47532d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
47632d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CTRL);
47732d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
47832d31c79SAmireddy Mallikarjuna reddy }
47932d31c79SAmireddy Mallikarjuna reddy
ldma_dev_cfg(struct ldma_dev * d)48032d31c79SAmireddy Mallikarjuna reddy static int ldma_dev_cfg(struct ldma_dev *d)
48132d31c79SAmireddy Mallikarjuna reddy {
48232d31c79SAmireddy Mallikarjuna reddy bool enable;
48332d31c79SAmireddy Mallikarjuna reddy
48432d31c79SAmireddy Mallikarjuna reddy ldma_dev_pkt_arb_cfg(d, true);
48532d31c79SAmireddy Mallikarjuna reddy ldma_dev_global_polling_enable(d);
48632d31c79SAmireddy Mallikarjuna reddy
48732d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_DFT_DRB);
48832d31c79SAmireddy Mallikarjuna reddy ldma_dev_drb_cfg(d, enable);
48932d31c79SAmireddy Mallikarjuna reddy
49032d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_EN_BYTE_EN);
49132d31c79SAmireddy Mallikarjuna reddy ldma_dev_byte_enable_cfg(d, enable);
49232d31c79SAmireddy Mallikarjuna reddy
49332d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_CHAN_FLOW_CTL);
49432d31c79SAmireddy Mallikarjuna reddy ldma_dev_chan_flow_ctl_cfg(d, enable);
49532d31c79SAmireddy Mallikarjuna reddy
49632d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_DESC_FOD);
49732d31c79SAmireddy Mallikarjuna reddy ldma_dev_desc_fetch_on_demand_cfg(d, enable);
49832d31c79SAmireddy Mallikarjuna reddy
49932d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_DESC_IN_SRAM);
50032d31c79SAmireddy Mallikarjuna reddy ldma_dev_sram_desc_cfg(d, enable);
50132d31c79SAmireddy Mallikarjuna reddy
50232d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_DBURST_WR);
50332d31c79SAmireddy Mallikarjuna reddy ldma_dev_dburst_wr_cfg(d, enable);
50432d31c79SAmireddy Mallikarjuna reddy
50532d31c79SAmireddy Mallikarjuna reddy enable = !!(d->flags & DMA_VALID_DESC_FETCH_ACK);
50632d31c79SAmireddy Mallikarjuna reddy ldma_dev_vld_fetch_ack_cfg(d, enable);
50732d31c79SAmireddy Mallikarjuna reddy
50832d31c79SAmireddy Mallikarjuna reddy if (d->ver > DMA_VER22) {
50932d31c79SAmireddy Mallikarjuna reddy ldma_dev_orrc_cfg(d);
51032d31c79SAmireddy Mallikarjuna reddy ldma_dev_df_tout_cfg(d, true, DMA_DFT_DESC_TCNT);
51132d31c79SAmireddy Mallikarjuna reddy }
51232d31c79SAmireddy Mallikarjuna reddy
51332d31c79SAmireddy Mallikarjuna reddy dev_dbg(d->dev, "%s Controller 0x%08x configuration done\n",
51432d31c79SAmireddy Mallikarjuna reddy d->inst->name, readl(d->base + DMA_CTRL));
51532d31c79SAmireddy Mallikarjuna reddy
51632d31c79SAmireddy Mallikarjuna reddy return 0;
51732d31c79SAmireddy Mallikarjuna reddy }
51832d31c79SAmireddy Mallikarjuna reddy
ldma_chan_cctrl_cfg(struct ldma_chan * c,u32 val)51932d31c79SAmireddy Mallikarjuna reddy static int ldma_chan_cctrl_cfg(struct ldma_chan *c, u32 val)
52032d31c79SAmireddy Mallikarjuna reddy {
52132d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
52232d31c79SAmireddy Mallikarjuna reddy u32 class_low, class_high;
52332d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
52432d31c79SAmireddy Mallikarjuna reddy u32 reg;
52532d31c79SAmireddy Mallikarjuna reddy
52632d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
52732d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
52832d31c79SAmireddy Mallikarjuna reddy reg = readl(d->base + DMA_CCTRL);
52932d31c79SAmireddy Mallikarjuna reddy /* Read from hardware */
53032d31c79SAmireddy Mallikarjuna reddy if (reg & DMA_CCTRL_DIR_TX)
53132d31c79SAmireddy Mallikarjuna reddy c->flags |= DMA_TX_CH;
53232d31c79SAmireddy Mallikarjuna reddy else
53332d31c79SAmireddy Mallikarjuna reddy c->flags |= DMA_RX_CH;
53432d31c79SAmireddy Mallikarjuna reddy
53532d31c79SAmireddy Mallikarjuna reddy /* Keep the class value unchanged */
53632d31c79SAmireddy Mallikarjuna reddy class_low = FIELD_GET(DMA_CCTRL_CLASS, reg);
53732d31c79SAmireddy Mallikarjuna reddy class_high = FIELD_GET(DMA_CCTRL_CLASSH, reg);
53832d31c79SAmireddy Mallikarjuna reddy val &= ~DMA_CCTRL_CLASS;
53932d31c79SAmireddy Mallikarjuna reddy val |= FIELD_PREP(DMA_CCTRL_CLASS, class_low);
54032d31c79SAmireddy Mallikarjuna reddy val &= ~DMA_CCTRL_CLASSH;
54132d31c79SAmireddy Mallikarjuna reddy val |= FIELD_PREP(DMA_CCTRL_CLASSH, class_high);
54232d31c79SAmireddy Mallikarjuna reddy writel(val, d->base + DMA_CCTRL);
54332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
54432d31c79SAmireddy Mallikarjuna reddy
54532d31c79SAmireddy Mallikarjuna reddy return 0;
54632d31c79SAmireddy Mallikarjuna reddy }
54732d31c79SAmireddy Mallikarjuna reddy
ldma_chan_irq_init(struct ldma_chan * c)54832d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_irq_init(struct ldma_chan *c)
54932d31c79SAmireddy Mallikarjuna reddy {
55032d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
55132d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
55232d31c79SAmireddy Mallikarjuna reddy u32 enofs, crofs;
55332d31c79SAmireddy Mallikarjuna reddy u32 cn_bit;
55432d31c79SAmireddy Mallikarjuna reddy
55532d31c79SAmireddy Mallikarjuna reddy if (c->nr < MAX_LOWER_CHANS) {
55632d31c79SAmireddy Mallikarjuna reddy enofs = DMA_IRNEN;
55732d31c79SAmireddy Mallikarjuna reddy crofs = DMA_IRNCR;
55832d31c79SAmireddy Mallikarjuna reddy } else {
55932d31c79SAmireddy Mallikarjuna reddy enofs = DMA_IRNEN1;
56032d31c79SAmireddy Mallikarjuna reddy crofs = DMA_IRNCR1;
56132d31c79SAmireddy Mallikarjuna reddy }
56232d31c79SAmireddy Mallikarjuna reddy
56332d31c79SAmireddy Mallikarjuna reddy cn_bit = BIT(c->nr & MASK_LOWER_CHANS);
56432d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
56532d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
56632d31c79SAmireddy Mallikarjuna reddy
56732d31c79SAmireddy Mallikarjuna reddy /* Clear all interrupts and disabled it */
56832d31c79SAmireddy Mallikarjuna reddy writel(0, d->base + DMA_CIE);
56932d31c79SAmireddy Mallikarjuna reddy writel(DMA_CI_ALL, d->base + DMA_CIS);
57032d31c79SAmireddy Mallikarjuna reddy
57132d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, cn_bit, 0, enofs);
57232d31c79SAmireddy Mallikarjuna reddy writel(cn_bit, d->base + crofs);
57332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
57432d31c79SAmireddy Mallikarjuna reddy }
57532d31c79SAmireddy Mallikarjuna reddy
ldma_chan_set_class(struct ldma_chan * c,u32 val)57632d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_set_class(struct ldma_chan *c, u32 val)
57732d31c79SAmireddy Mallikarjuna reddy {
57832d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
57932d31c79SAmireddy Mallikarjuna reddy u32 class_val;
58032d31c79SAmireddy Mallikarjuna reddy
58132d31c79SAmireddy Mallikarjuna reddy if (d->inst->type == DMA_TYPE_MCPY || val > DMA_MAX_CLASS)
58232d31c79SAmireddy Mallikarjuna reddy return;
58332d31c79SAmireddy Mallikarjuna reddy
58432d31c79SAmireddy Mallikarjuna reddy /* 3 bits low */
58532d31c79SAmireddy Mallikarjuna reddy class_val = FIELD_PREP(DMA_CCTRL_CLASS, val & 0x7);
58632d31c79SAmireddy Mallikarjuna reddy /* 2 bits high */
58732d31c79SAmireddy Mallikarjuna reddy class_val |= FIELD_PREP(DMA_CCTRL_CLASSH, (val >> 3) & 0x3);
58832d31c79SAmireddy Mallikarjuna reddy
58932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
59032d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CCTRL_CLASS | DMA_CCTRL_CLASSH, class_val,
59132d31c79SAmireddy Mallikarjuna reddy DMA_CCTRL);
59232d31c79SAmireddy Mallikarjuna reddy }
59332d31c79SAmireddy Mallikarjuna reddy
ldma_chan_on(struct ldma_chan * c)59432d31c79SAmireddy Mallikarjuna reddy static int ldma_chan_on(struct ldma_chan *c)
59532d31c79SAmireddy Mallikarjuna reddy {
59632d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
59732d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
59832d31c79SAmireddy Mallikarjuna reddy
59932d31c79SAmireddy Mallikarjuna reddy /* If descriptors not configured, not allow to turn on channel */
60032d31c79SAmireddy Mallikarjuna reddy if (WARN_ON(!c->desc_init))
60132d31c79SAmireddy Mallikarjuna reddy return -EINVAL;
60232d31c79SAmireddy Mallikarjuna reddy
60332d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
60432d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
60532d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CCTRL_ON, DMA_CCTRL_ON, DMA_CCTRL);
60632d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
60732d31c79SAmireddy Mallikarjuna reddy
60832d31c79SAmireddy Mallikarjuna reddy c->onoff = DMA_CH_ON;
60932d31c79SAmireddy Mallikarjuna reddy
61032d31c79SAmireddy Mallikarjuna reddy return 0;
61132d31c79SAmireddy Mallikarjuna reddy }
61232d31c79SAmireddy Mallikarjuna reddy
ldma_chan_off(struct ldma_chan * c)61332d31c79SAmireddy Mallikarjuna reddy static int ldma_chan_off(struct ldma_chan *c)
61432d31c79SAmireddy Mallikarjuna reddy {
61532d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
61632d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
61732d31c79SAmireddy Mallikarjuna reddy u32 val;
61832d31c79SAmireddy Mallikarjuna reddy int ret;
61932d31c79SAmireddy Mallikarjuna reddy
62032d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
62132d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
62232d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CCTRL_ON, 0, DMA_CCTRL);
62332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
62432d31c79SAmireddy Mallikarjuna reddy
62532d31c79SAmireddy Mallikarjuna reddy ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val,
62632d31c79SAmireddy Mallikarjuna reddy !(val & DMA_CCTRL_ON), 0, 10000);
62732d31c79SAmireddy Mallikarjuna reddy if (ret)
62832d31c79SAmireddy Mallikarjuna reddy return ret;
62932d31c79SAmireddy Mallikarjuna reddy
63032d31c79SAmireddy Mallikarjuna reddy c->onoff = DMA_CH_OFF;
63132d31c79SAmireddy Mallikarjuna reddy
63232d31c79SAmireddy Mallikarjuna reddy return 0;
63332d31c79SAmireddy Mallikarjuna reddy }
63432d31c79SAmireddy Mallikarjuna reddy
ldma_chan_desc_hw_cfg(struct ldma_chan * c,dma_addr_t desc_base,int desc_num)63532d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_desc_hw_cfg(struct ldma_chan *c, dma_addr_t desc_base,
63632d31c79SAmireddy Mallikarjuna reddy int desc_num)
63732d31c79SAmireddy Mallikarjuna reddy {
63832d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
63932d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
64032d31c79SAmireddy Mallikarjuna reddy
64132d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
64232d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
64332d31c79SAmireddy Mallikarjuna reddy writel(lower_32_bits(desc_base), d->base + DMA_CDBA);
64432d31c79SAmireddy Mallikarjuna reddy
64532d31c79SAmireddy Mallikarjuna reddy /* Higher 4 bits of 36 bit addressing */
64632d31c79SAmireddy Mallikarjuna reddy if (IS_ENABLED(CONFIG_64BIT)) {
64732d31c79SAmireddy Mallikarjuna reddy u32 hi = upper_32_bits(desc_base) & HIGH_4_BITS;
64832d31c79SAmireddy Mallikarjuna reddy
64932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CDBA_MSB,
65032d31c79SAmireddy Mallikarjuna reddy FIELD_PREP(DMA_CDBA_MSB, hi), DMA_CCTRL);
65132d31c79SAmireddy Mallikarjuna reddy }
65232d31c79SAmireddy Mallikarjuna reddy writel(desc_num, d->base + DMA_CDLEN);
65332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
65432d31c79SAmireddy Mallikarjuna reddy
65532d31c79SAmireddy Mallikarjuna reddy c->desc_init = true;
65632d31c79SAmireddy Mallikarjuna reddy }
65732d31c79SAmireddy Mallikarjuna reddy
65832d31c79SAmireddy Mallikarjuna reddy static struct dma_async_tx_descriptor *
ldma_chan_desc_cfg(struct dma_chan * chan,dma_addr_t desc_base,int desc_num)65932d31c79SAmireddy Mallikarjuna reddy ldma_chan_desc_cfg(struct dma_chan *chan, dma_addr_t desc_base, int desc_num)
66032d31c79SAmireddy Mallikarjuna reddy {
66132d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
66232d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
66332d31c79SAmireddy Mallikarjuna reddy struct dma_async_tx_descriptor *tx;
66432d31c79SAmireddy Mallikarjuna reddy struct dw2_desc_sw *ds;
66532d31c79SAmireddy Mallikarjuna reddy
66632d31c79SAmireddy Mallikarjuna reddy if (!desc_num) {
66732d31c79SAmireddy Mallikarjuna reddy dev_err(d->dev, "Channel %d must allocate descriptor first\n",
66832d31c79SAmireddy Mallikarjuna reddy c->nr);
66932d31c79SAmireddy Mallikarjuna reddy return NULL;
67032d31c79SAmireddy Mallikarjuna reddy }
67132d31c79SAmireddy Mallikarjuna reddy
67232d31c79SAmireddy Mallikarjuna reddy if (desc_num > DMA_MAX_DESC_NUM) {
67332d31c79SAmireddy Mallikarjuna reddy dev_err(d->dev, "Channel %d descriptor number out of range %d\n",
67432d31c79SAmireddy Mallikarjuna reddy c->nr, desc_num);
67532d31c79SAmireddy Mallikarjuna reddy return NULL;
67632d31c79SAmireddy Mallikarjuna reddy }
67732d31c79SAmireddy Mallikarjuna reddy
67832d31c79SAmireddy Mallikarjuna reddy ldma_chan_desc_hw_cfg(c, desc_base, desc_num);
67932d31c79SAmireddy Mallikarjuna reddy
68032d31c79SAmireddy Mallikarjuna reddy c->flags |= DMA_HW_DESC;
68132d31c79SAmireddy Mallikarjuna reddy c->desc_cnt = desc_num;
68232d31c79SAmireddy Mallikarjuna reddy c->desc_phys = desc_base;
68332d31c79SAmireddy Mallikarjuna reddy
68432d31c79SAmireddy Mallikarjuna reddy ds = kzalloc(sizeof(*ds), GFP_NOWAIT);
68532d31c79SAmireddy Mallikarjuna reddy if (!ds)
68632d31c79SAmireddy Mallikarjuna reddy return NULL;
68732d31c79SAmireddy Mallikarjuna reddy
68832d31c79SAmireddy Mallikarjuna reddy tx = &ds->vdesc.tx;
68932d31c79SAmireddy Mallikarjuna reddy dma_async_tx_descriptor_init(tx, chan);
69032d31c79SAmireddy Mallikarjuna reddy
69132d31c79SAmireddy Mallikarjuna reddy return tx;
69232d31c79SAmireddy Mallikarjuna reddy }
69332d31c79SAmireddy Mallikarjuna reddy
ldma_chan_reset(struct ldma_chan * c)69432d31c79SAmireddy Mallikarjuna reddy static int ldma_chan_reset(struct ldma_chan *c)
69532d31c79SAmireddy Mallikarjuna reddy {
69632d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
69732d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
69832d31c79SAmireddy Mallikarjuna reddy u32 val;
69932d31c79SAmireddy Mallikarjuna reddy int ret;
70032d31c79SAmireddy Mallikarjuna reddy
70132d31c79SAmireddy Mallikarjuna reddy ret = ldma_chan_off(c);
70232d31c79SAmireddy Mallikarjuna reddy if (ret)
70332d31c79SAmireddy Mallikarjuna reddy return ret;
70432d31c79SAmireddy Mallikarjuna reddy
70532d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
70632d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
70732d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CCTRL_RST, DMA_CCTRL_RST, DMA_CCTRL);
70832d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
70932d31c79SAmireddy Mallikarjuna reddy
71032d31c79SAmireddy Mallikarjuna reddy ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val,
71132d31c79SAmireddy Mallikarjuna reddy !(val & DMA_CCTRL_RST), 0, 10000);
71232d31c79SAmireddy Mallikarjuna reddy if (ret)
71332d31c79SAmireddy Mallikarjuna reddy return ret;
71432d31c79SAmireddy Mallikarjuna reddy
71532d31c79SAmireddy Mallikarjuna reddy c->rst = 1;
71632d31c79SAmireddy Mallikarjuna reddy c->desc_init = false;
71732d31c79SAmireddy Mallikarjuna reddy
71832d31c79SAmireddy Mallikarjuna reddy return 0;
71932d31c79SAmireddy Mallikarjuna reddy }
72032d31c79SAmireddy Mallikarjuna reddy
ldma_chan_byte_offset_cfg(struct ldma_chan * c,u32 boff_len)72132d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_byte_offset_cfg(struct ldma_chan *c, u32 boff_len)
72232d31c79SAmireddy Mallikarjuna reddy {
72332d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
72432d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_C_BOFF_EN | DMA_C_BOFF_BOF_LEN;
72532d31c79SAmireddy Mallikarjuna reddy u32 val;
72632d31c79SAmireddy Mallikarjuna reddy
72732d31c79SAmireddy Mallikarjuna reddy if (boff_len > 0 && boff_len <= DMA_CHAN_BOFF_MAX)
72832d31c79SAmireddy Mallikarjuna reddy val = FIELD_PREP(DMA_C_BOFF_BOF_LEN, boff_len) | DMA_C_BOFF_EN;
72932d31c79SAmireddy Mallikarjuna reddy else
73032d31c79SAmireddy Mallikarjuna reddy val = 0;
73132d31c79SAmireddy Mallikarjuna reddy
73232d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
73332d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_C_BOFF);
73432d31c79SAmireddy Mallikarjuna reddy }
73532d31c79SAmireddy Mallikarjuna reddy
ldma_chan_data_endian_cfg(struct ldma_chan * c,bool enable,u32 endian_type)73632d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_data_endian_cfg(struct ldma_chan *c, bool enable,
73732d31c79SAmireddy Mallikarjuna reddy u32 endian_type)
73832d31c79SAmireddy Mallikarjuna reddy {
73932d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
74032d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_C_END_DE_EN | DMA_C_END_DATAENDI;
74132d31c79SAmireddy Mallikarjuna reddy u32 val;
74232d31c79SAmireddy Mallikarjuna reddy
74332d31c79SAmireddy Mallikarjuna reddy if (enable)
74432d31c79SAmireddy Mallikarjuna reddy val = DMA_C_END_DE_EN | FIELD_PREP(DMA_C_END_DATAENDI, endian_type);
74532d31c79SAmireddy Mallikarjuna reddy else
74632d31c79SAmireddy Mallikarjuna reddy val = 0;
74732d31c79SAmireddy Mallikarjuna reddy
74832d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
74932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_C_ENDIAN);
75032d31c79SAmireddy Mallikarjuna reddy }
75132d31c79SAmireddy Mallikarjuna reddy
ldma_chan_desc_endian_cfg(struct ldma_chan * c,bool enable,u32 endian_type)75232d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_desc_endian_cfg(struct ldma_chan *c, bool enable,
75332d31c79SAmireddy Mallikarjuna reddy u32 endian_type)
75432d31c79SAmireddy Mallikarjuna reddy {
75532d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
75632d31c79SAmireddy Mallikarjuna reddy u32 mask = DMA_C_END_DES_EN | DMA_C_END_DESENDI;
75732d31c79SAmireddy Mallikarjuna reddy u32 val;
75832d31c79SAmireddy Mallikarjuna reddy
75932d31c79SAmireddy Mallikarjuna reddy if (enable)
76032d31c79SAmireddy Mallikarjuna reddy val = DMA_C_END_DES_EN | FIELD_PREP(DMA_C_END_DESENDI, endian_type);
76132d31c79SAmireddy Mallikarjuna reddy else
76232d31c79SAmireddy Mallikarjuna reddy val = 0;
76332d31c79SAmireddy Mallikarjuna reddy
76432d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
76532d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_C_ENDIAN);
76632d31c79SAmireddy Mallikarjuna reddy }
76732d31c79SAmireddy Mallikarjuna reddy
ldma_chan_hdr_mode_cfg(struct ldma_chan * c,u32 hdr_len,bool csum)76832d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_hdr_mode_cfg(struct ldma_chan *c, u32 hdr_len, bool csum)
76932d31c79SAmireddy Mallikarjuna reddy {
77032d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
77132d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
77232d31c79SAmireddy Mallikarjuna reddy
77332d31c79SAmireddy Mallikarjuna reddy /* NB, csum disabled, hdr length must be provided */
77432d31c79SAmireddy Mallikarjuna reddy if (!csum && (!hdr_len || hdr_len > DMA_HDR_LEN_MAX))
77532d31c79SAmireddy Mallikarjuna reddy return;
77632d31c79SAmireddy Mallikarjuna reddy
77732d31c79SAmireddy Mallikarjuna reddy mask = DMA_C_HDRM_HDR_SUM;
77832d31c79SAmireddy Mallikarjuna reddy val = DMA_C_HDRM_HDR_SUM;
77932d31c79SAmireddy Mallikarjuna reddy
78032d31c79SAmireddy Mallikarjuna reddy if (!csum && hdr_len)
78132d31c79SAmireddy Mallikarjuna reddy val = hdr_len;
78232d31c79SAmireddy Mallikarjuna reddy
78332d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
78432d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_C_HDRM);
78532d31c79SAmireddy Mallikarjuna reddy }
78632d31c79SAmireddy Mallikarjuna reddy
ldma_chan_rxwr_np_cfg(struct ldma_chan * c,bool enable)78732d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_rxwr_np_cfg(struct ldma_chan *c, bool enable)
78832d31c79SAmireddy Mallikarjuna reddy {
78932d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
79032d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
79132d31c79SAmireddy Mallikarjuna reddy
79232d31c79SAmireddy Mallikarjuna reddy /* Only valid for RX channel */
79332d31c79SAmireddy Mallikarjuna reddy if (ldma_chan_tx(c))
79432d31c79SAmireddy Mallikarjuna reddy return;
79532d31c79SAmireddy Mallikarjuna reddy
79632d31c79SAmireddy Mallikarjuna reddy mask = DMA_CCTRL_WR_NP_EN;
79732d31c79SAmireddy Mallikarjuna reddy val = enable ? DMA_CCTRL_WR_NP_EN : 0;
79832d31c79SAmireddy Mallikarjuna reddy
79932d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
80032d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CCTRL);
80132d31c79SAmireddy Mallikarjuna reddy }
80232d31c79SAmireddy Mallikarjuna reddy
ldma_chan_abc_cfg(struct ldma_chan * c,bool enable)80332d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_abc_cfg(struct ldma_chan *c, bool enable)
80432d31c79SAmireddy Mallikarjuna reddy {
80532d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
80632d31c79SAmireddy Mallikarjuna reddy u32 mask, val;
80732d31c79SAmireddy Mallikarjuna reddy
80832d31c79SAmireddy Mallikarjuna reddy if (d->ver < DMA_VER32 || ldma_chan_tx(c))
80932d31c79SAmireddy Mallikarjuna reddy return;
81032d31c79SAmireddy Mallikarjuna reddy
81132d31c79SAmireddy Mallikarjuna reddy mask = DMA_CCTRL_CH_ABC;
81232d31c79SAmireddy Mallikarjuna reddy val = enable ? DMA_CCTRL_CH_ABC : 0;
81332d31c79SAmireddy Mallikarjuna reddy
81432d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
81532d31c79SAmireddy Mallikarjuna reddy ldma_update_bits(d, mask, val, DMA_CCTRL);
81632d31c79SAmireddy Mallikarjuna reddy }
81732d31c79SAmireddy Mallikarjuna reddy
ldma_port_cfg(struct ldma_port * p)81832d31c79SAmireddy Mallikarjuna reddy static int ldma_port_cfg(struct ldma_port *p)
81932d31c79SAmireddy Mallikarjuna reddy {
82032d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
82132d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d;
82232d31c79SAmireddy Mallikarjuna reddy u32 reg;
82332d31c79SAmireddy Mallikarjuna reddy
82432d31c79SAmireddy Mallikarjuna reddy d = p->ldev;
82532d31c79SAmireddy Mallikarjuna reddy reg = FIELD_PREP(DMA_PCTRL_TXENDI, p->txendi);
82632d31c79SAmireddy Mallikarjuna reddy reg |= FIELD_PREP(DMA_PCTRL_RXENDI, p->rxendi);
82732d31c79SAmireddy Mallikarjuna reddy
82832d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22) {
82932d31c79SAmireddy Mallikarjuna reddy reg |= FIELD_PREP(DMA_PCTRL_TXBL, p->txbl);
83032d31c79SAmireddy Mallikarjuna reddy reg |= FIELD_PREP(DMA_PCTRL_RXBL, p->rxbl);
83132d31c79SAmireddy Mallikarjuna reddy } else {
83232d31c79SAmireddy Mallikarjuna reddy reg |= FIELD_PREP(DMA_PCTRL_PDEN, p->pkt_drop);
83332d31c79SAmireddy Mallikarjuna reddy
83432d31c79SAmireddy Mallikarjuna reddy if (p->txbl == DMA_BURSTL_32DW)
83532d31c79SAmireddy Mallikarjuna reddy reg |= DMA_PCTRL_TXBL32;
83632d31c79SAmireddy Mallikarjuna reddy else if (p->txbl == DMA_BURSTL_16DW)
83732d31c79SAmireddy Mallikarjuna reddy reg |= DMA_PCTRL_TXBL16;
83832d31c79SAmireddy Mallikarjuna reddy else
83932d31c79SAmireddy Mallikarjuna reddy reg |= FIELD_PREP(DMA_PCTRL_TXBL, DMA_PCTRL_TXBL_8);
84032d31c79SAmireddy Mallikarjuna reddy
84132d31c79SAmireddy Mallikarjuna reddy if (p->rxbl == DMA_BURSTL_32DW)
84232d31c79SAmireddy Mallikarjuna reddy reg |= DMA_PCTRL_RXBL32;
84332d31c79SAmireddy Mallikarjuna reddy else if (p->rxbl == DMA_BURSTL_16DW)
84432d31c79SAmireddy Mallikarjuna reddy reg |= DMA_PCTRL_RXBL16;
84532d31c79SAmireddy Mallikarjuna reddy else
84632d31c79SAmireddy Mallikarjuna reddy reg |= FIELD_PREP(DMA_PCTRL_RXBL, DMA_PCTRL_RXBL_8);
84732d31c79SAmireddy Mallikarjuna reddy }
84832d31c79SAmireddy Mallikarjuna reddy
84932d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
85032d31c79SAmireddy Mallikarjuna reddy writel(p->portid, d->base + DMA_PS);
85132d31c79SAmireddy Mallikarjuna reddy writel(reg, d->base + DMA_PCTRL);
85232d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
85332d31c79SAmireddy Mallikarjuna reddy
85432d31c79SAmireddy Mallikarjuna reddy reg = readl(d->base + DMA_PCTRL); /* read back */
85532d31c79SAmireddy Mallikarjuna reddy dev_dbg(d->dev, "Port Control 0x%08x configuration done\n", reg);
85632d31c79SAmireddy Mallikarjuna reddy
85732d31c79SAmireddy Mallikarjuna reddy return 0;
85832d31c79SAmireddy Mallikarjuna reddy }
85932d31c79SAmireddy Mallikarjuna reddy
ldma_chan_cfg(struct ldma_chan * c)86032d31c79SAmireddy Mallikarjuna reddy static int ldma_chan_cfg(struct ldma_chan *c)
86132d31c79SAmireddy Mallikarjuna reddy {
86232d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
86332d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
86432d31c79SAmireddy Mallikarjuna reddy u32 reg;
86532d31c79SAmireddy Mallikarjuna reddy
86632d31c79SAmireddy Mallikarjuna reddy reg = c->pden ? DMA_CCTRL_PDEN : 0;
86732d31c79SAmireddy Mallikarjuna reddy reg |= c->onoff ? DMA_CCTRL_ON : 0;
86832d31c79SAmireddy Mallikarjuna reddy reg |= c->rst ? DMA_CCTRL_RST : 0;
86932d31c79SAmireddy Mallikarjuna reddy
87032d31c79SAmireddy Mallikarjuna reddy ldma_chan_cctrl_cfg(c, reg);
87132d31c79SAmireddy Mallikarjuna reddy ldma_chan_irq_init(c);
87232d31c79SAmireddy Mallikarjuna reddy
87332d31c79SAmireddy Mallikarjuna reddy if (d->ver <= DMA_VER22)
87432d31c79SAmireddy Mallikarjuna reddy return 0;
87532d31c79SAmireddy Mallikarjuna reddy
87632d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
87732d31c79SAmireddy Mallikarjuna reddy ldma_chan_set_class(c, c->nr);
87832d31c79SAmireddy Mallikarjuna reddy ldma_chan_byte_offset_cfg(c, c->boff_len);
87932d31c79SAmireddy Mallikarjuna reddy ldma_chan_data_endian_cfg(c, c->data_endian_en, c->data_endian);
88032d31c79SAmireddy Mallikarjuna reddy ldma_chan_desc_endian_cfg(c, c->desc_endian_en, c->desc_endian);
88132d31c79SAmireddy Mallikarjuna reddy ldma_chan_hdr_mode_cfg(c, c->hdrm_len, c->hdrm_csum);
88232d31c79SAmireddy Mallikarjuna reddy ldma_chan_rxwr_np_cfg(c, c->desc_rx_np);
88332d31c79SAmireddy Mallikarjuna reddy ldma_chan_abc_cfg(c, c->abc_en);
88432d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
88532d31c79SAmireddy Mallikarjuna reddy
88632d31c79SAmireddy Mallikarjuna reddy if (ldma_chan_is_hw_desc(c))
88732d31c79SAmireddy Mallikarjuna reddy ldma_chan_desc_hw_cfg(c, c->desc_phys, c->desc_cnt);
88832d31c79SAmireddy Mallikarjuna reddy
88932d31c79SAmireddy Mallikarjuna reddy return 0;
89032d31c79SAmireddy Mallikarjuna reddy }
89132d31c79SAmireddy Mallikarjuna reddy
ldma_dev_init(struct ldma_dev * d)89232d31c79SAmireddy Mallikarjuna reddy static void ldma_dev_init(struct ldma_dev *d)
89332d31c79SAmireddy Mallikarjuna reddy {
89432d31c79SAmireddy Mallikarjuna reddy unsigned long ch_mask = (unsigned long)d->channels_mask;
89532d31c79SAmireddy Mallikarjuna reddy struct ldma_port *p;
89632d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c;
89732d31c79SAmireddy Mallikarjuna reddy int i;
89832d31c79SAmireddy Mallikarjuna reddy u32 j;
89932d31c79SAmireddy Mallikarjuna reddy
90032d31c79SAmireddy Mallikarjuna reddy spin_lock_init(&d->dev_lock);
90132d31c79SAmireddy Mallikarjuna reddy ldma_dev_reset(d);
90232d31c79SAmireddy Mallikarjuna reddy ldma_dev_cfg(d);
90332d31c79SAmireddy Mallikarjuna reddy
90432d31c79SAmireddy Mallikarjuna reddy /* DMA port initialization */
90532d31c79SAmireddy Mallikarjuna reddy for (i = 0; i < d->port_nrs; i++) {
90632d31c79SAmireddy Mallikarjuna reddy p = &d->ports[i];
90732d31c79SAmireddy Mallikarjuna reddy ldma_port_cfg(p);
90832d31c79SAmireddy Mallikarjuna reddy }
90932d31c79SAmireddy Mallikarjuna reddy
91032d31c79SAmireddy Mallikarjuna reddy /* DMA channel initialization */
91132d31c79SAmireddy Mallikarjuna reddy for_each_set_bit(j, &ch_mask, d->chan_nrs) {
91232d31c79SAmireddy Mallikarjuna reddy c = &d->chans[j];
91332d31c79SAmireddy Mallikarjuna reddy ldma_chan_cfg(c);
91432d31c79SAmireddy Mallikarjuna reddy }
91532d31c79SAmireddy Mallikarjuna reddy }
91632d31c79SAmireddy Mallikarjuna reddy
ldma_parse_dt(struct ldma_dev * d)91796b3bb18SPeter Harliman Liem static int ldma_parse_dt(struct ldma_dev *d)
91832d31c79SAmireddy Mallikarjuna reddy {
91932d31c79SAmireddy Mallikarjuna reddy struct fwnode_handle *fwnode = dev_fwnode(d->dev);
92032d31c79SAmireddy Mallikarjuna reddy struct ldma_port *p;
92132d31c79SAmireddy Mallikarjuna reddy int i;
92232d31c79SAmireddy Mallikarjuna reddy
92332d31c79SAmireddy Mallikarjuna reddy if (fwnode_property_read_bool(fwnode, "intel,dma-byte-en"))
92432d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_EN_BYTE_EN;
92532d31c79SAmireddy Mallikarjuna reddy
92632d31c79SAmireddy Mallikarjuna reddy if (fwnode_property_read_bool(fwnode, "intel,dma-dburst-wr"))
92732d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_DBURST_WR;
92832d31c79SAmireddy Mallikarjuna reddy
92932d31c79SAmireddy Mallikarjuna reddy if (fwnode_property_read_bool(fwnode, "intel,dma-drb"))
93032d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_DFT_DRB;
93132d31c79SAmireddy Mallikarjuna reddy
93232d31c79SAmireddy Mallikarjuna reddy if (fwnode_property_read_u32(fwnode, "intel,dma-poll-cnt",
93332d31c79SAmireddy Mallikarjuna reddy &d->pollcnt))
93432d31c79SAmireddy Mallikarjuna reddy d->pollcnt = DMA_DFT_POLL_CNT;
93532d31c79SAmireddy Mallikarjuna reddy
93632d31c79SAmireddy Mallikarjuna reddy if (d->inst->chan_fc)
93732d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_CHAN_FLOW_CTL;
93832d31c79SAmireddy Mallikarjuna reddy
93932d31c79SAmireddy Mallikarjuna reddy if (d->inst->desc_fod)
94032d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_DESC_FOD;
94132d31c79SAmireddy Mallikarjuna reddy
94232d31c79SAmireddy Mallikarjuna reddy if (d->inst->desc_in_sram)
94332d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_DESC_IN_SRAM;
94432d31c79SAmireddy Mallikarjuna reddy
94532d31c79SAmireddy Mallikarjuna reddy if (d->inst->valid_desc_fetch_ack)
94632d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_VALID_DESC_FETCH_ACK;
94732d31c79SAmireddy Mallikarjuna reddy
94832d31c79SAmireddy Mallikarjuna reddy if (d->ver > DMA_VER22) {
94932d31c79SAmireddy Mallikarjuna reddy if (!d->port_nrs)
95032d31c79SAmireddy Mallikarjuna reddy return -EINVAL;
95132d31c79SAmireddy Mallikarjuna reddy
95232d31c79SAmireddy Mallikarjuna reddy for (i = 0; i < d->port_nrs; i++) {
95332d31c79SAmireddy Mallikarjuna reddy p = &d->ports[i];
95432d31c79SAmireddy Mallikarjuna reddy p->rxendi = DMA_DFT_ENDIAN;
95532d31c79SAmireddy Mallikarjuna reddy p->txendi = DMA_DFT_ENDIAN;
95632d31c79SAmireddy Mallikarjuna reddy p->rxbl = DMA_DFT_BURST;
95732d31c79SAmireddy Mallikarjuna reddy p->txbl = DMA_DFT_BURST;
95832d31c79SAmireddy Mallikarjuna reddy p->pkt_drop = DMA_PKT_DROP_DIS;
95932d31c79SAmireddy Mallikarjuna reddy }
96032d31c79SAmireddy Mallikarjuna reddy }
96132d31c79SAmireddy Mallikarjuna reddy
96232d31c79SAmireddy Mallikarjuna reddy return 0;
96332d31c79SAmireddy Mallikarjuna reddy }
96432d31c79SAmireddy Mallikarjuna reddy
dma_free_desc_resource(struct virt_dma_desc * vdesc)96532d31c79SAmireddy Mallikarjuna reddy static void dma_free_desc_resource(struct virt_dma_desc *vdesc)
96632d31c79SAmireddy Mallikarjuna reddy {
96732d31c79SAmireddy Mallikarjuna reddy struct dw2_desc_sw *ds = to_lgm_dma_desc(vdesc);
96832d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = ds->chan;
96932d31c79SAmireddy Mallikarjuna reddy
97032d31c79SAmireddy Mallikarjuna reddy dma_pool_free(c->desc_pool, ds->desc_hw, ds->desc_phys);
97132d31c79SAmireddy Mallikarjuna reddy kfree(ds);
97232d31c79SAmireddy Mallikarjuna reddy }
97332d31c79SAmireddy Mallikarjuna reddy
97432d31c79SAmireddy Mallikarjuna reddy static struct dw2_desc_sw *
dma_alloc_desc_resource(int num,struct ldma_chan * c)97532d31c79SAmireddy Mallikarjuna reddy dma_alloc_desc_resource(int num, struct ldma_chan *c)
97632d31c79SAmireddy Mallikarjuna reddy {
97732d31c79SAmireddy Mallikarjuna reddy struct device *dev = c->vchan.chan.device->dev;
97832d31c79SAmireddy Mallikarjuna reddy struct dw2_desc_sw *ds;
97932d31c79SAmireddy Mallikarjuna reddy
98032d31c79SAmireddy Mallikarjuna reddy if (num > c->desc_num) {
98132d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "sg num %d exceed max %d\n", num, c->desc_num);
98232d31c79SAmireddy Mallikarjuna reddy return NULL;
98332d31c79SAmireddy Mallikarjuna reddy }
98432d31c79SAmireddy Mallikarjuna reddy
98532d31c79SAmireddy Mallikarjuna reddy ds = kzalloc(sizeof(*ds), GFP_NOWAIT);
98632d31c79SAmireddy Mallikarjuna reddy if (!ds)
98732d31c79SAmireddy Mallikarjuna reddy return NULL;
98832d31c79SAmireddy Mallikarjuna reddy
98932d31c79SAmireddy Mallikarjuna reddy ds->chan = c;
99032d31c79SAmireddy Mallikarjuna reddy ds->desc_hw = dma_pool_zalloc(c->desc_pool, GFP_ATOMIC,
99132d31c79SAmireddy Mallikarjuna reddy &ds->desc_phys);
99232d31c79SAmireddy Mallikarjuna reddy if (!ds->desc_hw) {
99332d31c79SAmireddy Mallikarjuna reddy dev_dbg(dev, "out of memory for link descriptor\n");
99432d31c79SAmireddy Mallikarjuna reddy kfree(ds);
99532d31c79SAmireddy Mallikarjuna reddy return NULL;
99632d31c79SAmireddy Mallikarjuna reddy }
99732d31c79SAmireddy Mallikarjuna reddy ds->desc_cnt = num;
99832d31c79SAmireddy Mallikarjuna reddy
99932d31c79SAmireddy Mallikarjuna reddy return ds;
100032d31c79SAmireddy Mallikarjuna reddy }
100132d31c79SAmireddy Mallikarjuna reddy
ldma_chan_irq_en(struct ldma_chan * c)100232d31c79SAmireddy Mallikarjuna reddy static void ldma_chan_irq_en(struct ldma_chan *c)
100332d31c79SAmireddy Mallikarjuna reddy {
100432d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
100532d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
100632d31c79SAmireddy Mallikarjuna reddy
100732d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&d->dev_lock, flags);
100832d31c79SAmireddy Mallikarjuna reddy writel(c->nr, d->base + DMA_CS);
100932d31c79SAmireddy Mallikarjuna reddy writel(DMA_CI_EOP, d->base + DMA_CIE);
101032d31c79SAmireddy Mallikarjuna reddy writel(BIT(c->nr), d->base + DMA_IRNEN);
101132d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&d->dev_lock, flags);
101232d31c79SAmireddy Mallikarjuna reddy }
101332d31c79SAmireddy Mallikarjuna reddy
ldma_issue_pending(struct dma_chan * chan)101432d31c79SAmireddy Mallikarjuna reddy static void ldma_issue_pending(struct dma_chan *chan)
101532d31c79SAmireddy Mallikarjuna reddy {
101632d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
101732d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
101832d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
101932d31c79SAmireddy Mallikarjuna reddy
102032d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22) {
102132d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&c->vchan.lock, flags);
102232d31c79SAmireddy Mallikarjuna reddy if (vchan_issue_pending(&c->vchan)) {
102332d31c79SAmireddy Mallikarjuna reddy struct virt_dma_desc *vdesc;
102432d31c79SAmireddy Mallikarjuna reddy
102532d31c79SAmireddy Mallikarjuna reddy /* Get the next descriptor */
102632d31c79SAmireddy Mallikarjuna reddy vdesc = vchan_next_desc(&c->vchan);
102732d31c79SAmireddy Mallikarjuna reddy if (!vdesc) {
102832d31c79SAmireddy Mallikarjuna reddy c->ds = NULL;
102932d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&c->vchan.lock, flags);
103032d31c79SAmireddy Mallikarjuna reddy return;
103132d31c79SAmireddy Mallikarjuna reddy }
103232d31c79SAmireddy Mallikarjuna reddy list_del(&vdesc->node);
103332d31c79SAmireddy Mallikarjuna reddy c->ds = to_lgm_dma_desc(vdesc);
103432d31c79SAmireddy Mallikarjuna reddy ldma_chan_desc_hw_cfg(c, c->ds->desc_phys, c->ds->desc_cnt);
103532d31c79SAmireddy Mallikarjuna reddy ldma_chan_irq_en(c);
103632d31c79SAmireddy Mallikarjuna reddy }
103732d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&c->vchan.lock, flags);
103832d31c79SAmireddy Mallikarjuna reddy }
103932d31c79SAmireddy Mallikarjuna reddy ldma_chan_on(c);
104032d31c79SAmireddy Mallikarjuna reddy }
104132d31c79SAmireddy Mallikarjuna reddy
ldma_synchronize(struct dma_chan * chan)104232d31c79SAmireddy Mallikarjuna reddy static void ldma_synchronize(struct dma_chan *chan)
104332d31c79SAmireddy Mallikarjuna reddy {
104432d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
104532d31c79SAmireddy Mallikarjuna reddy
104632d31c79SAmireddy Mallikarjuna reddy /*
104732d31c79SAmireddy Mallikarjuna reddy * clear any pending work if any. In that
104832d31c79SAmireddy Mallikarjuna reddy * case the resource needs to be free here.
104932d31c79SAmireddy Mallikarjuna reddy */
105032d31c79SAmireddy Mallikarjuna reddy cancel_work_sync(&c->work);
105132d31c79SAmireddy Mallikarjuna reddy vchan_synchronize(&c->vchan);
105232d31c79SAmireddy Mallikarjuna reddy if (c->ds)
105332d31c79SAmireddy Mallikarjuna reddy dma_free_desc_resource(&c->ds->vdesc);
105432d31c79SAmireddy Mallikarjuna reddy }
105532d31c79SAmireddy Mallikarjuna reddy
ldma_terminate_all(struct dma_chan * chan)105632d31c79SAmireddy Mallikarjuna reddy static int ldma_terminate_all(struct dma_chan *chan)
105732d31c79SAmireddy Mallikarjuna reddy {
105832d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
105932d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
106032d31c79SAmireddy Mallikarjuna reddy LIST_HEAD(head);
106132d31c79SAmireddy Mallikarjuna reddy
106232d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&c->vchan.lock, flags);
106332d31c79SAmireddy Mallikarjuna reddy vchan_get_all_descriptors(&c->vchan, &head);
106432d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&c->vchan.lock, flags);
106532d31c79SAmireddy Mallikarjuna reddy vchan_dma_desc_free_list(&c->vchan, &head);
106632d31c79SAmireddy Mallikarjuna reddy
106732d31c79SAmireddy Mallikarjuna reddy return ldma_chan_reset(c);
106832d31c79SAmireddy Mallikarjuna reddy }
106932d31c79SAmireddy Mallikarjuna reddy
ldma_resume_chan(struct dma_chan * chan)107032d31c79SAmireddy Mallikarjuna reddy static int ldma_resume_chan(struct dma_chan *chan)
107132d31c79SAmireddy Mallikarjuna reddy {
107232d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
107332d31c79SAmireddy Mallikarjuna reddy
107432d31c79SAmireddy Mallikarjuna reddy ldma_chan_on(c);
107532d31c79SAmireddy Mallikarjuna reddy
107632d31c79SAmireddy Mallikarjuna reddy return 0;
107732d31c79SAmireddy Mallikarjuna reddy }
107832d31c79SAmireddy Mallikarjuna reddy
ldma_pause_chan(struct dma_chan * chan)107932d31c79SAmireddy Mallikarjuna reddy static int ldma_pause_chan(struct dma_chan *chan)
108032d31c79SAmireddy Mallikarjuna reddy {
108132d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
108232d31c79SAmireddy Mallikarjuna reddy
108332d31c79SAmireddy Mallikarjuna reddy return ldma_chan_off(c);
108432d31c79SAmireddy Mallikarjuna reddy }
108532d31c79SAmireddy Mallikarjuna reddy
108632d31c79SAmireddy Mallikarjuna reddy static enum dma_status
ldma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)108732d31c79SAmireddy Mallikarjuna reddy ldma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
108832d31c79SAmireddy Mallikarjuna reddy struct dma_tx_state *txstate)
108932d31c79SAmireddy Mallikarjuna reddy {
109032d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
109132d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
109232d31c79SAmireddy Mallikarjuna reddy enum dma_status status = DMA_COMPLETE;
109332d31c79SAmireddy Mallikarjuna reddy
109432d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22)
109532d31c79SAmireddy Mallikarjuna reddy status = dma_cookie_status(chan, cookie, txstate);
109632d31c79SAmireddy Mallikarjuna reddy
109732d31c79SAmireddy Mallikarjuna reddy return status;
109832d31c79SAmireddy Mallikarjuna reddy }
109932d31c79SAmireddy Mallikarjuna reddy
dma_chan_irq(int irq,void * data)110032d31c79SAmireddy Mallikarjuna reddy static void dma_chan_irq(int irq, void *data)
110132d31c79SAmireddy Mallikarjuna reddy {
110232d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = data;
110332d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
110432d31c79SAmireddy Mallikarjuna reddy u32 stat;
110532d31c79SAmireddy Mallikarjuna reddy
110632d31c79SAmireddy Mallikarjuna reddy /* Disable channel interrupts */
110732d31c79SAmireddy Mallikarjuna reddy writel(c->nr, d->base + DMA_CS);
110832d31c79SAmireddy Mallikarjuna reddy stat = readl(d->base + DMA_CIS);
110932d31c79SAmireddy Mallikarjuna reddy if (!stat)
111032d31c79SAmireddy Mallikarjuna reddy return;
111132d31c79SAmireddy Mallikarjuna reddy
111232d31c79SAmireddy Mallikarjuna reddy writel(readl(d->base + DMA_CIE) & ~DMA_CI_ALL, d->base + DMA_CIE);
111332d31c79SAmireddy Mallikarjuna reddy writel(stat, d->base + DMA_CIS);
111432d31c79SAmireddy Mallikarjuna reddy queue_work(d->wq, &c->work);
111532d31c79SAmireddy Mallikarjuna reddy }
111632d31c79SAmireddy Mallikarjuna reddy
dma_interrupt(int irq,void * dev_id)111732d31c79SAmireddy Mallikarjuna reddy static irqreturn_t dma_interrupt(int irq, void *dev_id)
111832d31c79SAmireddy Mallikarjuna reddy {
111932d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = dev_id;
112032d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c;
112132d31c79SAmireddy Mallikarjuna reddy unsigned long irncr;
112232d31c79SAmireddy Mallikarjuna reddy u32 cid;
112332d31c79SAmireddy Mallikarjuna reddy
112432d31c79SAmireddy Mallikarjuna reddy irncr = readl(d->base + DMA_IRNCR);
112532d31c79SAmireddy Mallikarjuna reddy if (!irncr) {
112632d31c79SAmireddy Mallikarjuna reddy dev_err(d->dev, "dummy interrupt\n");
112732d31c79SAmireddy Mallikarjuna reddy return IRQ_NONE;
112832d31c79SAmireddy Mallikarjuna reddy }
112932d31c79SAmireddy Mallikarjuna reddy
113032d31c79SAmireddy Mallikarjuna reddy for_each_set_bit(cid, &irncr, d->chan_nrs) {
113132d31c79SAmireddy Mallikarjuna reddy /* Mask */
113232d31c79SAmireddy Mallikarjuna reddy writel(readl(d->base + DMA_IRNEN) & ~BIT(cid), d->base + DMA_IRNEN);
113332d31c79SAmireddy Mallikarjuna reddy /* Ack */
113432d31c79SAmireddy Mallikarjuna reddy writel(readl(d->base + DMA_IRNCR) | BIT(cid), d->base + DMA_IRNCR);
113532d31c79SAmireddy Mallikarjuna reddy
113632d31c79SAmireddy Mallikarjuna reddy c = &d->chans[cid];
113732d31c79SAmireddy Mallikarjuna reddy dma_chan_irq(irq, c);
113832d31c79SAmireddy Mallikarjuna reddy }
113932d31c79SAmireddy Mallikarjuna reddy
114032d31c79SAmireddy Mallikarjuna reddy return IRQ_HANDLED;
114132d31c79SAmireddy Mallikarjuna reddy }
114232d31c79SAmireddy Mallikarjuna reddy
prep_slave_burst_len(struct ldma_chan * c)114332d31c79SAmireddy Mallikarjuna reddy static void prep_slave_burst_len(struct ldma_chan *c)
114432d31c79SAmireddy Mallikarjuna reddy {
114532d31c79SAmireddy Mallikarjuna reddy struct ldma_port *p = c->port;
114632d31c79SAmireddy Mallikarjuna reddy struct dma_slave_config *cfg = &c->config;
114732d31c79SAmireddy Mallikarjuna reddy
114832d31c79SAmireddy Mallikarjuna reddy if (cfg->dst_maxburst)
114932d31c79SAmireddy Mallikarjuna reddy cfg->src_maxburst = cfg->dst_maxburst;
115032d31c79SAmireddy Mallikarjuna reddy
115132d31c79SAmireddy Mallikarjuna reddy /* TX and RX has the same burst length */
115232d31c79SAmireddy Mallikarjuna reddy p->txbl = ilog2(cfg->src_maxburst);
115332d31c79SAmireddy Mallikarjuna reddy p->rxbl = p->txbl;
115432d31c79SAmireddy Mallikarjuna reddy }
115532d31c79SAmireddy Mallikarjuna reddy
115632d31c79SAmireddy Mallikarjuna reddy static struct dma_async_tx_descriptor *
ldma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sglen,enum dma_transfer_direction dir,unsigned long flags,void * context)115732d31c79SAmireddy Mallikarjuna reddy ldma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
115832d31c79SAmireddy Mallikarjuna reddy unsigned int sglen, enum dma_transfer_direction dir,
115932d31c79SAmireddy Mallikarjuna reddy unsigned long flags, void *context)
116032d31c79SAmireddy Mallikarjuna reddy {
116132d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
116232d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
116332d31c79SAmireddy Mallikarjuna reddy size_t len, avail, total = 0;
116432d31c79SAmireddy Mallikarjuna reddy struct dw2_desc *hw_ds;
116532d31c79SAmireddy Mallikarjuna reddy struct dw2_desc_sw *ds;
116632d31c79SAmireddy Mallikarjuna reddy struct scatterlist *sg;
116732d31c79SAmireddy Mallikarjuna reddy int num = sglen, i;
116832d31c79SAmireddy Mallikarjuna reddy dma_addr_t addr;
116932d31c79SAmireddy Mallikarjuna reddy
117032d31c79SAmireddy Mallikarjuna reddy if (!sgl)
117132d31c79SAmireddy Mallikarjuna reddy return NULL;
117232d31c79SAmireddy Mallikarjuna reddy
117332d31c79SAmireddy Mallikarjuna reddy if (d->ver > DMA_VER22)
117432d31c79SAmireddy Mallikarjuna reddy return ldma_chan_desc_cfg(chan, sgl->dma_address, sglen);
117532d31c79SAmireddy Mallikarjuna reddy
117632d31c79SAmireddy Mallikarjuna reddy for_each_sg(sgl, sg, sglen, i) {
117732d31c79SAmireddy Mallikarjuna reddy avail = sg_dma_len(sg);
117832d31c79SAmireddy Mallikarjuna reddy if (avail > DMA_MAX_SIZE)
117932d31c79SAmireddy Mallikarjuna reddy num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
118032d31c79SAmireddy Mallikarjuna reddy }
118132d31c79SAmireddy Mallikarjuna reddy
118232d31c79SAmireddy Mallikarjuna reddy ds = dma_alloc_desc_resource(num, c);
118332d31c79SAmireddy Mallikarjuna reddy if (!ds)
118432d31c79SAmireddy Mallikarjuna reddy return NULL;
118532d31c79SAmireddy Mallikarjuna reddy
118632d31c79SAmireddy Mallikarjuna reddy c->ds = ds;
118732d31c79SAmireddy Mallikarjuna reddy
118832d31c79SAmireddy Mallikarjuna reddy num = 0;
118932d31c79SAmireddy Mallikarjuna reddy /* sop and eop has to be handled nicely */
119032d31c79SAmireddy Mallikarjuna reddy for_each_sg(sgl, sg, sglen, i) {
119132d31c79SAmireddy Mallikarjuna reddy addr = sg_dma_address(sg);
119232d31c79SAmireddy Mallikarjuna reddy avail = sg_dma_len(sg);
119332d31c79SAmireddy Mallikarjuna reddy total += avail;
119432d31c79SAmireddy Mallikarjuna reddy
119532d31c79SAmireddy Mallikarjuna reddy do {
119632d31c79SAmireddy Mallikarjuna reddy len = min_t(size_t, avail, DMA_MAX_SIZE);
119732d31c79SAmireddy Mallikarjuna reddy
119832d31c79SAmireddy Mallikarjuna reddy hw_ds = &ds->desc_hw[num];
119932d31c79SAmireddy Mallikarjuna reddy switch (sglen) {
120032d31c79SAmireddy Mallikarjuna reddy case 1:
120132d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_SOP;
120232d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_SOP, 1);
120332d31c79SAmireddy Mallikarjuna reddy
120432d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_EOP;
120532d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_EOP, 1);
120632d31c79SAmireddy Mallikarjuna reddy break;
120732d31c79SAmireddy Mallikarjuna reddy default:
120832d31c79SAmireddy Mallikarjuna reddy if (num == 0) {
120932d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_SOP;
121032d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_SOP, 1);
121132d31c79SAmireddy Mallikarjuna reddy
121232d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_EOP;
121332d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_EOP, 0);
121432d31c79SAmireddy Mallikarjuna reddy } else if (num == (sglen - 1)) {
121532d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_SOP;
121632d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_SOP, 0);
121732d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_EOP;
121832d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_EOP, 1);
121932d31c79SAmireddy Mallikarjuna reddy } else {
122032d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_SOP;
122132d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_SOP, 0);
122232d31c79SAmireddy Mallikarjuna reddy
122332d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_EOP;
122432d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_EOP, 0);
122532d31c79SAmireddy Mallikarjuna reddy }
122632d31c79SAmireddy Mallikarjuna reddy break;
122732d31c79SAmireddy Mallikarjuna reddy }
122832d31c79SAmireddy Mallikarjuna reddy /* Only 32 bit address supported */
122932d31c79SAmireddy Mallikarjuna reddy hw_ds->addr = (u32)addr;
123032d31c79SAmireddy Mallikarjuna reddy
123132d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_DATA_LEN;
123232d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_DATA_LEN, len);
123332d31c79SAmireddy Mallikarjuna reddy
123432d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_C;
123532d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_C, 0);
123632d31c79SAmireddy Mallikarjuna reddy
123732d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_BYTE_OFF;
123832d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_BYTE_OFF, addr & 0x3);
123932d31c79SAmireddy Mallikarjuna reddy
124032d31c79SAmireddy Mallikarjuna reddy /* Ensure data ready before ownership change */
124132d31c79SAmireddy Mallikarjuna reddy wmb();
124232d31c79SAmireddy Mallikarjuna reddy hw_ds->field &= ~DESC_OWN;
124332d31c79SAmireddy Mallikarjuna reddy hw_ds->field |= FIELD_PREP(DESC_OWN, DMA_OWN);
124432d31c79SAmireddy Mallikarjuna reddy
124532d31c79SAmireddy Mallikarjuna reddy /* Ensure ownership changed before moving forward */
124632d31c79SAmireddy Mallikarjuna reddy wmb();
124732d31c79SAmireddy Mallikarjuna reddy num++;
124832d31c79SAmireddy Mallikarjuna reddy addr += len;
124932d31c79SAmireddy Mallikarjuna reddy avail -= len;
125032d31c79SAmireddy Mallikarjuna reddy } while (avail);
125132d31c79SAmireddy Mallikarjuna reddy }
125232d31c79SAmireddy Mallikarjuna reddy
125332d31c79SAmireddy Mallikarjuna reddy ds->size = total;
125432d31c79SAmireddy Mallikarjuna reddy prep_slave_burst_len(c);
125532d31c79SAmireddy Mallikarjuna reddy
125632d31c79SAmireddy Mallikarjuna reddy return vchan_tx_prep(&c->vchan, &ds->vdesc, DMA_CTRL_ACK);
125732d31c79SAmireddy Mallikarjuna reddy }
125832d31c79SAmireddy Mallikarjuna reddy
125932d31c79SAmireddy Mallikarjuna reddy static int
ldma_slave_config(struct dma_chan * chan,struct dma_slave_config * cfg)126032d31c79SAmireddy Mallikarjuna reddy ldma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
126132d31c79SAmireddy Mallikarjuna reddy {
126232d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
126332d31c79SAmireddy Mallikarjuna reddy
126432d31c79SAmireddy Mallikarjuna reddy memcpy(&c->config, cfg, sizeof(c->config));
126532d31c79SAmireddy Mallikarjuna reddy
126632d31c79SAmireddy Mallikarjuna reddy return 0;
126732d31c79SAmireddy Mallikarjuna reddy }
126832d31c79SAmireddy Mallikarjuna reddy
ldma_alloc_chan_resources(struct dma_chan * chan)126932d31c79SAmireddy Mallikarjuna reddy static int ldma_alloc_chan_resources(struct dma_chan *chan)
127032d31c79SAmireddy Mallikarjuna reddy {
127132d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
127232d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
127332d31c79SAmireddy Mallikarjuna reddy struct device *dev = c->vchan.chan.device->dev;
127432d31c79SAmireddy Mallikarjuna reddy size_t desc_sz;
127532d31c79SAmireddy Mallikarjuna reddy
127632d31c79SAmireddy Mallikarjuna reddy if (d->ver > DMA_VER22) {
127732d31c79SAmireddy Mallikarjuna reddy c->flags |= CHAN_IN_USE;
127832d31c79SAmireddy Mallikarjuna reddy return 0;
127932d31c79SAmireddy Mallikarjuna reddy }
128032d31c79SAmireddy Mallikarjuna reddy
128132d31c79SAmireddy Mallikarjuna reddy if (c->desc_pool)
128232d31c79SAmireddy Mallikarjuna reddy return c->desc_num;
128332d31c79SAmireddy Mallikarjuna reddy
128432d31c79SAmireddy Mallikarjuna reddy desc_sz = c->desc_num * sizeof(struct dw2_desc);
128532d31c79SAmireddy Mallikarjuna reddy c->desc_pool = dma_pool_create(c->name, dev, desc_sz,
128632d31c79SAmireddy Mallikarjuna reddy __alignof__(struct dw2_desc), 0);
128732d31c79SAmireddy Mallikarjuna reddy
128832d31c79SAmireddy Mallikarjuna reddy if (!c->desc_pool) {
128932d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "unable to allocate descriptor pool\n");
129032d31c79SAmireddy Mallikarjuna reddy return -ENOMEM;
129132d31c79SAmireddy Mallikarjuna reddy }
129232d31c79SAmireddy Mallikarjuna reddy
129332d31c79SAmireddy Mallikarjuna reddy return c->desc_num;
129432d31c79SAmireddy Mallikarjuna reddy }
129532d31c79SAmireddy Mallikarjuna reddy
ldma_free_chan_resources(struct dma_chan * chan)129632d31c79SAmireddy Mallikarjuna reddy static void ldma_free_chan_resources(struct dma_chan *chan)
129732d31c79SAmireddy Mallikarjuna reddy {
129832d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = to_ldma_chan(chan);
129932d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
130032d31c79SAmireddy Mallikarjuna reddy
130132d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22) {
130232d31c79SAmireddy Mallikarjuna reddy dma_pool_destroy(c->desc_pool);
130332d31c79SAmireddy Mallikarjuna reddy c->desc_pool = NULL;
130432d31c79SAmireddy Mallikarjuna reddy vchan_free_chan_resources(to_virt_chan(chan));
130532d31c79SAmireddy Mallikarjuna reddy ldma_chan_reset(c);
130632d31c79SAmireddy Mallikarjuna reddy } else {
130732d31c79SAmireddy Mallikarjuna reddy c->flags &= ~CHAN_IN_USE;
130832d31c79SAmireddy Mallikarjuna reddy }
130932d31c79SAmireddy Mallikarjuna reddy }
131032d31c79SAmireddy Mallikarjuna reddy
dma_work(struct work_struct * work)131132d31c79SAmireddy Mallikarjuna reddy static void dma_work(struct work_struct *work)
131232d31c79SAmireddy Mallikarjuna reddy {
131332d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c = container_of(work, struct ldma_chan, work);
131432d31c79SAmireddy Mallikarjuna reddy struct dma_async_tx_descriptor *tx = &c->ds->vdesc.tx;
131532d31c79SAmireddy Mallikarjuna reddy struct virt_dma_chan *vc = &c->vchan;
131632d31c79SAmireddy Mallikarjuna reddy struct dmaengine_desc_callback cb;
131732d31c79SAmireddy Mallikarjuna reddy struct virt_dma_desc *vd, *_vd;
131832d31c79SAmireddy Mallikarjuna reddy unsigned long flags;
131932d31c79SAmireddy Mallikarjuna reddy LIST_HEAD(head);
132032d31c79SAmireddy Mallikarjuna reddy
132132d31c79SAmireddy Mallikarjuna reddy spin_lock_irqsave(&c->vchan.lock, flags);
132232d31c79SAmireddy Mallikarjuna reddy list_splice_tail_init(&vc->desc_completed, &head);
132332d31c79SAmireddy Mallikarjuna reddy spin_unlock_irqrestore(&c->vchan.lock, flags);
132432d31c79SAmireddy Mallikarjuna reddy dmaengine_desc_get_callback(tx, &cb);
132532d31c79SAmireddy Mallikarjuna reddy dma_cookie_complete(tx);
132632d31c79SAmireddy Mallikarjuna reddy dmaengine_desc_callback_invoke(&cb, NULL);
132732d31c79SAmireddy Mallikarjuna reddy
132832d31c79SAmireddy Mallikarjuna reddy list_for_each_entry_safe(vd, _vd, &head, node) {
132932d31c79SAmireddy Mallikarjuna reddy dmaengine_desc_get_callback(tx, &cb);
133032d31c79SAmireddy Mallikarjuna reddy dma_cookie_complete(tx);
133132d31c79SAmireddy Mallikarjuna reddy list_del(&vd->node);
133232d31c79SAmireddy Mallikarjuna reddy dmaengine_desc_callback_invoke(&cb, NULL);
133332d31c79SAmireddy Mallikarjuna reddy
133432d31c79SAmireddy Mallikarjuna reddy vchan_vdesc_fini(vd);
133532d31c79SAmireddy Mallikarjuna reddy }
133632d31c79SAmireddy Mallikarjuna reddy c->ds = NULL;
133732d31c79SAmireddy Mallikarjuna reddy }
133832d31c79SAmireddy Mallikarjuna reddy
133932d31c79SAmireddy Mallikarjuna reddy static void
update_burst_len_v22(struct ldma_chan * c,struct ldma_port * p,u32 burst)134032d31c79SAmireddy Mallikarjuna reddy update_burst_len_v22(struct ldma_chan *c, struct ldma_port *p, u32 burst)
134132d31c79SAmireddy Mallikarjuna reddy {
134232d31c79SAmireddy Mallikarjuna reddy if (ldma_chan_tx(c))
134332d31c79SAmireddy Mallikarjuna reddy p->txbl = ilog2(burst);
134432d31c79SAmireddy Mallikarjuna reddy else
134532d31c79SAmireddy Mallikarjuna reddy p->rxbl = ilog2(burst);
134632d31c79SAmireddy Mallikarjuna reddy }
134732d31c79SAmireddy Mallikarjuna reddy
134832d31c79SAmireddy Mallikarjuna reddy static void
update_burst_len_v3X(struct ldma_chan * c,struct ldma_port * p,u32 burst)134932d31c79SAmireddy Mallikarjuna reddy update_burst_len_v3X(struct ldma_chan *c, struct ldma_port *p, u32 burst)
135032d31c79SAmireddy Mallikarjuna reddy {
135132d31c79SAmireddy Mallikarjuna reddy if (ldma_chan_tx(c))
135232d31c79SAmireddy Mallikarjuna reddy p->txbl = burst;
135332d31c79SAmireddy Mallikarjuna reddy else
135432d31c79SAmireddy Mallikarjuna reddy p->rxbl = burst;
135532d31c79SAmireddy Mallikarjuna reddy }
135632d31c79SAmireddy Mallikarjuna reddy
135732d31c79SAmireddy Mallikarjuna reddy static int
update_client_configs(struct of_dma * ofdma,struct of_phandle_args * spec)135832d31c79SAmireddy Mallikarjuna reddy update_client_configs(struct of_dma *ofdma, struct of_phandle_args *spec)
135932d31c79SAmireddy Mallikarjuna reddy {
136032d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = ofdma->of_dma_data;
136132d31c79SAmireddy Mallikarjuna reddy u32 chan_id = spec->args[0];
136232d31c79SAmireddy Mallikarjuna reddy u32 port_id = spec->args[1];
136332d31c79SAmireddy Mallikarjuna reddy u32 burst = spec->args[2];
136432d31c79SAmireddy Mallikarjuna reddy struct ldma_port *p;
136532d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c;
136632d31c79SAmireddy Mallikarjuna reddy
136732d31c79SAmireddy Mallikarjuna reddy if (chan_id >= d->chan_nrs || port_id >= d->port_nrs)
136832d31c79SAmireddy Mallikarjuna reddy return 0;
136932d31c79SAmireddy Mallikarjuna reddy
137032d31c79SAmireddy Mallikarjuna reddy p = &d->ports[port_id];
137132d31c79SAmireddy Mallikarjuna reddy c = &d->chans[chan_id];
137232d31c79SAmireddy Mallikarjuna reddy c->port = p;
137332d31c79SAmireddy Mallikarjuna reddy
137432d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22)
137532d31c79SAmireddy Mallikarjuna reddy update_burst_len_v22(c, p, burst);
137632d31c79SAmireddy Mallikarjuna reddy else
137732d31c79SAmireddy Mallikarjuna reddy update_burst_len_v3X(c, p, burst);
137832d31c79SAmireddy Mallikarjuna reddy
137932d31c79SAmireddy Mallikarjuna reddy ldma_port_cfg(p);
138032d31c79SAmireddy Mallikarjuna reddy
138132d31c79SAmireddy Mallikarjuna reddy return 1;
138232d31c79SAmireddy Mallikarjuna reddy }
138332d31c79SAmireddy Mallikarjuna reddy
ldma_xlate(struct of_phandle_args * spec,struct of_dma * ofdma)138432d31c79SAmireddy Mallikarjuna reddy static struct dma_chan *ldma_xlate(struct of_phandle_args *spec,
138532d31c79SAmireddy Mallikarjuna reddy struct of_dma *ofdma)
138632d31c79SAmireddy Mallikarjuna reddy {
138732d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = ofdma->of_dma_data;
138832d31c79SAmireddy Mallikarjuna reddy u32 chan_id = spec->args[0];
138932d31c79SAmireddy Mallikarjuna reddy int ret;
139032d31c79SAmireddy Mallikarjuna reddy
139132d31c79SAmireddy Mallikarjuna reddy if (!spec->args_count)
139232d31c79SAmireddy Mallikarjuna reddy return NULL;
139332d31c79SAmireddy Mallikarjuna reddy
139432d31c79SAmireddy Mallikarjuna reddy /* if args_count is 1 driver use default settings */
139532d31c79SAmireddy Mallikarjuna reddy if (spec->args_count > 1) {
139632d31c79SAmireddy Mallikarjuna reddy ret = update_client_configs(ofdma, spec);
139732d31c79SAmireddy Mallikarjuna reddy if (!ret)
139832d31c79SAmireddy Mallikarjuna reddy return NULL;
139932d31c79SAmireddy Mallikarjuna reddy }
140032d31c79SAmireddy Mallikarjuna reddy
140132d31c79SAmireddy Mallikarjuna reddy return dma_get_slave_channel(&d->chans[chan_id].vchan.chan);
140232d31c79SAmireddy Mallikarjuna reddy }
140332d31c79SAmireddy Mallikarjuna reddy
ldma_dma_init_v22(int i,struct ldma_dev * d)140432d31c79SAmireddy Mallikarjuna reddy static void ldma_dma_init_v22(int i, struct ldma_dev *d)
140532d31c79SAmireddy Mallikarjuna reddy {
140632d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c;
140732d31c79SAmireddy Mallikarjuna reddy
140832d31c79SAmireddy Mallikarjuna reddy c = &d->chans[i];
140932d31c79SAmireddy Mallikarjuna reddy c->nr = i; /* Real channel number */
141032d31c79SAmireddy Mallikarjuna reddy c->rst = DMA_CHAN_RST;
141132d31c79SAmireddy Mallikarjuna reddy c->desc_num = DMA_DFT_DESC_NUM;
141232d31c79SAmireddy Mallikarjuna reddy snprintf(c->name, sizeof(c->name), "chan%d", c->nr);
141332d31c79SAmireddy Mallikarjuna reddy INIT_WORK(&c->work, dma_work);
141432d31c79SAmireddy Mallikarjuna reddy c->vchan.desc_free = dma_free_desc_resource;
141532d31c79SAmireddy Mallikarjuna reddy vchan_init(&c->vchan, &d->dma_dev);
141632d31c79SAmireddy Mallikarjuna reddy }
141732d31c79SAmireddy Mallikarjuna reddy
ldma_dma_init_v3X(int i,struct ldma_dev * d)141832d31c79SAmireddy Mallikarjuna reddy static void ldma_dma_init_v3X(int i, struct ldma_dev *d)
141932d31c79SAmireddy Mallikarjuna reddy {
142032d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c;
142132d31c79SAmireddy Mallikarjuna reddy
142232d31c79SAmireddy Mallikarjuna reddy c = &d->chans[i];
142332d31c79SAmireddy Mallikarjuna reddy c->data_endian = DMA_DFT_ENDIAN;
142432d31c79SAmireddy Mallikarjuna reddy c->desc_endian = DMA_DFT_ENDIAN;
142532d31c79SAmireddy Mallikarjuna reddy c->data_endian_en = false;
142632d31c79SAmireddy Mallikarjuna reddy c->desc_endian_en = false;
142732d31c79SAmireddy Mallikarjuna reddy c->desc_rx_np = false;
142832d31c79SAmireddy Mallikarjuna reddy c->flags |= DEVICE_ALLOC_DESC;
142932d31c79SAmireddy Mallikarjuna reddy c->onoff = DMA_CH_OFF;
143032d31c79SAmireddy Mallikarjuna reddy c->rst = DMA_CHAN_RST;
143132d31c79SAmireddy Mallikarjuna reddy c->abc_en = true;
143232d31c79SAmireddy Mallikarjuna reddy c->hdrm_csum = false;
143332d31c79SAmireddy Mallikarjuna reddy c->boff_len = 0;
143432d31c79SAmireddy Mallikarjuna reddy c->nr = i;
143532d31c79SAmireddy Mallikarjuna reddy c->vchan.desc_free = dma_free_desc_resource;
143632d31c79SAmireddy Mallikarjuna reddy vchan_init(&c->vchan, &d->dma_dev);
143732d31c79SAmireddy Mallikarjuna reddy }
143832d31c79SAmireddy Mallikarjuna reddy
ldma_init_v22(struct ldma_dev * d,struct platform_device * pdev)143932d31c79SAmireddy Mallikarjuna reddy static int ldma_init_v22(struct ldma_dev *d, struct platform_device *pdev)
144032d31c79SAmireddy Mallikarjuna reddy {
144132d31c79SAmireddy Mallikarjuna reddy int ret;
144232d31c79SAmireddy Mallikarjuna reddy
144332d31c79SAmireddy Mallikarjuna reddy ret = device_property_read_u32(d->dev, "dma-channels", &d->chan_nrs);
144432d31c79SAmireddy Mallikarjuna reddy if (ret < 0) {
144532d31c79SAmireddy Mallikarjuna reddy dev_err(d->dev, "unable to read dma-channels property\n");
144632d31c79SAmireddy Mallikarjuna reddy return ret;
144732d31c79SAmireddy Mallikarjuna reddy }
144832d31c79SAmireddy Mallikarjuna reddy
144932d31c79SAmireddy Mallikarjuna reddy d->irq = platform_get_irq(pdev, 0);
145032d31c79SAmireddy Mallikarjuna reddy if (d->irq < 0)
145132d31c79SAmireddy Mallikarjuna reddy return d->irq;
145232d31c79SAmireddy Mallikarjuna reddy
145332d31c79SAmireddy Mallikarjuna reddy ret = devm_request_irq(&pdev->dev, d->irq, dma_interrupt, 0,
145432d31c79SAmireddy Mallikarjuna reddy DRIVER_NAME, d);
145532d31c79SAmireddy Mallikarjuna reddy if (ret)
145632d31c79SAmireddy Mallikarjuna reddy return ret;
145732d31c79SAmireddy Mallikarjuna reddy
145832d31c79SAmireddy Mallikarjuna reddy d->wq = alloc_ordered_workqueue("dma_wq", WQ_MEM_RECLAIM |
145932d31c79SAmireddy Mallikarjuna reddy WQ_HIGHPRI);
146032d31c79SAmireddy Mallikarjuna reddy if (!d->wq)
146132d31c79SAmireddy Mallikarjuna reddy return -ENOMEM;
146232d31c79SAmireddy Mallikarjuna reddy
146332d31c79SAmireddy Mallikarjuna reddy return 0;
146432d31c79SAmireddy Mallikarjuna reddy }
146532d31c79SAmireddy Mallikarjuna reddy
ldma_clk_disable(void * data)146632d31c79SAmireddy Mallikarjuna reddy static void ldma_clk_disable(void *data)
146732d31c79SAmireddy Mallikarjuna reddy {
146832d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d = data;
146932d31c79SAmireddy Mallikarjuna reddy
147032d31c79SAmireddy Mallikarjuna reddy clk_disable_unprepare(d->core_clk);
147132d31c79SAmireddy Mallikarjuna reddy reset_control_assert(d->rst);
147232d31c79SAmireddy Mallikarjuna reddy }
147332d31c79SAmireddy Mallikarjuna reddy
147432d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data dma0 = {
147532d31c79SAmireddy Mallikarjuna reddy .name = "dma0",
147632d31c79SAmireddy Mallikarjuna reddy .chan_fc = false,
147732d31c79SAmireddy Mallikarjuna reddy .desc_fod = false,
147832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = false,
147932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = false,
148032d31c79SAmireddy Mallikarjuna reddy };
148132d31c79SAmireddy Mallikarjuna reddy
148232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data dma2tx = {
148332d31c79SAmireddy Mallikarjuna reddy .name = "dma2tx",
148432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_TX,
148532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
148632d31c79SAmireddy Mallikarjuna reddy .chan_fc = true,
148732d31c79SAmireddy Mallikarjuna reddy .desc_fod = true,
148832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
148932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = true,
149032d31c79SAmireddy Mallikarjuna reddy };
149132d31c79SAmireddy Mallikarjuna reddy
149232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data dma1rx = {
149332d31c79SAmireddy Mallikarjuna reddy .name = "dma1rx",
149432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_RX,
149532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
149632d31c79SAmireddy Mallikarjuna reddy .chan_fc = false,
149732d31c79SAmireddy Mallikarjuna reddy .desc_fod = true,
149832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
149932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = false,
150032d31c79SAmireddy Mallikarjuna reddy };
150132d31c79SAmireddy Mallikarjuna reddy
150232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data dma1tx = {
150332d31c79SAmireddy Mallikarjuna reddy .name = "dma1tx",
150432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_TX,
150532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
150632d31c79SAmireddy Mallikarjuna reddy .chan_fc = true,
150732d31c79SAmireddy Mallikarjuna reddy .desc_fod = true,
150832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
150932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = true,
151032d31c79SAmireddy Mallikarjuna reddy };
151132d31c79SAmireddy Mallikarjuna reddy
151232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data dma0tx = {
151332d31c79SAmireddy Mallikarjuna reddy .name = "dma0tx",
151432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_TX,
151532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
151632d31c79SAmireddy Mallikarjuna reddy .chan_fc = true,
151732d31c79SAmireddy Mallikarjuna reddy .desc_fod = true,
151832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
151932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = true,
152032d31c79SAmireddy Mallikarjuna reddy };
152132d31c79SAmireddy Mallikarjuna reddy
152232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data dma3 = {
152332d31c79SAmireddy Mallikarjuna reddy .name = "dma3",
152432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_MCPY,
152532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
152632d31c79SAmireddy Mallikarjuna reddy .chan_fc = false,
152732d31c79SAmireddy Mallikarjuna reddy .desc_fod = false,
152832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
152932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = false,
153032d31c79SAmireddy Mallikarjuna reddy };
153132d31c79SAmireddy Mallikarjuna reddy
153232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data toe_dma30 = {
153332d31c79SAmireddy Mallikarjuna reddy .name = "toe_dma30",
153432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_MCPY,
153532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
153632d31c79SAmireddy Mallikarjuna reddy .chan_fc = false,
153732d31c79SAmireddy Mallikarjuna reddy .desc_fod = false,
153832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
153932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = true,
154032d31c79SAmireddy Mallikarjuna reddy };
154132d31c79SAmireddy Mallikarjuna reddy
154232d31c79SAmireddy Mallikarjuna reddy static const struct ldma_inst_data toe_dma31 = {
154332d31c79SAmireddy Mallikarjuna reddy .name = "toe_dma31",
154432d31c79SAmireddy Mallikarjuna reddy .type = DMA_TYPE_MCPY,
154532d31c79SAmireddy Mallikarjuna reddy .orrc = 16,
154632d31c79SAmireddy Mallikarjuna reddy .chan_fc = false,
154732d31c79SAmireddy Mallikarjuna reddy .desc_fod = false,
154832d31c79SAmireddy Mallikarjuna reddy .desc_in_sram = true,
154932d31c79SAmireddy Mallikarjuna reddy .valid_desc_fetch_ack = true,
155032d31c79SAmireddy Mallikarjuna reddy };
155132d31c79SAmireddy Mallikarjuna reddy
155232d31c79SAmireddy Mallikarjuna reddy static const struct of_device_id intel_ldma_match[] = {
155332d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-cdma", .data = &dma0},
155432d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-dma2tx", .data = &dma2tx},
155532d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-dma1rx", .data = &dma1rx},
155632d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-dma1tx", .data = &dma1tx},
155732d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-dma0tx", .data = &dma0tx},
155832d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-dma3", .data = &dma3},
155932d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-toe-dma30", .data = &toe_dma30},
156032d31c79SAmireddy Mallikarjuna reddy { .compatible = "intel,lgm-toe-dma31", .data = &toe_dma31},
156132d31c79SAmireddy Mallikarjuna reddy {}
156232d31c79SAmireddy Mallikarjuna reddy };
156332d31c79SAmireddy Mallikarjuna reddy
intel_ldma_probe(struct platform_device * pdev)156432d31c79SAmireddy Mallikarjuna reddy static int intel_ldma_probe(struct platform_device *pdev)
156532d31c79SAmireddy Mallikarjuna reddy {
156632d31c79SAmireddy Mallikarjuna reddy struct device *dev = &pdev->dev;
156732d31c79SAmireddy Mallikarjuna reddy struct dma_device *dma_dev;
156832d31c79SAmireddy Mallikarjuna reddy unsigned long ch_mask;
156932d31c79SAmireddy Mallikarjuna reddy struct ldma_chan *c;
157032d31c79SAmireddy Mallikarjuna reddy struct ldma_port *p;
157132d31c79SAmireddy Mallikarjuna reddy struct ldma_dev *d;
157232d31c79SAmireddy Mallikarjuna reddy u32 id, bitn = 32, j;
157332d31c79SAmireddy Mallikarjuna reddy int i, ret;
157432d31c79SAmireddy Mallikarjuna reddy
157532d31c79SAmireddy Mallikarjuna reddy d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
157632d31c79SAmireddy Mallikarjuna reddy if (!d)
157732d31c79SAmireddy Mallikarjuna reddy return -ENOMEM;
157832d31c79SAmireddy Mallikarjuna reddy
157932d31c79SAmireddy Mallikarjuna reddy /* Link controller to platform device */
158032d31c79SAmireddy Mallikarjuna reddy d->dev = &pdev->dev;
158132d31c79SAmireddy Mallikarjuna reddy
158232d31c79SAmireddy Mallikarjuna reddy d->inst = device_get_match_data(dev);
158332d31c79SAmireddy Mallikarjuna reddy if (!d->inst) {
158432d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "No device match found\n");
158532d31c79SAmireddy Mallikarjuna reddy return -ENODEV;
158632d31c79SAmireddy Mallikarjuna reddy }
158732d31c79SAmireddy Mallikarjuna reddy
158832d31c79SAmireddy Mallikarjuna reddy d->base = devm_platform_ioremap_resource(pdev, 0);
158932d31c79SAmireddy Mallikarjuna reddy if (IS_ERR(d->base))
159032d31c79SAmireddy Mallikarjuna reddy return PTR_ERR(d->base);
159132d31c79SAmireddy Mallikarjuna reddy
159232d31c79SAmireddy Mallikarjuna reddy /* Power up and reset the dma engine, some DMAs always on?? */
159332d31c79SAmireddy Mallikarjuna reddy d->core_clk = devm_clk_get_optional(dev, NULL);
159432d31c79SAmireddy Mallikarjuna reddy if (IS_ERR(d->core_clk))
159532d31c79SAmireddy Mallikarjuna reddy return PTR_ERR(d->core_clk);
159632d31c79SAmireddy Mallikarjuna reddy
159732d31c79SAmireddy Mallikarjuna reddy d->rst = devm_reset_control_get_optional(dev, NULL);
159832d31c79SAmireddy Mallikarjuna reddy if (IS_ERR(d->rst))
159932d31c79SAmireddy Mallikarjuna reddy return PTR_ERR(d->rst);
16001dbe67b9SChristophe JAILLET
16011dbe67b9SChristophe JAILLET clk_prepare_enable(d->core_clk);
160232d31c79SAmireddy Mallikarjuna reddy reset_control_deassert(d->rst);
160332d31c79SAmireddy Mallikarjuna reddy
160432d31c79SAmireddy Mallikarjuna reddy ret = devm_add_action_or_reset(dev, ldma_clk_disable, d);
160532d31c79SAmireddy Mallikarjuna reddy if (ret) {
160632d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret);
160732d31c79SAmireddy Mallikarjuna reddy return ret;
160832d31c79SAmireddy Mallikarjuna reddy }
160932d31c79SAmireddy Mallikarjuna reddy
161032d31c79SAmireddy Mallikarjuna reddy id = readl(d->base + DMA_ID);
161132d31c79SAmireddy Mallikarjuna reddy d->chan_nrs = FIELD_GET(DMA_ID_CHNR, id);
161232d31c79SAmireddy Mallikarjuna reddy d->port_nrs = FIELD_GET(DMA_ID_PNR, id);
161332d31c79SAmireddy Mallikarjuna reddy d->ver = FIELD_GET(DMA_ID_REV, id);
161432d31c79SAmireddy Mallikarjuna reddy
161532d31c79SAmireddy Mallikarjuna reddy if (id & DMA_ID_AW_36B)
161632d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_ADDR_36BIT;
161732d31c79SAmireddy Mallikarjuna reddy
161832d31c79SAmireddy Mallikarjuna reddy if (IS_ENABLED(CONFIG_64BIT) && (id & DMA_ID_AW_36B))
161932d31c79SAmireddy Mallikarjuna reddy bitn = 36;
162032d31c79SAmireddy Mallikarjuna reddy
162132d31c79SAmireddy Mallikarjuna reddy if (id & DMA_ID_DW_128B)
162232d31c79SAmireddy Mallikarjuna reddy d->flags |= DMA_DATA_128BIT;
162332d31c79SAmireddy Mallikarjuna reddy
162432d31c79SAmireddy Mallikarjuna reddy ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(bitn));
162532d31c79SAmireddy Mallikarjuna reddy if (ret) {
162632d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "No usable DMA configuration\n");
162732d31c79SAmireddy Mallikarjuna reddy return ret;
162832d31c79SAmireddy Mallikarjuna reddy }
162932d31c79SAmireddy Mallikarjuna reddy
163032d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22) {
163132d31c79SAmireddy Mallikarjuna reddy ret = ldma_init_v22(d, pdev);
163232d31c79SAmireddy Mallikarjuna reddy if (ret)
163332d31c79SAmireddy Mallikarjuna reddy return ret;
163432d31c79SAmireddy Mallikarjuna reddy }
163532d31c79SAmireddy Mallikarjuna reddy
163632d31c79SAmireddy Mallikarjuna reddy ret = device_property_read_u32(dev, "dma-channel-mask", &d->channels_mask);
163732d31c79SAmireddy Mallikarjuna reddy if (ret < 0)
163832d31c79SAmireddy Mallikarjuna reddy d->channels_mask = GENMASK(d->chan_nrs - 1, 0);
163932d31c79SAmireddy Mallikarjuna reddy
164032d31c79SAmireddy Mallikarjuna reddy dma_dev = &d->dma_dev;
164132d31c79SAmireddy Mallikarjuna reddy
164232d31c79SAmireddy Mallikarjuna reddy dma_cap_zero(dma_dev->cap_mask);
164332d31c79SAmireddy Mallikarjuna reddy dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
164432d31c79SAmireddy Mallikarjuna reddy
164532d31c79SAmireddy Mallikarjuna reddy /* Channel initializations */
164632d31c79SAmireddy Mallikarjuna reddy INIT_LIST_HEAD(&dma_dev->channels);
164732d31c79SAmireddy Mallikarjuna reddy
164832d31c79SAmireddy Mallikarjuna reddy /* Port Initializations */
164932d31c79SAmireddy Mallikarjuna reddy d->ports = devm_kcalloc(dev, d->port_nrs, sizeof(*p), GFP_KERNEL);
165032d31c79SAmireddy Mallikarjuna reddy if (!d->ports)
165132d31c79SAmireddy Mallikarjuna reddy return -ENOMEM;
165232d31c79SAmireddy Mallikarjuna reddy
165332d31c79SAmireddy Mallikarjuna reddy /* Channels Initializations */
165432d31c79SAmireddy Mallikarjuna reddy d->chans = devm_kcalloc(d->dev, d->chan_nrs, sizeof(*c), GFP_KERNEL);
165532d31c79SAmireddy Mallikarjuna reddy if (!d->chans)
165632d31c79SAmireddy Mallikarjuna reddy return -ENOMEM;
165732d31c79SAmireddy Mallikarjuna reddy
165832d31c79SAmireddy Mallikarjuna reddy for (i = 0; i < d->port_nrs; i++) {
165932d31c79SAmireddy Mallikarjuna reddy p = &d->ports[i];
166032d31c79SAmireddy Mallikarjuna reddy p->portid = i;
166132d31c79SAmireddy Mallikarjuna reddy p->ldev = d;
166232d31c79SAmireddy Mallikarjuna reddy }
166332d31c79SAmireddy Mallikarjuna reddy
166432d31c79SAmireddy Mallikarjuna reddy dma_dev->dev = &pdev->dev;
166532d31c79SAmireddy Mallikarjuna reddy
166632d31c79SAmireddy Mallikarjuna reddy ch_mask = (unsigned long)d->channels_mask;
166732d31c79SAmireddy Mallikarjuna reddy for_each_set_bit(j, &ch_mask, d->chan_nrs) {
166832d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22)
166932d31c79SAmireddy Mallikarjuna reddy ldma_dma_init_v22(j, d);
167032d31c79SAmireddy Mallikarjuna reddy else
167132d31c79SAmireddy Mallikarjuna reddy ldma_dma_init_v3X(j, d);
167232d31c79SAmireddy Mallikarjuna reddy }
167332d31c79SAmireddy Mallikarjuna reddy
167496b3bb18SPeter Harliman Liem ret = ldma_parse_dt(d);
167596b3bb18SPeter Harliman Liem if (ret)
167696b3bb18SPeter Harliman Liem return ret;
167796b3bb18SPeter Harliman Liem
167832d31c79SAmireddy Mallikarjuna reddy dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources;
167932d31c79SAmireddy Mallikarjuna reddy dma_dev->device_free_chan_resources = ldma_free_chan_resources;
168032d31c79SAmireddy Mallikarjuna reddy dma_dev->device_terminate_all = ldma_terminate_all;
168132d31c79SAmireddy Mallikarjuna reddy dma_dev->device_issue_pending = ldma_issue_pending;
168232d31c79SAmireddy Mallikarjuna reddy dma_dev->device_tx_status = ldma_tx_status;
168332d31c79SAmireddy Mallikarjuna reddy dma_dev->device_resume = ldma_resume_chan;
168432d31c79SAmireddy Mallikarjuna reddy dma_dev->device_pause = ldma_pause_chan;
168532d31c79SAmireddy Mallikarjuna reddy dma_dev->device_prep_slave_sg = ldma_prep_slave_sg;
168632d31c79SAmireddy Mallikarjuna reddy
168732d31c79SAmireddy Mallikarjuna reddy if (d->ver == DMA_VER22) {
168832d31c79SAmireddy Mallikarjuna reddy dma_dev->device_config = ldma_slave_config;
168932d31c79SAmireddy Mallikarjuna reddy dma_dev->device_synchronize = ldma_synchronize;
169032d31c79SAmireddy Mallikarjuna reddy dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
169132d31c79SAmireddy Mallikarjuna reddy dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
169232d31c79SAmireddy Mallikarjuna reddy dma_dev->directions = BIT(DMA_MEM_TO_DEV) |
169332d31c79SAmireddy Mallikarjuna reddy BIT(DMA_DEV_TO_MEM);
169432d31c79SAmireddy Mallikarjuna reddy dma_dev->residue_granularity =
169532d31c79SAmireddy Mallikarjuna reddy DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
169632d31c79SAmireddy Mallikarjuna reddy }
169732d31c79SAmireddy Mallikarjuna reddy
169832d31c79SAmireddy Mallikarjuna reddy platform_set_drvdata(pdev, d);
169932d31c79SAmireddy Mallikarjuna reddy
170032d31c79SAmireddy Mallikarjuna reddy ldma_dev_init(d);
170132d31c79SAmireddy Mallikarjuna reddy
170232d31c79SAmireddy Mallikarjuna reddy ret = dma_async_device_register(dma_dev);
170332d31c79SAmireddy Mallikarjuna reddy if (ret) {
170432d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "Failed to register slave DMA engine device\n");
170532d31c79SAmireddy Mallikarjuna reddy return ret;
170632d31c79SAmireddy Mallikarjuna reddy }
170732d31c79SAmireddy Mallikarjuna reddy
170832d31c79SAmireddy Mallikarjuna reddy ret = of_dma_controller_register(pdev->dev.of_node, ldma_xlate, d);
170932d31c79SAmireddy Mallikarjuna reddy if (ret) {
171032d31c79SAmireddy Mallikarjuna reddy dev_err(dev, "Failed to register of DMA controller\n");
171132d31c79SAmireddy Mallikarjuna reddy dma_async_device_unregister(dma_dev);
171232d31c79SAmireddy Mallikarjuna reddy return ret;
171332d31c79SAmireddy Mallikarjuna reddy }
171432d31c79SAmireddy Mallikarjuna reddy
171532d31c79SAmireddy Mallikarjuna reddy dev_info(dev, "Init done - rev: %x, ports: %d channels: %d\n", d->ver,
171632d31c79SAmireddy Mallikarjuna reddy d->port_nrs, d->chan_nrs);
171732d31c79SAmireddy Mallikarjuna reddy
171832d31c79SAmireddy Mallikarjuna reddy return 0;
171932d31c79SAmireddy Mallikarjuna reddy }
172032d31c79SAmireddy Mallikarjuna reddy
172132d31c79SAmireddy Mallikarjuna reddy static struct platform_driver intel_ldma_driver = {
172232d31c79SAmireddy Mallikarjuna reddy .probe = intel_ldma_probe,
172332d31c79SAmireddy Mallikarjuna reddy .driver = {
172432d31c79SAmireddy Mallikarjuna reddy .name = DRIVER_NAME,
172532d31c79SAmireddy Mallikarjuna reddy .of_match_table = intel_ldma_match,
172632d31c79SAmireddy Mallikarjuna reddy },
172732d31c79SAmireddy Mallikarjuna reddy };
172832d31c79SAmireddy Mallikarjuna reddy
172932d31c79SAmireddy Mallikarjuna reddy /*
173032d31c79SAmireddy Mallikarjuna reddy * Perform this driver as device_initcall to make sure initialization happens
173132d31c79SAmireddy Mallikarjuna reddy * before its DMA clients of some are platform specific and also to provide
173232d31c79SAmireddy Mallikarjuna reddy * registered DMA channels and DMA capabilities to clients before their
173332d31c79SAmireddy Mallikarjuna reddy * initialization.
173432d31c79SAmireddy Mallikarjuna reddy */
1735*8674ca39SLi Zetao builtin_platform_driver(intel_ldma_driver);
1736