1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * DMA driver for NVIDIA Tegra GPC DMA controller.
4  *
5  * Copyright (c) 2014-2022, NVIDIA CORPORATION.  All rights reserved.
6  */
7 
8 #include <linux/bitfield.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/iommu.h>
13 #include <linux/iopoll.h>
14 #include <linux/minmax.h>
15 #include <linux/module.h>
16 #include <linux/of_device.h>
17 #include <linux/of_dma.h>
18 #include <linux/platform_device.h>
19 #include <linux/reset.h>
20 #include <linux/slab.h>
21 #include <dt-bindings/memory/tegra186-mc.h>
22 #include "virt-dma.h"
23 
24 /* CSR register */
25 #define TEGRA_GPCDMA_CHAN_CSR			0x00
26 #define TEGRA_GPCDMA_CSR_ENB			BIT(31)
27 #define TEGRA_GPCDMA_CSR_IE_EOC			BIT(30)
28 #define TEGRA_GPCDMA_CSR_ONCE			BIT(27)
29 
30 #define TEGRA_GPCDMA_CSR_FC_MODE		GENMASK(25, 24)
31 #define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO	\
32 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0)
33 #define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO	\
34 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1)
35 #define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO	\
36 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2)
37 #define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO	\
38 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3)
39 
40 #define TEGRA_GPCDMA_CSR_DMA			GENMASK(23, 21)
41 #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC	\
42 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0)
43 #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC		\
44 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1)
45 #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC	\
46 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2)
47 #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC		\
48 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3)
49 #define TEGRA_GPCDMA_CSR_DMA_MEM2MEM		\
50 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4)
51 #define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT		\
52 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6)
53 
54 #define TEGRA_GPCDMA_CSR_REQ_SEL_MASK		GENMASK(20, 16)
55 #define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED		\
56 					FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4)
57 #define TEGRA_GPCDMA_CSR_IRQ_MASK		BIT(15)
58 #define TEGRA_GPCDMA_CSR_WEIGHT			GENMASK(13, 10)
59 
60 /* STATUS register */
61 #define TEGRA_GPCDMA_CHAN_STATUS		0x004
62 #define TEGRA_GPCDMA_STATUS_BUSY		BIT(31)
63 #define TEGRA_GPCDMA_STATUS_ISE_EOC		BIT(30)
64 #define TEGRA_GPCDMA_STATUS_PING_PONG		BIT(28)
65 #define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY	BIT(27)
66 #define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE	BIT(26)
67 #define TEGRA_GPCDMA_STATUS_CHANNEL_RX		BIT(25)
68 #define TEGRA_GPCDMA_STATUS_CHANNEL_TX		BIT(24)
69 #define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA	BIT(23)
70 #define TEGRA_GPCDMA_STATUS_IRQ_STA		BIT(21)
71 #define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA	BIT(20)
72 
73 #define TEGRA_GPCDMA_CHAN_CSRE			0x008
74 #define TEGRA_GPCDMA_CHAN_CSRE_PAUSE		BIT(31)
75 
76 /* Source address */
77 #define TEGRA_GPCDMA_CHAN_SRC_PTR		0x00C
78 
79 /* Destination address */
80 #define TEGRA_GPCDMA_CHAN_DST_PTR		0x010
81 
82 /* High address pointer */
83 #define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR		0x014
84 #define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR		GENMASK(7, 0)
85 #define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR		GENMASK(23, 16)
86 
87 /* MC sequence register */
88 #define TEGRA_GPCDMA_CHAN_MCSEQ			0x18
89 #define TEGRA_GPCDMA_MCSEQ_DATA_SWAP		BIT(31)
90 #define TEGRA_GPCDMA_MCSEQ_REQ_COUNT		GENMASK(30, 25)
91 #define TEGRA_GPCDMA_MCSEQ_BURST		GENMASK(24, 23)
92 #define TEGRA_GPCDMA_MCSEQ_BURST_2		\
93 		FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0)
94 #define TEGRA_GPCDMA_MCSEQ_BURST_16		\
95 		FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3)
96 #define TEGRA_GPCDMA_MCSEQ_WRAP1		GENMASK(22, 20)
97 #define TEGRA_GPCDMA_MCSEQ_WRAP0		GENMASK(19, 17)
98 #define TEGRA_GPCDMA_MCSEQ_WRAP_NONE		0
99 
100 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK	GENMASK(13, 7)
101 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK	GENMASK(6, 0)
102 
103 /* MMIO sequence register */
104 #define TEGRA_GPCDMA_CHAN_MMIOSEQ			0x01c
105 #define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF		BIT(31)
106 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH		GENMASK(30, 28)
107 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8	\
108 		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0)
109 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16	\
110 		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1)
111 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32	\
112 		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2)
113 #define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP		BIT(27)
114 #define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT	23
115 #define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN		2U
116 #define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX		32U
117 #define TEGRA_GPCDMA_MMIOSEQ_BURST(bs)	\
118 		(GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT)
119 #define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID		GENMASK(22, 19)
120 #define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD		GENMASK(18, 16)
121 #define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT		GENMASK(8, 7)
122 
123 /* Channel WCOUNT */
124 #define TEGRA_GPCDMA_CHAN_WCOUNT		0x20
125 
126 /* Transfer count */
127 #define TEGRA_GPCDMA_CHAN_XFER_COUNT		0x24
128 
129 /* DMA byte count status */
130 #define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS	0x28
131 
132 /* Error Status Register */
133 #define TEGRA_GPCDMA_CHAN_ERR_STATUS		0x30
134 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT	8
135 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK	0xF
136 #define TEGRA_GPCDMA_CHAN_ERR_TYPE(err)	(			\
137 		((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) &	\
138 		TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK)
139 #define TEGRA_DMA_BM_FIFO_FULL_ERR		0xF
140 #define TEGRA_DMA_PERIPH_FIFO_FULL_ERR		0xE
141 #define TEGRA_DMA_PERIPH_ID_ERR			0xD
142 #define TEGRA_DMA_STREAM_ID_ERR			0xC
143 #define TEGRA_DMA_MC_SLAVE_ERR			0xB
144 #define TEGRA_DMA_MMIO_SLAVE_ERR		0xA
145 
146 /* Fixed Pattern */
147 #define TEGRA_GPCDMA_CHAN_FIXED_PATTERN		0x34
148 
149 #define TEGRA_GPCDMA_CHAN_TZ			0x38
150 #define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1	BIT(0)
151 #define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1		BIT(1)
152 
153 #define TEGRA_GPCDMA_CHAN_SPARE			0x3c
154 #define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC	BIT(16)
155 
156 /*
157  * If any burst is in flight and DMA paused then this is the time to complete
158  * on-flight burst and update DMA status register.
159  */
160 #define TEGRA_GPCDMA_BURST_COMPLETE_TIME	10
161 #define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT	5000 /* 5 msec */
162 
163 /* Channel base address offset from GPCDMA base address */
164 #define TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET	0x20000
165 
166 struct tegra_dma;
167 struct tegra_dma_channel;
168 
169 /*
170  * tegra_dma_chip_data Tegra chip specific DMA data
171  * @nr_channels: Number of channels available in the controller.
172  * @channel_reg_size: Channel register size.
173  * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
174  * @hw_support_pause: DMA HW engine support pause of the channel.
175  */
176 struct tegra_dma_chip_data {
177 	bool hw_support_pause;
178 	unsigned int nr_channels;
179 	unsigned int channel_reg_size;
180 	unsigned int max_dma_count;
181 	int (*terminate)(struct tegra_dma_channel *tdc);
182 };
183 
184 /* DMA channel registers */
185 struct tegra_dma_channel_regs {
186 	u32 csr;
187 	u32 src_ptr;
188 	u32 dst_ptr;
189 	u32 high_addr_ptr;
190 	u32 mc_seq;
191 	u32 mmio_seq;
192 	u32 wcount;
193 	u32 fixed_pattern;
194 };
195 
196 /*
197  * tegra_dma_sg_req: DMA request details to configure hardware. This
198  * contains the details for one transfer to configure DMA hw.
199  * The client's request for data transfer can be broken into multiple
200  * sub-transfer as per requester details and hw support. This sub transfer
201  * get added as an array in Tegra DMA desc which manages the transfer details.
202  */
203 struct tegra_dma_sg_req {
204 	unsigned int len;
205 	struct tegra_dma_channel_regs ch_regs;
206 };
207 
208 /*
209  * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to
210  * manage client request and keep track of transfer status, callbacks
211  * and request counts etc.
212  */
213 struct tegra_dma_desc {
214 	bool cyclic;
215 	unsigned int bytes_req;
216 	unsigned int bytes_xfer;
217 	unsigned int sg_idx;
218 	unsigned int sg_count;
219 	struct virt_dma_desc vd;
220 	struct tegra_dma_channel *tdc;
221 	struct tegra_dma_sg_req sg_req[];
222 };
223 
224 /*
225  * tegra_dma_channel: Channel specific information
226  */
227 struct tegra_dma_channel {
228 	bool config_init;
229 	char name[30];
230 	enum dma_transfer_direction sid_dir;
231 	int id;
232 	int irq;
233 	int slave_id;
234 	struct tegra_dma *tdma;
235 	struct virt_dma_chan vc;
236 	struct tegra_dma_desc *dma_desc;
237 	struct dma_slave_config dma_sconfig;
238 	unsigned int stream_id;
239 	unsigned long chan_base_offset;
240 };
241 
242 /*
243  * tegra_dma: Tegra DMA specific information
244  */
245 struct tegra_dma {
246 	const struct tegra_dma_chip_data *chip_data;
247 	unsigned long sid_m2d_reserved;
248 	unsigned long sid_d2m_reserved;
249 	void __iomem *base_addr;
250 	struct device *dev;
251 	struct dma_device dma_dev;
252 	struct reset_control *rst;
253 	struct tegra_dma_channel channels[];
254 };
255 
256 static inline void tdc_write(struct tegra_dma_channel *tdc,
257 			     u32 reg, u32 val)
258 {
259 	writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);
260 }
261 
262 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
263 {
264 	return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg);
265 }
266 
267 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
268 {
269 	return container_of(dc, struct tegra_dma_channel, vc.chan);
270 }
271 
272 static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd)
273 {
274 	return container_of(vd, struct tegra_dma_desc, vd);
275 }
276 
277 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
278 {
279 	return tdc->vc.chan.device->dev;
280 }
281 
282 static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc)
283 {
284 	dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n",
285 		tdc->id, tdc->name);
286 	dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n",
287 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR),
288 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS),
289 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE),
290 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR),
291 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR)
292 	);
293 	dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n",
294 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ),
295 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ),
296 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT),
297 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT),
298 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS)
299 	);
300 	dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n",
301 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS));
302 }
303 
304 static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc,
305 				 enum dma_transfer_direction direction)
306 {
307 	struct tegra_dma *tdma = tdc->tdma;
308 	int sid = tdc->slave_id;
309 
310 	if (!is_slave_direction(direction))
311 		return 0;
312 
313 	switch (direction) {
314 	case DMA_MEM_TO_DEV:
315 		if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) {
316 			dev_err(tdma->dev, "slave id already in use\n");
317 			return -EINVAL;
318 		}
319 		break;
320 	case DMA_DEV_TO_MEM:
321 		if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) {
322 			dev_err(tdma->dev, "slave id already in use\n");
323 			return -EINVAL;
324 		}
325 		break;
326 	default:
327 		break;
328 	}
329 
330 	tdc->sid_dir = direction;
331 
332 	return 0;
333 }
334 
335 static void tegra_dma_sid_free(struct tegra_dma_channel *tdc)
336 {
337 	struct tegra_dma *tdma = tdc->tdma;
338 	int sid = tdc->slave_id;
339 
340 	switch (tdc->sid_dir) {
341 	case DMA_MEM_TO_DEV:
342 		clear_bit(sid,  &tdma->sid_m2d_reserved);
343 		break;
344 	case DMA_DEV_TO_MEM:
345 		clear_bit(sid,  &tdma->sid_d2m_reserved);
346 		break;
347 	default:
348 		break;
349 	}
350 
351 	tdc->sid_dir = DMA_TRANS_NONE;
352 }
353 
354 static void tegra_dma_desc_free(struct virt_dma_desc *vd)
355 {
356 	kfree(container_of(vd, struct tegra_dma_desc, vd));
357 }
358 
359 static int tegra_dma_slave_config(struct dma_chan *dc,
360 				  struct dma_slave_config *sconfig)
361 {
362 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
363 
364 	memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
365 	tdc->config_init = true;
366 
367 	return 0;
368 }
369 
370 static int tegra_dma_pause(struct tegra_dma_channel *tdc)
371 {
372 	int ret;
373 	u32 val;
374 
375 	val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
376 	val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
377 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
378 
379 	/* Wait until busy bit is de-asserted */
380 	ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
381 			tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
382 			val,
383 			!(val & TEGRA_GPCDMA_STATUS_BUSY),
384 			TEGRA_GPCDMA_BURST_COMPLETE_TIME,
385 			TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
386 
387 	if (ret) {
388 		dev_err(tdc2dev(tdc), "DMA pause timed out\n");
389 		tegra_dma_dump_chan_regs(tdc);
390 	}
391 
392 	return ret;
393 }
394 
395 static int tegra_dma_device_pause(struct dma_chan *dc)
396 {
397 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
398 	unsigned long flags;
399 	int ret;
400 
401 	if (!tdc->tdma->chip_data->hw_support_pause)
402 		return -ENOSYS;
403 
404 	spin_lock_irqsave(&tdc->vc.lock, flags);
405 	ret = tegra_dma_pause(tdc);
406 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
407 
408 	return ret;
409 }
410 
411 static void tegra_dma_resume(struct tegra_dma_channel *tdc)
412 {
413 	u32 val;
414 
415 	val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
416 	val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
417 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
418 }
419 
420 static int tegra_dma_device_resume(struct dma_chan *dc)
421 {
422 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
423 	unsigned long flags;
424 
425 	if (!tdc->tdma->chip_data->hw_support_pause)
426 		return -ENOSYS;
427 
428 	spin_lock_irqsave(&tdc->vc.lock, flags);
429 	tegra_dma_resume(tdc);
430 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
431 
432 	return 0;
433 }
434 
435 static inline int tegra_dma_pause_noerr(struct tegra_dma_channel *tdc)
436 {
437 	/* Return 0 irrespective of PAUSE status.
438 	 * This is useful to recover channels that can exit out of flush
439 	 * state when the channel is disabled.
440 	 */
441 
442 	tegra_dma_pause(tdc);
443 	return 0;
444 }
445 
446 static void tegra_dma_disable(struct tegra_dma_channel *tdc)
447 {
448 	u32 csr, status;
449 
450 	csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
451 
452 	/* Disable interrupts */
453 	csr &= ~TEGRA_GPCDMA_CSR_IE_EOC;
454 
455 	/* Disable DMA */
456 	csr &= ~TEGRA_GPCDMA_CSR_ENB;
457 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
458 
459 	/* Clear interrupt status if it is there */
460 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
461 	if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) {
462 		dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
463 		tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status);
464 	}
465 }
466 
467 static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)
468 {
469 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
470 	struct tegra_dma_channel_regs *ch_regs;
471 	int ret;
472 	u32 val;
473 
474 	dma_desc->sg_idx++;
475 
476 	/* Reset the sg index for cyclic transfers */
477 	if (dma_desc->sg_idx == dma_desc->sg_count)
478 		dma_desc->sg_idx = 0;
479 
480 	/* Configure next transfer immediately after DMA is busy */
481 	ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
482 			tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
483 			val,
484 			(val & TEGRA_GPCDMA_STATUS_BUSY), 0,
485 			TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
486 	if (ret)
487 		return;
488 
489 	ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
490 
491 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
492 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
493 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
494 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
495 
496 	/* Start DMA */
497 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
498 		  ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
499 }
500 
501 static void tegra_dma_start(struct tegra_dma_channel *tdc)
502 {
503 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
504 	struct tegra_dma_channel_regs *ch_regs;
505 	struct virt_dma_desc *vdesc;
506 
507 	if (!dma_desc) {
508 		vdesc = vchan_next_desc(&tdc->vc);
509 		if (!vdesc)
510 			return;
511 
512 		dma_desc = vd_to_tegra_dma_desc(vdesc);
513 		list_del(&vdesc->node);
514 		dma_desc->tdc = tdc;
515 		tdc->dma_desc = dma_desc;
516 
517 		tegra_dma_resume(tdc);
518 	}
519 
520 	ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
521 
522 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
523 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0);
524 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
525 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
526 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
527 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern);
528 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq);
529 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq);
530 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr);
531 
532 	/* Start DMA */
533 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
534 		  ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
535 }
536 
537 static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc)
538 {
539 	vchan_cookie_complete(&tdc->dma_desc->vd);
540 
541 	tegra_dma_sid_free(tdc);
542 	tdc->dma_desc = NULL;
543 }
544 
545 static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc,
546 					unsigned int err_status)
547 {
548 	switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) {
549 	case TEGRA_DMA_BM_FIFO_FULL_ERR:
550 		dev_err(tdc->tdma->dev,
551 			"GPCDMA CH%d bm fifo full\n", tdc->id);
552 		break;
553 
554 	case TEGRA_DMA_PERIPH_FIFO_FULL_ERR:
555 		dev_err(tdc->tdma->dev,
556 			"GPCDMA CH%d peripheral fifo full\n", tdc->id);
557 		break;
558 
559 	case TEGRA_DMA_PERIPH_ID_ERR:
560 		dev_err(tdc->tdma->dev,
561 			"GPCDMA CH%d illegal peripheral id\n", tdc->id);
562 		break;
563 
564 	case TEGRA_DMA_STREAM_ID_ERR:
565 		dev_err(tdc->tdma->dev,
566 			"GPCDMA CH%d illegal stream id\n", tdc->id);
567 		break;
568 
569 	case TEGRA_DMA_MC_SLAVE_ERR:
570 		dev_err(tdc->tdma->dev,
571 			"GPCDMA CH%d mc slave error\n", tdc->id);
572 		break;
573 
574 	case TEGRA_DMA_MMIO_SLAVE_ERR:
575 		dev_err(tdc->tdma->dev,
576 			"GPCDMA CH%d mmio slave error\n", tdc->id);
577 		break;
578 
579 	default:
580 		dev_err(tdc->tdma->dev,
581 			"GPCDMA CH%d security violation %x\n", tdc->id,
582 			err_status);
583 	}
584 }
585 
586 static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
587 {
588 	struct tegra_dma_channel *tdc = dev_id;
589 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
590 	struct tegra_dma_sg_req *sg_req;
591 	u32 status;
592 
593 	/* Check channel error status register */
594 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS);
595 	if (status) {
596 		tegra_dma_chan_decode_error(tdc, status);
597 		tegra_dma_dump_chan_regs(tdc);
598 		tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF);
599 	}
600 
601 	spin_lock(&tdc->vc.lock);
602 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
603 	if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC))
604 		goto irq_done;
605 
606 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS,
607 		  TEGRA_GPCDMA_STATUS_ISE_EOC);
608 
609 	if (!dma_desc)
610 		goto irq_done;
611 
612 	sg_req = dma_desc->sg_req;
613 	dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len;
614 
615 	if (dma_desc->cyclic) {
616 		vchan_cyclic_callback(&dma_desc->vd);
617 		tegra_dma_configure_next_sg(tdc);
618 	} else {
619 		dma_desc->sg_idx++;
620 		if (dma_desc->sg_idx == dma_desc->sg_count)
621 			tegra_dma_xfer_complete(tdc);
622 		else
623 			tegra_dma_start(tdc);
624 	}
625 
626 irq_done:
627 	spin_unlock(&tdc->vc.lock);
628 	return IRQ_HANDLED;
629 }
630 
631 static void tegra_dma_issue_pending(struct dma_chan *dc)
632 {
633 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
634 	unsigned long flags;
635 
636 	if (tdc->dma_desc)
637 		return;
638 
639 	spin_lock_irqsave(&tdc->vc.lock, flags);
640 	if (vchan_issue_pending(&tdc->vc))
641 		tegra_dma_start(tdc);
642 
643 	/*
644 	 * For cyclic DMA transfers, program the second
645 	 * transfer parameters as soon as the first DMA
646 	 * transfer is started inorder for the DMA
647 	 * controller to trigger the second transfer
648 	 * with the correct parameters.
649 	 */
650 	if (tdc->dma_desc && tdc->dma_desc->cyclic)
651 		tegra_dma_configure_next_sg(tdc);
652 
653 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
654 }
655 
656 static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)
657 {
658 	int ret;
659 	u32 status, csr;
660 
661 	/*
662 	 * Change the client associated with the DMA channel
663 	 * to stop DMA engine from starting any more bursts for
664 	 * the given client and wait for in flight bursts to complete
665 	 */
666 	csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
667 	csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK);
668 	csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED;
669 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
670 
671 	/* Wait for in flight data transfer to finish */
672 	udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME);
673 
674 	/* If TX/RX path is still active wait till it becomes
675 	 * inactive
676 	 */
677 
678 	ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
679 				tdc->chan_base_offset +
680 				TEGRA_GPCDMA_CHAN_STATUS,
681 				status,
682 				!(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX |
683 				TEGRA_GPCDMA_STATUS_CHANNEL_RX)),
684 				5,
685 				TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
686 	if (ret) {
687 		dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n");
688 		tegra_dma_dump_chan_regs(tdc);
689 	}
690 
691 	return ret;
692 }
693 
694 static int tegra_dma_terminate_all(struct dma_chan *dc)
695 {
696 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
697 	unsigned long flags;
698 	LIST_HEAD(head);
699 	int err;
700 
701 	spin_lock_irqsave(&tdc->vc.lock, flags);
702 
703 	if (tdc->dma_desc) {
704 		err = tdc->tdma->chip_data->terminate(tdc);
705 		if (err) {
706 			spin_unlock_irqrestore(&tdc->vc.lock, flags);
707 			return err;
708 		}
709 
710 		tegra_dma_disable(tdc);
711 		tdc->dma_desc = NULL;
712 	}
713 
714 	tegra_dma_sid_free(tdc);
715 	vchan_get_all_descriptors(&tdc->vc, &head);
716 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
717 
718 	vchan_dma_desc_free_list(&tdc->vc, &head);
719 
720 	return 0;
721 }
722 
723 static int tegra_dma_get_residual(struct tegra_dma_channel *tdc)
724 {
725 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
726 	struct tegra_dma_sg_req *sg_req = dma_desc->sg_req;
727 	unsigned int bytes_xfer, residual;
728 	u32 wcount = 0, status;
729 
730 	wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT);
731 
732 	/*
733 	 * Set wcount = 0 if EOC bit is set. The transfer would have
734 	 * already completed and the CHAN_XFER_COUNT could have updated
735 	 * for the next transfer, specifically in case of cyclic transfers.
736 	 */
737 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
738 	if (status & TEGRA_GPCDMA_STATUS_ISE_EOC)
739 		wcount = 0;
740 
741 	bytes_xfer = dma_desc->bytes_xfer +
742 		     sg_req[dma_desc->sg_idx].len - (wcount * 4);
743 
744 	residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req);
745 
746 	return residual;
747 }
748 
749 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
750 					   dma_cookie_t cookie,
751 					   struct dma_tx_state *txstate)
752 {
753 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
754 	struct tegra_dma_desc *dma_desc;
755 	struct virt_dma_desc *vd;
756 	unsigned int residual;
757 	unsigned long flags;
758 	enum dma_status ret;
759 
760 	ret = dma_cookie_status(dc, cookie, txstate);
761 	if (ret == DMA_COMPLETE)
762 		return ret;
763 
764 	spin_lock_irqsave(&tdc->vc.lock, flags);
765 	vd = vchan_find_desc(&tdc->vc, cookie);
766 	if (vd) {
767 		dma_desc = vd_to_tegra_dma_desc(vd);
768 		residual = dma_desc->bytes_req;
769 		dma_set_residue(txstate, residual);
770 	} else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) {
771 		residual =  tegra_dma_get_residual(tdc);
772 		dma_set_residue(txstate, residual);
773 	} else {
774 		dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie);
775 	}
776 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
777 
778 	return ret;
779 }
780 
781 static inline int get_bus_width(struct tegra_dma_channel *tdc,
782 				enum dma_slave_buswidth slave_bw)
783 {
784 	switch (slave_bw) {
785 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
786 		return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8;
787 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
788 		return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16;
789 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
790 		return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32;
791 	default:
792 		dev_err(tdc2dev(tdc), "given slave bus width is not supported\n");
793 		return -EINVAL;
794 	}
795 }
796 
797 static unsigned int get_burst_size(struct tegra_dma_channel *tdc,
798 				   u32 burst_size, enum dma_slave_buswidth slave_bw,
799 				   int len)
800 {
801 	unsigned int burst_mmio_width, burst_byte;
802 
803 	/*
804 	 * burst_size from client is in terms of the bus_width.
805 	 * convert that into words.
806 	 * If burst_size is not specified from client, then use
807 	 * len to calculate the optimum burst size
808 	 */
809 	burst_byte = burst_size ? burst_size * slave_bw : len;
810 	burst_mmio_width = burst_byte / 4;
811 
812 	if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN)
813 		return 0;
814 
815 	burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX);
816 
817 	return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width);
818 }
819 
820 static int get_transfer_param(struct tegra_dma_channel *tdc,
821 			      enum dma_transfer_direction direction,
822 			      u32 *apb_addr,
823 			      u32 *mmio_seq,
824 			      u32 *csr,
825 			      unsigned int *burst_size,
826 			      enum dma_slave_buswidth *slave_bw)
827 {
828 	switch (direction) {
829 	case DMA_MEM_TO_DEV:
830 		*apb_addr = tdc->dma_sconfig.dst_addr;
831 		*mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
832 		*burst_size = tdc->dma_sconfig.dst_maxburst;
833 		*slave_bw = tdc->dma_sconfig.dst_addr_width;
834 		*csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC;
835 		return 0;
836 	case DMA_DEV_TO_MEM:
837 		*apb_addr = tdc->dma_sconfig.src_addr;
838 		*mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
839 		*burst_size = tdc->dma_sconfig.src_maxburst;
840 		*slave_bw = tdc->dma_sconfig.src_addr_width;
841 		*csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC;
842 		return 0;
843 	default:
844 		dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
845 	}
846 
847 	return -EINVAL;
848 }
849 
850 static struct dma_async_tx_descriptor *
851 tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,
852 			  size_t len, unsigned long flags)
853 {
854 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
855 	unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
856 	struct tegra_dma_sg_req *sg_req;
857 	struct tegra_dma_desc *dma_desc;
858 	u32 csr, mc_seq;
859 
860 	if ((len & 3) || (dest & 3) || len > max_dma_count) {
861 		dev_err(tdc2dev(tdc),
862 			"DMA length/memory address is not supported\n");
863 		return NULL;
864 	}
865 
866 	/* Set DMA mode to fixed pattern */
867 	csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT;
868 	/* Enable once or continuous mode */
869 	csr |= TEGRA_GPCDMA_CSR_ONCE;
870 	/* Enable IRQ mask */
871 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
872 	/* Enable the DMA interrupt */
873 	if (flags & DMA_PREP_INTERRUPT)
874 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
875 	/* Configure default priority weight for the channel */
876 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
877 
878 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
879 	/* retain stream-id and clean rest */
880 	mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
881 
882 	/* Set the address wrapping */
883 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
884 						TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
885 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
886 						TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
887 
888 	/* Program outstanding MC requests */
889 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
890 	/* Set burst size */
891 	mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
892 
893 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
894 	if (!dma_desc)
895 		return NULL;
896 
897 	dma_desc->bytes_req = len;
898 	dma_desc->sg_count = 1;
899 	sg_req = dma_desc->sg_req;
900 
901 	sg_req[0].ch_regs.src_ptr = 0;
902 	sg_req[0].ch_regs.dst_ptr = dest;
903 	sg_req[0].ch_regs.high_addr_ptr =
904 			FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
905 	sg_req[0].ch_regs.fixed_pattern = value;
906 	/* Word count reg takes value as (N +1) words */
907 	sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
908 	sg_req[0].ch_regs.csr = csr;
909 	sg_req[0].ch_regs.mmio_seq = 0;
910 	sg_req[0].ch_regs.mc_seq = mc_seq;
911 	sg_req[0].len = len;
912 
913 	dma_desc->cyclic = false;
914 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
915 }
916 
917 static struct dma_async_tx_descriptor *
918 tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,
919 			  dma_addr_t src, size_t len, unsigned long flags)
920 {
921 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
922 	struct tegra_dma_sg_req *sg_req;
923 	struct tegra_dma_desc *dma_desc;
924 	unsigned int max_dma_count;
925 	u32 csr, mc_seq;
926 
927 	max_dma_count = tdc->tdma->chip_data->max_dma_count;
928 	if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) {
929 		dev_err(tdc2dev(tdc),
930 			"DMA length/memory address is not supported\n");
931 		return NULL;
932 	}
933 
934 	/* Set DMA mode to memory to memory transfer */
935 	csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM;
936 	/* Enable once or continuous mode */
937 	csr |= TEGRA_GPCDMA_CSR_ONCE;
938 	/* Enable IRQ mask */
939 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
940 	/* Enable the DMA interrupt */
941 	if (flags & DMA_PREP_INTERRUPT)
942 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
943 	/* Configure default priority weight for the channel */
944 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
945 
946 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
947 	/* retain stream-id and clean rest */
948 	mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) |
949 		  (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
950 
951 	/* Set the address wrapping */
952 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
953 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
954 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
955 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
956 
957 	/* Program outstanding MC requests */
958 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
959 	/* Set burst size */
960 	mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
961 
962 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
963 	if (!dma_desc)
964 		return NULL;
965 
966 	dma_desc->bytes_req = len;
967 	dma_desc->sg_count = 1;
968 	sg_req = dma_desc->sg_req;
969 
970 	sg_req[0].ch_regs.src_ptr = src;
971 	sg_req[0].ch_regs.dst_ptr = dest;
972 	sg_req[0].ch_regs.high_addr_ptr =
973 		FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32));
974 	sg_req[0].ch_regs.high_addr_ptr |=
975 		FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
976 	/* Word count reg takes value as (N +1) words */
977 	sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
978 	sg_req[0].ch_regs.csr = csr;
979 	sg_req[0].ch_regs.mmio_seq = 0;
980 	sg_req[0].ch_regs.mc_seq = mc_seq;
981 	sg_req[0].len = len;
982 
983 	dma_desc->cyclic = false;
984 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
985 }
986 
987 static struct dma_async_tx_descriptor *
988 tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
989 			unsigned int sg_len, enum dma_transfer_direction direction,
990 			unsigned long flags, void *context)
991 {
992 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
993 	unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
994 	enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
995 	u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0;
996 	struct tegra_dma_sg_req *sg_req;
997 	struct tegra_dma_desc *dma_desc;
998 	struct scatterlist *sg;
999 	u32 burst_size;
1000 	unsigned int i;
1001 	int ret;
1002 
1003 	if (!tdc->config_init) {
1004 		dev_err(tdc2dev(tdc), "DMA channel is not configured\n");
1005 		return NULL;
1006 	}
1007 	if (sg_len < 1) {
1008 		dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
1009 		return NULL;
1010 	}
1011 
1012 	ret = tegra_dma_sid_reserve(tdc, direction);
1013 	if (ret)
1014 		return NULL;
1015 
1016 	ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1017 				 &burst_size, &slave_bw);
1018 	if (ret < 0)
1019 		return NULL;
1020 
1021 	/* Enable once or continuous mode */
1022 	csr |= TEGRA_GPCDMA_CSR_ONCE;
1023 	/* Program the slave id in requestor select */
1024 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1025 	/* Enable IRQ mask */
1026 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1027 	/* Configure default priority weight for the channel*/
1028 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1029 
1030 	/* Enable the DMA interrupt */
1031 	if (flags & DMA_PREP_INTERRUPT)
1032 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1033 
1034 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1035 	/* retain stream-id and clean rest */
1036 	mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1037 
1038 	/* Set the address wrapping on both MC and MMIO side */
1039 
1040 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1041 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1042 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1043 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1044 	mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1045 
1046 	/* Program 2 MC outstanding requests by default. */
1047 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1048 
1049 	/* Setting MC burst size depending on MMIO burst size */
1050 	if (burst_size == 64)
1051 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1052 	else
1053 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1054 
1055 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, sg_len), GFP_NOWAIT);
1056 	if (!dma_desc)
1057 		return NULL;
1058 
1059 	dma_desc->sg_count = sg_len;
1060 	sg_req = dma_desc->sg_req;
1061 
1062 	/* Make transfer requests */
1063 	for_each_sg(sgl, sg, sg_len, i) {
1064 		u32 len;
1065 		dma_addr_t mem;
1066 
1067 		mem = sg_dma_address(sg);
1068 		len = sg_dma_len(sg);
1069 
1070 		if ((len & 3) || (mem & 3) || len > max_dma_count) {
1071 			dev_err(tdc2dev(tdc),
1072 				"DMA length/memory address is not supported\n");
1073 			kfree(dma_desc);
1074 			return NULL;
1075 		}
1076 
1077 		mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1078 		dma_desc->bytes_req += len;
1079 
1080 		if (direction == DMA_MEM_TO_DEV) {
1081 			sg_req[i].ch_regs.src_ptr = mem;
1082 			sg_req[i].ch_regs.dst_ptr = apb_ptr;
1083 			sg_req[i].ch_regs.high_addr_ptr =
1084 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1085 		} else if (direction == DMA_DEV_TO_MEM) {
1086 			sg_req[i].ch_regs.src_ptr = apb_ptr;
1087 			sg_req[i].ch_regs.dst_ptr = mem;
1088 			sg_req[i].ch_regs.high_addr_ptr =
1089 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1090 		}
1091 
1092 		/*
1093 		 * Word count register takes input in words. Writing a value
1094 		 * of N into word count register means a req of (N+1) words.
1095 		 */
1096 		sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1097 		sg_req[i].ch_regs.csr = csr;
1098 		sg_req[i].ch_regs.mmio_seq = mmio_seq;
1099 		sg_req[i].ch_regs.mc_seq = mc_seq;
1100 		sg_req[i].len = len;
1101 	}
1102 
1103 	dma_desc->cyclic = false;
1104 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1105 }
1106 
1107 static struct dma_async_tx_descriptor *
1108 tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1109 			  size_t period_len, enum dma_transfer_direction direction,
1110 			  unsigned long flags)
1111 {
1112 	enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
1113 	u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size;
1114 	unsigned int max_dma_count, len, period_count, i;
1115 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1116 	struct tegra_dma_desc *dma_desc;
1117 	struct tegra_dma_sg_req *sg_req;
1118 	dma_addr_t mem = buf_addr;
1119 	int ret;
1120 
1121 	if (!buf_len || !period_len) {
1122 		dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
1123 		return NULL;
1124 	}
1125 
1126 	if (!tdc->config_init) {
1127 		dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
1128 		return NULL;
1129 	}
1130 
1131 	ret = tegra_dma_sid_reserve(tdc, direction);
1132 	if (ret)
1133 		return NULL;
1134 
1135 	/*
1136 	 * We only support cycle transfer when buf_len is multiple of
1137 	 * period_len.
1138 	 */
1139 	if (buf_len % period_len) {
1140 		dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
1141 		return NULL;
1142 	}
1143 
1144 	len = period_len;
1145 	max_dma_count = tdc->tdma->chip_data->max_dma_count;
1146 	if ((len & 3) || (buf_addr & 3) || len > max_dma_count) {
1147 		dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1148 		return NULL;
1149 	}
1150 
1151 	ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1152 				 &burst_size, &slave_bw);
1153 	if (ret < 0)
1154 		return NULL;
1155 
1156 	/* Enable once or continuous mode */
1157 	csr &= ~TEGRA_GPCDMA_CSR_ONCE;
1158 	/* Program the slave id in requestor select */
1159 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1160 	/* Enable IRQ mask */
1161 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1162 	/* Configure default priority weight for the channel*/
1163 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1164 
1165 	/* Enable the DMA interrupt */
1166 	if (flags & DMA_PREP_INTERRUPT)
1167 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1168 
1169 	mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1170 
1171 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1172 	/* retain stream-id and clean rest */
1173 	mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1174 
1175 	/* Set the address wrapping on both MC and MMIO side */
1176 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1177 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1178 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1179 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1180 
1181 	/* Program 2 MC outstanding requests by default. */
1182 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1183 	/* Setting MC burst size depending on MMIO burst size */
1184 	if (burst_size == 64)
1185 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1186 	else
1187 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1188 
1189 	period_count = buf_len / period_len;
1190 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, period_count),
1191 			   GFP_NOWAIT);
1192 	if (!dma_desc)
1193 		return NULL;
1194 
1195 	dma_desc->bytes_req = buf_len;
1196 	dma_desc->sg_count = period_count;
1197 	sg_req = dma_desc->sg_req;
1198 
1199 	/* Split transfer equal to period size */
1200 	for (i = 0; i < period_count; i++) {
1201 		mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1202 		if (direction == DMA_MEM_TO_DEV) {
1203 			sg_req[i].ch_regs.src_ptr = mem;
1204 			sg_req[i].ch_regs.dst_ptr = apb_ptr;
1205 			sg_req[i].ch_regs.high_addr_ptr =
1206 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1207 		} else if (direction == DMA_DEV_TO_MEM) {
1208 			sg_req[i].ch_regs.src_ptr = apb_ptr;
1209 			sg_req[i].ch_regs.dst_ptr = mem;
1210 			sg_req[i].ch_regs.high_addr_ptr =
1211 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1212 		}
1213 		/*
1214 		 * Word count register takes input in words. Writing a value
1215 		 * of N into word count register means a req of (N+1) words.
1216 		 */
1217 		sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1218 		sg_req[i].ch_regs.csr = csr;
1219 		sg_req[i].ch_regs.mmio_seq = mmio_seq;
1220 		sg_req[i].ch_regs.mc_seq = mc_seq;
1221 		sg_req[i].len = len;
1222 
1223 		mem += len;
1224 	}
1225 
1226 	dma_desc->cyclic = true;
1227 
1228 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1229 }
1230 
1231 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1232 {
1233 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1234 	int ret;
1235 
1236 	ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);
1237 	if (ret) {
1238 		dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name);
1239 		return ret;
1240 	}
1241 
1242 	dma_cookie_init(&tdc->vc.chan);
1243 	tdc->config_init = false;
1244 	return 0;
1245 }
1246 
1247 static void tegra_dma_chan_synchronize(struct dma_chan *dc)
1248 {
1249 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1250 
1251 	synchronize_irq(tdc->irq);
1252 	vchan_synchronize(&tdc->vc);
1253 }
1254 
1255 static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1256 {
1257 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1258 
1259 	dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1260 
1261 	tegra_dma_terminate_all(dc);
1262 	synchronize_irq(tdc->irq);
1263 
1264 	tasklet_kill(&tdc->vc.task);
1265 	tdc->config_init = false;
1266 	tdc->slave_id = -1;
1267 	tdc->sid_dir = DMA_TRANS_NONE;
1268 	free_irq(tdc->irq, tdc);
1269 
1270 	vchan_free_chan_resources(&tdc->vc);
1271 }
1272 
1273 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
1274 					   struct of_dma *ofdma)
1275 {
1276 	struct tegra_dma *tdma = ofdma->of_dma_data;
1277 	struct tegra_dma_channel *tdc;
1278 	struct dma_chan *chan;
1279 
1280 	chan = dma_get_any_slave_channel(&tdma->dma_dev);
1281 	if (!chan)
1282 		return NULL;
1283 
1284 	tdc = to_tegra_dma_chan(chan);
1285 	tdc->slave_id = dma_spec->args[0];
1286 
1287 	return chan;
1288 }
1289 
1290 static const struct tegra_dma_chip_data tegra186_dma_chip_data = {
1291 	.nr_channels = 31,
1292 	.channel_reg_size = SZ_64K,
1293 	.max_dma_count = SZ_1G,
1294 	.hw_support_pause = false,
1295 	.terminate = tegra_dma_stop_client,
1296 };
1297 
1298 static const struct tegra_dma_chip_data tegra194_dma_chip_data = {
1299 	.nr_channels = 31,
1300 	.channel_reg_size = SZ_64K,
1301 	.max_dma_count = SZ_1G,
1302 	.hw_support_pause = true,
1303 	.terminate = tegra_dma_pause,
1304 };
1305 
1306 static const struct tegra_dma_chip_data tegra234_dma_chip_data = {
1307 	.nr_channels = 31,
1308 	.channel_reg_size = SZ_64K,
1309 	.max_dma_count = SZ_1G,
1310 	.hw_support_pause = true,
1311 	.terminate = tegra_dma_pause_noerr,
1312 };
1313 
1314 static const struct of_device_id tegra_dma_of_match[] = {
1315 	{
1316 		.compatible = "nvidia,tegra186-gpcdma",
1317 		.data = &tegra186_dma_chip_data,
1318 	}, {
1319 		.compatible = "nvidia,tegra194-gpcdma",
1320 		.data = &tegra194_dma_chip_data,
1321 	}, {
1322 		.compatible = "nvidia,tegra234-gpcdma",
1323 		.data = &tegra234_dma_chip_data,
1324 	}, {
1325 	},
1326 };
1327 MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
1328 
1329 static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)
1330 {
1331 	unsigned int reg_val =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1332 
1333 	reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK);
1334 	reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
1335 
1336 	reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id);
1337 	reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id);
1338 
1339 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val);
1340 	return 0;
1341 }
1342 
1343 static int tegra_dma_probe(struct platform_device *pdev)
1344 {
1345 	const struct tegra_dma_chip_data *cdata = NULL;
1346 	struct iommu_fwspec *iommu_spec;
1347 	unsigned int stream_id, i;
1348 	struct tegra_dma *tdma;
1349 	int ret;
1350 
1351 	cdata = of_device_get_match_data(&pdev->dev);
1352 
1353 	tdma = devm_kzalloc(&pdev->dev,
1354 			    struct_size(tdma, channels, cdata->nr_channels),
1355 			    GFP_KERNEL);
1356 	if (!tdma)
1357 		return -ENOMEM;
1358 
1359 	tdma->dev = &pdev->dev;
1360 	tdma->chip_data = cdata;
1361 	platform_set_drvdata(pdev, tdma);
1362 
1363 	tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
1364 	if (IS_ERR(tdma->base_addr))
1365 		return PTR_ERR(tdma->base_addr);
1366 
1367 	tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma");
1368 	if (IS_ERR(tdma->rst)) {
1369 		return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst),
1370 			      "Missing controller reset\n");
1371 	}
1372 	reset_control_reset(tdma->rst);
1373 
1374 	tdma->dma_dev.dev = &pdev->dev;
1375 
1376 	iommu_spec = dev_iommu_fwspec_get(&pdev->dev);
1377 	if (!iommu_spec) {
1378 		dev_err(&pdev->dev, "Missing iommu stream-id\n");
1379 		return -EINVAL;
1380 	}
1381 	stream_id = iommu_spec->ids[0] & 0xffff;
1382 
1383 	INIT_LIST_HEAD(&tdma->dma_dev.channels);
1384 	for (i = 0; i < cdata->nr_channels; i++) {
1385 		struct tegra_dma_channel *tdc = &tdma->channels[i];
1386 
1387 		tdc->irq = platform_get_irq(pdev, i);
1388 		if (tdc->irq < 0)
1389 			return tdc->irq;
1390 
1391 		tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET +
1392 					i * cdata->channel_reg_size;
1393 		snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i);
1394 		tdc->tdma = tdma;
1395 		tdc->id = i;
1396 		tdc->slave_id = -1;
1397 
1398 		vchan_init(&tdc->vc, &tdma->dma_dev);
1399 		tdc->vc.desc_free = tegra_dma_desc_free;
1400 
1401 		/* program stream-id for this channel */
1402 		tegra_dma_program_sid(tdc, stream_id);
1403 		tdc->stream_id = stream_id;
1404 	}
1405 
1406 	dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
1407 	dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1408 	dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask);
1409 	dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask);
1410 	dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1411 
1412 	/*
1413 	 * Only word aligned transfers are supported. Set the copy
1414 	 * alignment shift.
1415 	 */
1416 	tdma->dma_dev.copy_align = 2;
1417 	tdma->dma_dev.fill_align = 2;
1418 	tdma->dma_dev.device_alloc_chan_resources =
1419 					tegra_dma_alloc_chan_resources;
1420 	tdma->dma_dev.device_free_chan_resources =
1421 					tegra_dma_free_chan_resources;
1422 	tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
1423 	tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy;
1424 	tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset;
1425 	tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
1426 	tdma->dma_dev.device_config = tegra_dma_slave_config;
1427 	tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
1428 	tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1429 	tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1430 	tdma->dma_dev.device_pause = tegra_dma_device_pause;
1431 	tdma->dma_dev.device_resume = tegra_dma_device_resume;
1432 	tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize;
1433 	tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1434 
1435 	ret = dma_async_device_register(&tdma->dma_dev);
1436 	if (ret < 0) {
1437 		dev_err_probe(&pdev->dev, ret,
1438 			      "GPC DMA driver registration failed\n");
1439 		return ret;
1440 	}
1441 
1442 	ret = of_dma_controller_register(pdev->dev.of_node,
1443 					 tegra_dma_of_xlate, tdma);
1444 	if (ret < 0) {
1445 		dev_err_probe(&pdev->dev, ret,
1446 			      "GPC DMA OF registration failed\n");
1447 
1448 		dma_async_device_unregister(&tdma->dma_dev);
1449 		return ret;
1450 	}
1451 
1452 	dev_info(&pdev->dev, "GPC DMA driver register %d channels\n",
1453 		 cdata->nr_channels);
1454 
1455 	return 0;
1456 }
1457 
1458 static int tegra_dma_remove(struct platform_device *pdev)
1459 {
1460 	struct tegra_dma *tdma = platform_get_drvdata(pdev);
1461 
1462 	of_dma_controller_free(pdev->dev.of_node);
1463 	dma_async_device_unregister(&tdma->dma_dev);
1464 
1465 	return 0;
1466 }
1467 
1468 static int __maybe_unused tegra_dma_pm_suspend(struct device *dev)
1469 {
1470 	struct tegra_dma *tdma = dev_get_drvdata(dev);
1471 	unsigned int i;
1472 
1473 	for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1474 		struct tegra_dma_channel *tdc = &tdma->channels[i];
1475 
1476 		if (tdc->dma_desc) {
1477 			dev_err(tdma->dev, "channel %u busy\n", i);
1478 			return -EBUSY;
1479 		}
1480 	}
1481 
1482 	return 0;
1483 }
1484 
1485 static int __maybe_unused tegra_dma_pm_resume(struct device *dev)
1486 {
1487 	struct tegra_dma *tdma = dev_get_drvdata(dev);
1488 	unsigned int i;
1489 
1490 	reset_control_reset(tdma->rst);
1491 
1492 	for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1493 		struct tegra_dma_channel *tdc = &tdma->channels[i];
1494 
1495 		tegra_dma_program_sid(tdc, tdc->stream_id);
1496 	}
1497 
1498 	return 0;
1499 }
1500 
1501 static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1502 	SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)
1503 };
1504 
1505 static struct platform_driver tegra_dma_driver = {
1506 	.driver = {
1507 		.name	= "tegra-gpcdma",
1508 		.pm	= &tegra_dma_dev_pm_ops,
1509 		.of_match_table = tegra_dma_of_match,
1510 	},
1511 	.probe		= tegra_dma_probe,
1512 	.remove		= tegra_dma_remove,
1513 };
1514 
1515 module_platform_driver(tegra_dma_driver);
1516 
1517 MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver");
1518 MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>");
1519 MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>");
1520 MODULE_LICENSE("GPL");
1521