1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * DMA driver for NVIDIA Tegra GPC DMA controller.
4  *
5  * Copyright (c) 2014-2022, NVIDIA CORPORATION.  All rights reserved.
6  */
7 
8 #include <linux/bitfield.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/iommu.h>
13 #include <linux/iopoll.h>
14 #include <linux/minmax.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/of_dma.h>
18 #include <linux/platform_device.h>
19 #include <linux/reset.h>
20 #include <linux/slab.h>
21 #include <dt-bindings/memory/tegra186-mc.h>
22 #include "virt-dma.h"
23 
24 /* CSR register */
25 #define TEGRA_GPCDMA_CHAN_CSR			0x00
26 #define TEGRA_GPCDMA_CSR_ENB			BIT(31)
27 #define TEGRA_GPCDMA_CSR_IE_EOC			BIT(30)
28 #define TEGRA_GPCDMA_CSR_ONCE			BIT(27)
29 
30 #define TEGRA_GPCDMA_CSR_FC_MODE		GENMASK(25, 24)
31 #define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO	\
32 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0)
33 #define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO	\
34 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1)
35 #define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO	\
36 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2)
37 #define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO	\
38 		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3)
39 
40 #define TEGRA_GPCDMA_CSR_DMA			GENMASK(23, 21)
41 #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC	\
42 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0)
43 #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC		\
44 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1)
45 #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC	\
46 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2)
47 #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC		\
48 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3)
49 #define TEGRA_GPCDMA_CSR_DMA_MEM2MEM		\
50 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4)
51 #define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT		\
52 		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6)
53 
54 #define TEGRA_GPCDMA_CSR_REQ_SEL_MASK		GENMASK(20, 16)
55 #define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED		\
56 					FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4)
57 #define TEGRA_GPCDMA_CSR_IRQ_MASK		BIT(15)
58 #define TEGRA_GPCDMA_CSR_WEIGHT			GENMASK(13, 10)
59 
60 /* STATUS register */
61 #define TEGRA_GPCDMA_CHAN_STATUS		0x004
62 #define TEGRA_GPCDMA_STATUS_BUSY		BIT(31)
63 #define TEGRA_GPCDMA_STATUS_ISE_EOC		BIT(30)
64 #define TEGRA_GPCDMA_STATUS_PING_PONG		BIT(28)
65 #define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY	BIT(27)
66 #define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE	BIT(26)
67 #define TEGRA_GPCDMA_STATUS_CHANNEL_RX		BIT(25)
68 #define TEGRA_GPCDMA_STATUS_CHANNEL_TX		BIT(24)
69 #define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA	BIT(23)
70 #define TEGRA_GPCDMA_STATUS_IRQ_STA		BIT(21)
71 #define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA	BIT(20)
72 
73 #define TEGRA_GPCDMA_CHAN_CSRE			0x008
74 #define TEGRA_GPCDMA_CHAN_CSRE_PAUSE		BIT(31)
75 
76 /* Source address */
77 #define TEGRA_GPCDMA_CHAN_SRC_PTR		0x00C
78 
79 /* Destination address */
80 #define TEGRA_GPCDMA_CHAN_DST_PTR		0x010
81 
82 /* High address pointer */
83 #define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR		0x014
84 #define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR		GENMASK(7, 0)
85 #define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR		GENMASK(23, 16)
86 
87 /* MC sequence register */
88 #define TEGRA_GPCDMA_CHAN_MCSEQ			0x18
89 #define TEGRA_GPCDMA_MCSEQ_DATA_SWAP		BIT(31)
90 #define TEGRA_GPCDMA_MCSEQ_REQ_COUNT		GENMASK(30, 25)
91 #define TEGRA_GPCDMA_MCSEQ_BURST		GENMASK(24, 23)
92 #define TEGRA_GPCDMA_MCSEQ_BURST_2		\
93 		FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0)
94 #define TEGRA_GPCDMA_MCSEQ_BURST_16		\
95 		FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3)
96 #define TEGRA_GPCDMA_MCSEQ_WRAP1		GENMASK(22, 20)
97 #define TEGRA_GPCDMA_MCSEQ_WRAP0		GENMASK(19, 17)
98 #define TEGRA_GPCDMA_MCSEQ_WRAP_NONE		0
99 
100 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK	GENMASK(13, 7)
101 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK	GENMASK(6, 0)
102 
103 /* MMIO sequence register */
104 #define TEGRA_GPCDMA_CHAN_MMIOSEQ			0x01c
105 #define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF		BIT(31)
106 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH		GENMASK(30, 28)
107 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8	\
108 		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0)
109 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16	\
110 		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1)
111 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32	\
112 		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2)
113 #define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP		BIT(27)
114 #define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT	23
115 #define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN		2U
116 #define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX		32U
117 #define TEGRA_GPCDMA_MMIOSEQ_BURST(bs)	\
118 		(GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT)
119 #define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID		GENMASK(22, 19)
120 #define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD		GENMASK(18, 16)
121 #define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT		GENMASK(8, 7)
122 
123 /* Channel WCOUNT */
124 #define TEGRA_GPCDMA_CHAN_WCOUNT		0x20
125 
126 /* Transfer count */
127 #define TEGRA_GPCDMA_CHAN_XFER_COUNT		0x24
128 
129 /* DMA byte count status */
130 #define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS	0x28
131 
132 /* Error Status Register */
133 #define TEGRA_GPCDMA_CHAN_ERR_STATUS		0x30
134 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT	8
135 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK	0xF
136 #define TEGRA_GPCDMA_CHAN_ERR_TYPE(err)	(			\
137 		((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) &	\
138 		TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK)
139 #define TEGRA_DMA_BM_FIFO_FULL_ERR		0xF
140 #define TEGRA_DMA_PERIPH_FIFO_FULL_ERR		0xE
141 #define TEGRA_DMA_PERIPH_ID_ERR			0xD
142 #define TEGRA_DMA_STREAM_ID_ERR			0xC
143 #define TEGRA_DMA_MC_SLAVE_ERR			0xB
144 #define TEGRA_DMA_MMIO_SLAVE_ERR		0xA
145 
146 /* Fixed Pattern */
147 #define TEGRA_GPCDMA_CHAN_FIXED_PATTERN		0x34
148 
149 #define TEGRA_GPCDMA_CHAN_TZ			0x38
150 #define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1	BIT(0)
151 #define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1		BIT(1)
152 
153 #define TEGRA_GPCDMA_CHAN_SPARE			0x3c
154 #define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC	BIT(16)
155 
156 /*
157  * If any burst is in flight and DMA paused then this is the time to complete
158  * on-flight burst and update DMA status register.
159  */
160 #define TEGRA_GPCDMA_BURST_COMPLETE_TIME	10
161 #define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT	5000 /* 5 msec */
162 
163 /* Channel base address offset from GPCDMA base address */
164 #define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET	0x10000
165 
166 /* Default channel mask reserving channel0 */
167 #define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK	0xfffffffe
168 
169 struct tegra_dma;
170 struct tegra_dma_channel;
171 
172 /*
173  * tegra_dma_chip_data Tegra chip specific DMA data
174  * @nr_channels: Number of channels available in the controller.
175  * @channel_reg_size: Channel register size.
176  * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
177  * @hw_support_pause: DMA HW engine support pause of the channel.
178  */
179 struct tegra_dma_chip_data {
180 	bool hw_support_pause;
181 	unsigned int nr_channels;
182 	unsigned int channel_reg_size;
183 	unsigned int max_dma_count;
184 	int (*terminate)(struct tegra_dma_channel *tdc);
185 };
186 
187 /* DMA channel registers */
188 struct tegra_dma_channel_regs {
189 	u32 csr;
190 	u32 src_ptr;
191 	u32 dst_ptr;
192 	u32 high_addr_ptr;
193 	u32 mc_seq;
194 	u32 mmio_seq;
195 	u32 wcount;
196 	u32 fixed_pattern;
197 };
198 
199 /*
200  * tegra_dma_sg_req: DMA request details to configure hardware. This
201  * contains the details for one transfer to configure DMA hw.
202  * The client's request for data transfer can be broken into multiple
203  * sub-transfer as per requester details and hw support. This sub transfer
204  * get added as an array in Tegra DMA desc which manages the transfer details.
205  */
206 struct tegra_dma_sg_req {
207 	unsigned int len;
208 	struct tegra_dma_channel_regs ch_regs;
209 };
210 
211 /*
212  * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to
213  * manage client request and keep track of transfer status, callbacks
214  * and request counts etc.
215  */
216 struct tegra_dma_desc {
217 	bool cyclic;
218 	unsigned int bytes_req;
219 	unsigned int bytes_xfer;
220 	unsigned int sg_idx;
221 	unsigned int sg_count;
222 	struct virt_dma_desc vd;
223 	struct tegra_dma_channel *tdc;
224 	struct tegra_dma_sg_req sg_req[];
225 };
226 
227 /*
228  * tegra_dma_channel: Channel specific information
229  */
230 struct tegra_dma_channel {
231 	bool config_init;
232 	char name[30];
233 	enum dma_transfer_direction sid_dir;
234 	int id;
235 	int irq;
236 	int slave_id;
237 	struct tegra_dma *tdma;
238 	struct virt_dma_chan vc;
239 	struct tegra_dma_desc *dma_desc;
240 	struct dma_slave_config dma_sconfig;
241 	unsigned int stream_id;
242 	unsigned long chan_base_offset;
243 };
244 
245 /*
246  * tegra_dma: Tegra DMA specific information
247  */
248 struct tegra_dma {
249 	const struct tegra_dma_chip_data *chip_data;
250 	unsigned long sid_m2d_reserved;
251 	unsigned long sid_d2m_reserved;
252 	u32 chan_mask;
253 	void __iomem *base_addr;
254 	struct device *dev;
255 	struct dma_device dma_dev;
256 	struct reset_control *rst;
257 	struct tegra_dma_channel channels[];
258 };
259 
260 static inline void tdc_write(struct tegra_dma_channel *tdc,
261 			     u32 reg, u32 val)
262 {
263 	writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);
264 }
265 
266 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
267 {
268 	return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg);
269 }
270 
271 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
272 {
273 	return container_of(dc, struct tegra_dma_channel, vc.chan);
274 }
275 
276 static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd)
277 {
278 	return container_of(vd, struct tegra_dma_desc, vd);
279 }
280 
281 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
282 {
283 	return tdc->vc.chan.device->dev;
284 }
285 
286 static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc)
287 {
288 	dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n",
289 		tdc->id, tdc->name);
290 	dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n",
291 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR),
292 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS),
293 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE),
294 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR),
295 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR)
296 	);
297 	dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n",
298 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ),
299 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ),
300 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT),
301 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT),
302 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS)
303 	);
304 	dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n",
305 		tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS));
306 }
307 
308 static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc,
309 				 enum dma_transfer_direction direction)
310 {
311 	struct tegra_dma *tdma = tdc->tdma;
312 	int sid = tdc->slave_id;
313 
314 	if (!is_slave_direction(direction))
315 		return 0;
316 
317 	switch (direction) {
318 	case DMA_MEM_TO_DEV:
319 		if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) {
320 			dev_err(tdma->dev, "slave id already in use\n");
321 			return -EINVAL;
322 		}
323 		break;
324 	case DMA_DEV_TO_MEM:
325 		if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) {
326 			dev_err(tdma->dev, "slave id already in use\n");
327 			return -EINVAL;
328 		}
329 		break;
330 	default:
331 		break;
332 	}
333 
334 	tdc->sid_dir = direction;
335 
336 	return 0;
337 }
338 
339 static void tegra_dma_sid_free(struct tegra_dma_channel *tdc)
340 {
341 	struct tegra_dma *tdma = tdc->tdma;
342 	int sid = tdc->slave_id;
343 
344 	switch (tdc->sid_dir) {
345 	case DMA_MEM_TO_DEV:
346 		clear_bit(sid,  &tdma->sid_m2d_reserved);
347 		break;
348 	case DMA_DEV_TO_MEM:
349 		clear_bit(sid,  &tdma->sid_d2m_reserved);
350 		break;
351 	default:
352 		break;
353 	}
354 
355 	tdc->sid_dir = DMA_TRANS_NONE;
356 }
357 
358 static void tegra_dma_desc_free(struct virt_dma_desc *vd)
359 {
360 	kfree(container_of(vd, struct tegra_dma_desc, vd));
361 }
362 
363 static int tegra_dma_slave_config(struct dma_chan *dc,
364 				  struct dma_slave_config *sconfig)
365 {
366 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
367 
368 	memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
369 	tdc->config_init = true;
370 
371 	return 0;
372 }
373 
374 static int tegra_dma_pause(struct tegra_dma_channel *tdc)
375 {
376 	int ret;
377 	u32 val;
378 
379 	val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
380 	val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
381 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
382 
383 	/* Wait until busy bit is de-asserted */
384 	ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
385 			tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
386 			val,
387 			!(val & TEGRA_GPCDMA_STATUS_BUSY),
388 			TEGRA_GPCDMA_BURST_COMPLETE_TIME,
389 			TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
390 
391 	if (ret) {
392 		dev_err(tdc2dev(tdc), "DMA pause timed out\n");
393 		tegra_dma_dump_chan_regs(tdc);
394 	}
395 
396 	return ret;
397 }
398 
399 static int tegra_dma_device_pause(struct dma_chan *dc)
400 {
401 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
402 	unsigned long flags;
403 	int ret;
404 
405 	if (!tdc->tdma->chip_data->hw_support_pause)
406 		return -ENOSYS;
407 
408 	spin_lock_irqsave(&tdc->vc.lock, flags);
409 	ret = tegra_dma_pause(tdc);
410 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
411 
412 	return ret;
413 }
414 
415 static void tegra_dma_resume(struct tegra_dma_channel *tdc)
416 {
417 	u32 val;
418 
419 	val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
420 	val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
421 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
422 }
423 
424 static int tegra_dma_device_resume(struct dma_chan *dc)
425 {
426 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
427 	unsigned long flags;
428 
429 	if (!tdc->tdma->chip_data->hw_support_pause)
430 		return -ENOSYS;
431 
432 	spin_lock_irqsave(&tdc->vc.lock, flags);
433 	tegra_dma_resume(tdc);
434 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
435 
436 	return 0;
437 }
438 
439 static inline int tegra_dma_pause_noerr(struct tegra_dma_channel *tdc)
440 {
441 	/* Return 0 irrespective of PAUSE status.
442 	 * This is useful to recover channels that can exit out of flush
443 	 * state when the channel is disabled.
444 	 */
445 
446 	tegra_dma_pause(tdc);
447 	return 0;
448 }
449 
450 static void tegra_dma_disable(struct tegra_dma_channel *tdc)
451 {
452 	u32 csr, status;
453 
454 	csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
455 
456 	/* Disable interrupts */
457 	csr &= ~TEGRA_GPCDMA_CSR_IE_EOC;
458 
459 	/* Disable DMA */
460 	csr &= ~TEGRA_GPCDMA_CSR_ENB;
461 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
462 
463 	/* Clear interrupt status if it is there */
464 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
465 	if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) {
466 		dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
467 		tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status);
468 	}
469 }
470 
471 static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)
472 {
473 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
474 	struct tegra_dma_channel_regs *ch_regs;
475 	int ret;
476 	u32 val;
477 
478 	dma_desc->sg_idx++;
479 
480 	/* Reset the sg index for cyclic transfers */
481 	if (dma_desc->sg_idx == dma_desc->sg_count)
482 		dma_desc->sg_idx = 0;
483 
484 	/* Configure next transfer immediately after DMA is busy */
485 	ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
486 			tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
487 			val,
488 			(val & TEGRA_GPCDMA_STATUS_BUSY), 0,
489 			TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
490 	if (ret)
491 		return;
492 
493 	ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
494 
495 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
496 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
497 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
498 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
499 
500 	/* Start DMA */
501 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
502 		  ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
503 }
504 
505 static void tegra_dma_start(struct tegra_dma_channel *tdc)
506 {
507 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
508 	struct tegra_dma_channel_regs *ch_regs;
509 	struct virt_dma_desc *vdesc;
510 
511 	if (!dma_desc) {
512 		vdesc = vchan_next_desc(&tdc->vc);
513 		if (!vdesc)
514 			return;
515 
516 		dma_desc = vd_to_tegra_dma_desc(vdesc);
517 		list_del(&vdesc->node);
518 		dma_desc->tdc = tdc;
519 		tdc->dma_desc = dma_desc;
520 
521 		tegra_dma_resume(tdc);
522 	}
523 
524 	ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
525 
526 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
527 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0);
528 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
529 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
530 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
531 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern);
532 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq);
533 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq);
534 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr);
535 
536 	/* Start DMA */
537 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
538 		  ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
539 }
540 
541 static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc)
542 {
543 	vchan_cookie_complete(&tdc->dma_desc->vd);
544 
545 	tegra_dma_sid_free(tdc);
546 	tdc->dma_desc = NULL;
547 }
548 
549 static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc,
550 					unsigned int err_status)
551 {
552 	switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) {
553 	case TEGRA_DMA_BM_FIFO_FULL_ERR:
554 		dev_err(tdc->tdma->dev,
555 			"GPCDMA CH%d bm fifo full\n", tdc->id);
556 		break;
557 
558 	case TEGRA_DMA_PERIPH_FIFO_FULL_ERR:
559 		dev_err(tdc->tdma->dev,
560 			"GPCDMA CH%d peripheral fifo full\n", tdc->id);
561 		break;
562 
563 	case TEGRA_DMA_PERIPH_ID_ERR:
564 		dev_err(tdc->tdma->dev,
565 			"GPCDMA CH%d illegal peripheral id\n", tdc->id);
566 		break;
567 
568 	case TEGRA_DMA_STREAM_ID_ERR:
569 		dev_err(tdc->tdma->dev,
570 			"GPCDMA CH%d illegal stream id\n", tdc->id);
571 		break;
572 
573 	case TEGRA_DMA_MC_SLAVE_ERR:
574 		dev_err(tdc->tdma->dev,
575 			"GPCDMA CH%d mc slave error\n", tdc->id);
576 		break;
577 
578 	case TEGRA_DMA_MMIO_SLAVE_ERR:
579 		dev_err(tdc->tdma->dev,
580 			"GPCDMA CH%d mmio slave error\n", tdc->id);
581 		break;
582 
583 	default:
584 		dev_err(tdc->tdma->dev,
585 			"GPCDMA CH%d security violation %x\n", tdc->id,
586 			err_status);
587 	}
588 }
589 
590 static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
591 {
592 	struct tegra_dma_channel *tdc = dev_id;
593 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
594 	struct tegra_dma_sg_req *sg_req;
595 	u32 status;
596 
597 	/* Check channel error status register */
598 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS);
599 	if (status) {
600 		tegra_dma_chan_decode_error(tdc, status);
601 		tegra_dma_dump_chan_regs(tdc);
602 		tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF);
603 	}
604 
605 	spin_lock(&tdc->vc.lock);
606 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
607 	if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC))
608 		goto irq_done;
609 
610 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS,
611 		  TEGRA_GPCDMA_STATUS_ISE_EOC);
612 
613 	if (!dma_desc)
614 		goto irq_done;
615 
616 	sg_req = dma_desc->sg_req;
617 	dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len;
618 
619 	if (dma_desc->cyclic) {
620 		vchan_cyclic_callback(&dma_desc->vd);
621 		tegra_dma_configure_next_sg(tdc);
622 	} else {
623 		dma_desc->sg_idx++;
624 		if (dma_desc->sg_idx == dma_desc->sg_count)
625 			tegra_dma_xfer_complete(tdc);
626 		else
627 			tegra_dma_start(tdc);
628 	}
629 
630 irq_done:
631 	spin_unlock(&tdc->vc.lock);
632 	return IRQ_HANDLED;
633 }
634 
635 static void tegra_dma_issue_pending(struct dma_chan *dc)
636 {
637 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
638 	unsigned long flags;
639 
640 	if (tdc->dma_desc)
641 		return;
642 
643 	spin_lock_irqsave(&tdc->vc.lock, flags);
644 	if (vchan_issue_pending(&tdc->vc))
645 		tegra_dma_start(tdc);
646 
647 	/*
648 	 * For cyclic DMA transfers, program the second
649 	 * transfer parameters as soon as the first DMA
650 	 * transfer is started inorder for the DMA
651 	 * controller to trigger the second transfer
652 	 * with the correct parameters.
653 	 */
654 	if (tdc->dma_desc && tdc->dma_desc->cyclic)
655 		tegra_dma_configure_next_sg(tdc);
656 
657 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
658 }
659 
660 static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)
661 {
662 	int ret;
663 	u32 status, csr;
664 
665 	/*
666 	 * Change the client associated with the DMA channel
667 	 * to stop DMA engine from starting any more bursts for
668 	 * the given client and wait for in flight bursts to complete
669 	 */
670 	csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
671 	csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK);
672 	csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED;
673 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
674 
675 	/* Wait for in flight data transfer to finish */
676 	udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME);
677 
678 	/* If TX/RX path is still active wait till it becomes
679 	 * inactive
680 	 */
681 
682 	ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
683 				tdc->chan_base_offset +
684 				TEGRA_GPCDMA_CHAN_STATUS,
685 				status,
686 				!(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX |
687 				TEGRA_GPCDMA_STATUS_CHANNEL_RX)),
688 				5,
689 				TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
690 	if (ret) {
691 		dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n");
692 		tegra_dma_dump_chan_regs(tdc);
693 	}
694 
695 	return ret;
696 }
697 
698 static int tegra_dma_terminate_all(struct dma_chan *dc)
699 {
700 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
701 	unsigned long flags;
702 	LIST_HEAD(head);
703 	int err;
704 
705 	spin_lock_irqsave(&tdc->vc.lock, flags);
706 
707 	if (tdc->dma_desc) {
708 		err = tdc->tdma->chip_data->terminate(tdc);
709 		if (err) {
710 			spin_unlock_irqrestore(&tdc->vc.lock, flags);
711 			return err;
712 		}
713 
714 		vchan_terminate_vdesc(&tdc->dma_desc->vd);
715 		tegra_dma_disable(tdc);
716 		tdc->dma_desc = NULL;
717 	}
718 
719 	tegra_dma_sid_free(tdc);
720 	vchan_get_all_descriptors(&tdc->vc, &head);
721 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
722 
723 	vchan_dma_desc_free_list(&tdc->vc, &head);
724 
725 	return 0;
726 }
727 
728 static int tegra_dma_get_residual(struct tegra_dma_channel *tdc)
729 {
730 	struct tegra_dma_desc *dma_desc = tdc->dma_desc;
731 	struct tegra_dma_sg_req *sg_req = dma_desc->sg_req;
732 	unsigned int bytes_xfer, residual;
733 	u32 wcount = 0, status;
734 
735 	wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT);
736 
737 	/*
738 	 * Set wcount = 0 if EOC bit is set. The transfer would have
739 	 * already completed and the CHAN_XFER_COUNT could have updated
740 	 * for the next transfer, specifically in case of cyclic transfers.
741 	 */
742 	status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
743 	if (status & TEGRA_GPCDMA_STATUS_ISE_EOC)
744 		wcount = 0;
745 
746 	bytes_xfer = dma_desc->bytes_xfer +
747 		     sg_req[dma_desc->sg_idx].len - (wcount * 4);
748 
749 	if (dma_desc->bytes_req == bytes_xfer)
750 		return 0;
751 
752 	residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req);
753 
754 	return residual;
755 }
756 
757 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
758 					   dma_cookie_t cookie,
759 					   struct dma_tx_state *txstate)
760 {
761 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
762 	struct tegra_dma_desc *dma_desc;
763 	struct virt_dma_desc *vd;
764 	unsigned int residual;
765 	unsigned long flags;
766 	enum dma_status ret;
767 
768 	ret = dma_cookie_status(dc, cookie, txstate);
769 	if (ret == DMA_COMPLETE)
770 		return ret;
771 
772 	spin_lock_irqsave(&tdc->vc.lock, flags);
773 	vd = vchan_find_desc(&tdc->vc, cookie);
774 	if (vd) {
775 		dma_desc = vd_to_tegra_dma_desc(vd);
776 		residual = dma_desc->bytes_req;
777 		dma_set_residue(txstate, residual);
778 	} else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) {
779 		residual =  tegra_dma_get_residual(tdc);
780 		dma_set_residue(txstate, residual);
781 	} else {
782 		dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie);
783 	}
784 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
785 
786 	return ret;
787 }
788 
789 static inline int get_bus_width(struct tegra_dma_channel *tdc,
790 				enum dma_slave_buswidth slave_bw)
791 {
792 	switch (slave_bw) {
793 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
794 		return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8;
795 	case DMA_SLAVE_BUSWIDTH_2_BYTES:
796 		return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16;
797 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
798 		return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32;
799 	default:
800 		dev_err(tdc2dev(tdc), "given slave bus width is not supported\n");
801 		return -EINVAL;
802 	}
803 }
804 
805 static unsigned int get_burst_size(struct tegra_dma_channel *tdc,
806 				   u32 burst_size, enum dma_slave_buswidth slave_bw,
807 				   int len)
808 {
809 	unsigned int burst_mmio_width, burst_byte;
810 
811 	/*
812 	 * burst_size from client is in terms of the bus_width.
813 	 * convert that into words.
814 	 * If burst_size is not specified from client, then use
815 	 * len to calculate the optimum burst size
816 	 */
817 	burst_byte = burst_size ? burst_size * slave_bw : len;
818 	burst_mmio_width = burst_byte / 4;
819 
820 	if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN)
821 		return 0;
822 
823 	burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX);
824 
825 	return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width);
826 }
827 
828 static int get_transfer_param(struct tegra_dma_channel *tdc,
829 			      enum dma_transfer_direction direction,
830 			      u32 *apb_addr,
831 			      u32 *mmio_seq,
832 			      u32 *csr,
833 			      unsigned int *burst_size,
834 			      enum dma_slave_buswidth *slave_bw)
835 {
836 	switch (direction) {
837 	case DMA_MEM_TO_DEV:
838 		*apb_addr = tdc->dma_sconfig.dst_addr;
839 		*mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
840 		*burst_size = tdc->dma_sconfig.dst_maxburst;
841 		*slave_bw = tdc->dma_sconfig.dst_addr_width;
842 		*csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC;
843 		return 0;
844 	case DMA_DEV_TO_MEM:
845 		*apb_addr = tdc->dma_sconfig.src_addr;
846 		*mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
847 		*burst_size = tdc->dma_sconfig.src_maxburst;
848 		*slave_bw = tdc->dma_sconfig.src_addr_width;
849 		*csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC;
850 		return 0;
851 	default:
852 		dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
853 	}
854 
855 	return -EINVAL;
856 }
857 
858 static struct dma_async_tx_descriptor *
859 tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,
860 			  size_t len, unsigned long flags)
861 {
862 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
863 	unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
864 	struct tegra_dma_sg_req *sg_req;
865 	struct tegra_dma_desc *dma_desc;
866 	u32 csr, mc_seq;
867 
868 	if ((len & 3) || (dest & 3) || len > max_dma_count) {
869 		dev_err(tdc2dev(tdc),
870 			"DMA length/memory address is not supported\n");
871 		return NULL;
872 	}
873 
874 	/* Set DMA mode to fixed pattern */
875 	csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT;
876 	/* Enable once or continuous mode */
877 	csr |= TEGRA_GPCDMA_CSR_ONCE;
878 	/* Enable IRQ mask */
879 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
880 	/* Enable the DMA interrupt */
881 	if (flags & DMA_PREP_INTERRUPT)
882 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
883 	/* Configure default priority weight for the channel */
884 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
885 
886 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
887 	/* retain stream-id and clean rest */
888 	mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
889 
890 	/* Set the address wrapping */
891 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
892 						TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
893 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
894 						TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
895 
896 	/* Program outstanding MC requests */
897 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
898 	/* Set burst size */
899 	mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
900 
901 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
902 	if (!dma_desc)
903 		return NULL;
904 
905 	dma_desc->bytes_req = len;
906 	dma_desc->sg_count = 1;
907 	sg_req = dma_desc->sg_req;
908 
909 	sg_req[0].ch_regs.src_ptr = 0;
910 	sg_req[0].ch_regs.dst_ptr = dest;
911 	sg_req[0].ch_regs.high_addr_ptr =
912 			FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
913 	sg_req[0].ch_regs.fixed_pattern = value;
914 	/* Word count reg takes value as (N +1) words */
915 	sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
916 	sg_req[0].ch_regs.csr = csr;
917 	sg_req[0].ch_regs.mmio_seq = 0;
918 	sg_req[0].ch_regs.mc_seq = mc_seq;
919 	sg_req[0].len = len;
920 
921 	dma_desc->cyclic = false;
922 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
923 }
924 
925 static struct dma_async_tx_descriptor *
926 tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,
927 			  dma_addr_t src, size_t len, unsigned long flags)
928 {
929 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
930 	struct tegra_dma_sg_req *sg_req;
931 	struct tegra_dma_desc *dma_desc;
932 	unsigned int max_dma_count;
933 	u32 csr, mc_seq;
934 
935 	max_dma_count = tdc->tdma->chip_data->max_dma_count;
936 	if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) {
937 		dev_err(tdc2dev(tdc),
938 			"DMA length/memory address is not supported\n");
939 		return NULL;
940 	}
941 
942 	/* Set DMA mode to memory to memory transfer */
943 	csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM;
944 	/* Enable once or continuous mode */
945 	csr |= TEGRA_GPCDMA_CSR_ONCE;
946 	/* Enable IRQ mask */
947 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
948 	/* Enable the DMA interrupt */
949 	if (flags & DMA_PREP_INTERRUPT)
950 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
951 	/* Configure default priority weight for the channel */
952 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
953 
954 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
955 	/* retain stream-id and clean rest */
956 	mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) |
957 		  (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
958 
959 	/* Set the address wrapping */
960 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
961 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
962 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
963 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
964 
965 	/* Program outstanding MC requests */
966 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
967 	/* Set burst size */
968 	mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
969 
970 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
971 	if (!dma_desc)
972 		return NULL;
973 
974 	dma_desc->bytes_req = len;
975 	dma_desc->sg_count = 1;
976 	sg_req = dma_desc->sg_req;
977 
978 	sg_req[0].ch_regs.src_ptr = src;
979 	sg_req[0].ch_regs.dst_ptr = dest;
980 	sg_req[0].ch_regs.high_addr_ptr =
981 		FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32));
982 	sg_req[0].ch_regs.high_addr_ptr |=
983 		FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
984 	/* Word count reg takes value as (N +1) words */
985 	sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
986 	sg_req[0].ch_regs.csr = csr;
987 	sg_req[0].ch_regs.mmio_seq = 0;
988 	sg_req[0].ch_regs.mc_seq = mc_seq;
989 	sg_req[0].len = len;
990 
991 	dma_desc->cyclic = false;
992 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
993 }
994 
995 static struct dma_async_tx_descriptor *
996 tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
997 			unsigned int sg_len, enum dma_transfer_direction direction,
998 			unsigned long flags, void *context)
999 {
1000 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1001 	unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
1002 	enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
1003 	u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0;
1004 	struct tegra_dma_sg_req *sg_req;
1005 	struct tegra_dma_desc *dma_desc;
1006 	struct scatterlist *sg;
1007 	u32 burst_size;
1008 	unsigned int i;
1009 	int ret;
1010 
1011 	if (!tdc->config_init) {
1012 		dev_err(tdc2dev(tdc), "DMA channel is not configured\n");
1013 		return NULL;
1014 	}
1015 	if (sg_len < 1) {
1016 		dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
1017 		return NULL;
1018 	}
1019 
1020 	ret = tegra_dma_sid_reserve(tdc, direction);
1021 	if (ret)
1022 		return NULL;
1023 
1024 	ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1025 				 &burst_size, &slave_bw);
1026 	if (ret < 0)
1027 		return NULL;
1028 
1029 	/* Enable once or continuous mode */
1030 	csr |= TEGRA_GPCDMA_CSR_ONCE;
1031 	/* Program the slave id in requestor select */
1032 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1033 	/* Enable IRQ mask */
1034 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1035 	/* Configure default priority weight for the channel*/
1036 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1037 
1038 	/* Enable the DMA interrupt */
1039 	if (flags & DMA_PREP_INTERRUPT)
1040 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1041 
1042 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1043 	/* retain stream-id and clean rest */
1044 	mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1045 
1046 	/* Set the address wrapping on both MC and MMIO side */
1047 
1048 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1049 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1050 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1051 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1052 	mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1053 
1054 	/* Program 2 MC outstanding requests by default. */
1055 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1056 
1057 	/* Setting MC burst size depending on MMIO burst size */
1058 	if (burst_size == 64)
1059 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1060 	else
1061 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1062 
1063 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, sg_len), GFP_NOWAIT);
1064 	if (!dma_desc)
1065 		return NULL;
1066 
1067 	dma_desc->sg_count = sg_len;
1068 	sg_req = dma_desc->sg_req;
1069 
1070 	/* Make transfer requests */
1071 	for_each_sg(sgl, sg, sg_len, i) {
1072 		u32 len;
1073 		dma_addr_t mem;
1074 
1075 		mem = sg_dma_address(sg);
1076 		len = sg_dma_len(sg);
1077 
1078 		if ((len & 3) || (mem & 3) || len > max_dma_count) {
1079 			dev_err(tdc2dev(tdc),
1080 				"DMA length/memory address is not supported\n");
1081 			kfree(dma_desc);
1082 			return NULL;
1083 		}
1084 
1085 		mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1086 		dma_desc->bytes_req += len;
1087 
1088 		if (direction == DMA_MEM_TO_DEV) {
1089 			sg_req[i].ch_regs.src_ptr = mem;
1090 			sg_req[i].ch_regs.dst_ptr = apb_ptr;
1091 			sg_req[i].ch_regs.high_addr_ptr =
1092 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1093 		} else if (direction == DMA_DEV_TO_MEM) {
1094 			sg_req[i].ch_regs.src_ptr = apb_ptr;
1095 			sg_req[i].ch_regs.dst_ptr = mem;
1096 			sg_req[i].ch_regs.high_addr_ptr =
1097 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1098 		}
1099 
1100 		/*
1101 		 * Word count register takes input in words. Writing a value
1102 		 * of N into word count register means a req of (N+1) words.
1103 		 */
1104 		sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1105 		sg_req[i].ch_regs.csr = csr;
1106 		sg_req[i].ch_regs.mmio_seq = mmio_seq;
1107 		sg_req[i].ch_regs.mc_seq = mc_seq;
1108 		sg_req[i].len = len;
1109 	}
1110 
1111 	dma_desc->cyclic = false;
1112 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1113 }
1114 
1115 static struct dma_async_tx_descriptor *
1116 tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1117 			  size_t period_len, enum dma_transfer_direction direction,
1118 			  unsigned long flags)
1119 {
1120 	enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
1121 	u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size;
1122 	unsigned int max_dma_count, len, period_count, i;
1123 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1124 	struct tegra_dma_desc *dma_desc;
1125 	struct tegra_dma_sg_req *sg_req;
1126 	dma_addr_t mem = buf_addr;
1127 	int ret;
1128 
1129 	if (!buf_len || !period_len) {
1130 		dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
1131 		return NULL;
1132 	}
1133 
1134 	if (!tdc->config_init) {
1135 		dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
1136 		return NULL;
1137 	}
1138 
1139 	ret = tegra_dma_sid_reserve(tdc, direction);
1140 	if (ret)
1141 		return NULL;
1142 
1143 	/*
1144 	 * We only support cycle transfer when buf_len is multiple of
1145 	 * period_len.
1146 	 */
1147 	if (buf_len % period_len) {
1148 		dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
1149 		return NULL;
1150 	}
1151 
1152 	len = period_len;
1153 	max_dma_count = tdc->tdma->chip_data->max_dma_count;
1154 	if ((len & 3) || (buf_addr & 3) || len > max_dma_count) {
1155 		dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1156 		return NULL;
1157 	}
1158 
1159 	ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1160 				 &burst_size, &slave_bw);
1161 	if (ret < 0)
1162 		return NULL;
1163 
1164 	/* Enable once or continuous mode */
1165 	csr &= ~TEGRA_GPCDMA_CSR_ONCE;
1166 	/* Program the slave id in requestor select */
1167 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1168 	/* Enable IRQ mask */
1169 	csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1170 	/* Configure default priority weight for the channel*/
1171 	csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1172 
1173 	/* Enable the DMA interrupt */
1174 	if (flags & DMA_PREP_INTERRUPT)
1175 		csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1176 
1177 	mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1178 
1179 	mc_seq =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1180 	/* retain stream-id and clean rest */
1181 	mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1182 
1183 	/* Set the address wrapping on both MC and MMIO side */
1184 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1185 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1186 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1187 			     TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1188 
1189 	/* Program 2 MC outstanding requests by default. */
1190 	mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1191 	/* Setting MC burst size depending on MMIO burst size */
1192 	if (burst_size == 64)
1193 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1194 	else
1195 		mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1196 
1197 	period_count = buf_len / period_len;
1198 	dma_desc = kzalloc(struct_size(dma_desc, sg_req, period_count),
1199 			   GFP_NOWAIT);
1200 	if (!dma_desc)
1201 		return NULL;
1202 
1203 	dma_desc->bytes_req = buf_len;
1204 	dma_desc->sg_count = period_count;
1205 	sg_req = dma_desc->sg_req;
1206 
1207 	/* Split transfer equal to period size */
1208 	for (i = 0; i < period_count; i++) {
1209 		mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1210 		if (direction == DMA_MEM_TO_DEV) {
1211 			sg_req[i].ch_regs.src_ptr = mem;
1212 			sg_req[i].ch_regs.dst_ptr = apb_ptr;
1213 			sg_req[i].ch_regs.high_addr_ptr =
1214 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1215 		} else if (direction == DMA_DEV_TO_MEM) {
1216 			sg_req[i].ch_regs.src_ptr = apb_ptr;
1217 			sg_req[i].ch_regs.dst_ptr = mem;
1218 			sg_req[i].ch_regs.high_addr_ptr =
1219 				FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1220 		}
1221 		/*
1222 		 * Word count register takes input in words. Writing a value
1223 		 * of N into word count register means a req of (N+1) words.
1224 		 */
1225 		sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1226 		sg_req[i].ch_regs.csr = csr;
1227 		sg_req[i].ch_regs.mmio_seq = mmio_seq;
1228 		sg_req[i].ch_regs.mc_seq = mc_seq;
1229 		sg_req[i].len = len;
1230 
1231 		mem += len;
1232 	}
1233 
1234 	dma_desc->cyclic = true;
1235 
1236 	return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1237 }
1238 
1239 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1240 {
1241 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1242 	int ret;
1243 
1244 	ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);
1245 	if (ret) {
1246 		dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name);
1247 		return ret;
1248 	}
1249 
1250 	dma_cookie_init(&tdc->vc.chan);
1251 	tdc->config_init = false;
1252 	return 0;
1253 }
1254 
1255 static void tegra_dma_chan_synchronize(struct dma_chan *dc)
1256 {
1257 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1258 
1259 	synchronize_irq(tdc->irq);
1260 	vchan_synchronize(&tdc->vc);
1261 }
1262 
1263 static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1264 {
1265 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1266 
1267 	dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1268 
1269 	tegra_dma_terminate_all(dc);
1270 	synchronize_irq(tdc->irq);
1271 
1272 	tasklet_kill(&tdc->vc.task);
1273 	tdc->config_init = false;
1274 	tdc->slave_id = -1;
1275 	tdc->sid_dir = DMA_TRANS_NONE;
1276 	free_irq(tdc->irq, tdc);
1277 
1278 	vchan_free_chan_resources(&tdc->vc);
1279 }
1280 
1281 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
1282 					   struct of_dma *ofdma)
1283 {
1284 	struct tegra_dma *tdma = ofdma->of_dma_data;
1285 	struct tegra_dma_channel *tdc;
1286 	struct dma_chan *chan;
1287 
1288 	chan = dma_get_any_slave_channel(&tdma->dma_dev);
1289 	if (!chan)
1290 		return NULL;
1291 
1292 	tdc = to_tegra_dma_chan(chan);
1293 	tdc->slave_id = dma_spec->args[0];
1294 
1295 	return chan;
1296 }
1297 
1298 static const struct tegra_dma_chip_data tegra186_dma_chip_data = {
1299 	.nr_channels = 32,
1300 	.channel_reg_size = SZ_64K,
1301 	.max_dma_count = SZ_1G,
1302 	.hw_support_pause = false,
1303 	.terminate = tegra_dma_stop_client,
1304 };
1305 
1306 static const struct tegra_dma_chip_data tegra194_dma_chip_data = {
1307 	.nr_channels = 32,
1308 	.channel_reg_size = SZ_64K,
1309 	.max_dma_count = SZ_1G,
1310 	.hw_support_pause = true,
1311 	.terminate = tegra_dma_pause,
1312 };
1313 
1314 static const struct tegra_dma_chip_data tegra234_dma_chip_data = {
1315 	.nr_channels = 32,
1316 	.channel_reg_size = SZ_64K,
1317 	.max_dma_count = SZ_1G,
1318 	.hw_support_pause = true,
1319 	.terminate = tegra_dma_pause_noerr,
1320 };
1321 
1322 static const struct of_device_id tegra_dma_of_match[] = {
1323 	{
1324 		.compatible = "nvidia,tegra186-gpcdma",
1325 		.data = &tegra186_dma_chip_data,
1326 	}, {
1327 		.compatible = "nvidia,tegra194-gpcdma",
1328 		.data = &tegra194_dma_chip_data,
1329 	}, {
1330 		.compatible = "nvidia,tegra234-gpcdma",
1331 		.data = &tegra234_dma_chip_data,
1332 	}, {
1333 	},
1334 };
1335 MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
1336 
1337 static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)
1338 {
1339 	unsigned int reg_val =  tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1340 
1341 	reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK);
1342 	reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
1343 
1344 	reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id);
1345 	reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id);
1346 
1347 	tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val);
1348 	return 0;
1349 }
1350 
1351 static int tegra_dma_probe(struct platform_device *pdev)
1352 {
1353 	const struct tegra_dma_chip_data *cdata = NULL;
1354 	struct iommu_fwspec *iommu_spec;
1355 	unsigned int stream_id, i;
1356 	struct tegra_dma *tdma;
1357 	int ret;
1358 
1359 	cdata = of_device_get_match_data(&pdev->dev);
1360 
1361 	tdma = devm_kzalloc(&pdev->dev,
1362 			    struct_size(tdma, channels, cdata->nr_channels),
1363 			    GFP_KERNEL);
1364 	if (!tdma)
1365 		return -ENOMEM;
1366 
1367 	tdma->dev = &pdev->dev;
1368 	tdma->chip_data = cdata;
1369 	platform_set_drvdata(pdev, tdma);
1370 
1371 	tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
1372 	if (IS_ERR(tdma->base_addr))
1373 		return PTR_ERR(tdma->base_addr);
1374 
1375 	tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma");
1376 	if (IS_ERR(tdma->rst)) {
1377 		return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst),
1378 			      "Missing controller reset\n");
1379 	}
1380 	reset_control_reset(tdma->rst);
1381 
1382 	tdma->dma_dev.dev = &pdev->dev;
1383 
1384 	iommu_spec = dev_iommu_fwspec_get(&pdev->dev);
1385 	if (!iommu_spec) {
1386 		dev_err(&pdev->dev, "Missing iommu stream-id\n");
1387 		return -EINVAL;
1388 	}
1389 	stream_id = iommu_spec->ids[0] & 0xffff;
1390 
1391 	ret = device_property_read_u32(&pdev->dev, "dma-channel-mask",
1392 				       &tdma->chan_mask);
1393 	if (ret) {
1394 		dev_warn(&pdev->dev,
1395 			 "Missing dma-channel-mask property, using default channel mask %#x\n",
1396 			 TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK);
1397 		tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK;
1398 	}
1399 
1400 	INIT_LIST_HEAD(&tdma->dma_dev.channels);
1401 	for (i = 0; i < cdata->nr_channels; i++) {
1402 		struct tegra_dma_channel *tdc = &tdma->channels[i];
1403 
1404 		/* Check for channel mask */
1405 		if (!(tdma->chan_mask & BIT(i)))
1406 			continue;
1407 
1408 		tdc->irq = platform_get_irq(pdev, i);
1409 		if (tdc->irq < 0)
1410 			return tdc->irq;
1411 
1412 		tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET +
1413 					i * cdata->channel_reg_size;
1414 		snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i);
1415 		tdc->tdma = tdma;
1416 		tdc->id = i;
1417 		tdc->slave_id = -1;
1418 
1419 		vchan_init(&tdc->vc, &tdma->dma_dev);
1420 		tdc->vc.desc_free = tegra_dma_desc_free;
1421 
1422 		/* program stream-id for this channel */
1423 		tegra_dma_program_sid(tdc, stream_id);
1424 		tdc->stream_id = stream_id;
1425 	}
1426 
1427 	dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
1428 	dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1429 	dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask);
1430 	dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask);
1431 	dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1432 
1433 	/*
1434 	 * Only word aligned transfers are supported. Set the copy
1435 	 * alignment shift.
1436 	 */
1437 	tdma->dma_dev.copy_align = 2;
1438 	tdma->dma_dev.fill_align = 2;
1439 	tdma->dma_dev.device_alloc_chan_resources =
1440 					tegra_dma_alloc_chan_resources;
1441 	tdma->dma_dev.device_free_chan_resources =
1442 					tegra_dma_free_chan_resources;
1443 	tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
1444 	tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy;
1445 	tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset;
1446 	tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
1447 	tdma->dma_dev.device_config = tegra_dma_slave_config;
1448 	tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
1449 	tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1450 	tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1451 	tdma->dma_dev.device_pause = tegra_dma_device_pause;
1452 	tdma->dma_dev.device_resume = tegra_dma_device_resume;
1453 	tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize;
1454 	tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1455 
1456 	ret = dma_async_device_register(&tdma->dma_dev);
1457 	if (ret < 0) {
1458 		dev_err_probe(&pdev->dev, ret,
1459 			      "GPC DMA driver registration failed\n");
1460 		return ret;
1461 	}
1462 
1463 	ret = of_dma_controller_register(pdev->dev.of_node,
1464 					 tegra_dma_of_xlate, tdma);
1465 	if (ret < 0) {
1466 		dev_err_probe(&pdev->dev, ret,
1467 			      "GPC DMA OF registration failed\n");
1468 
1469 		dma_async_device_unregister(&tdma->dma_dev);
1470 		return ret;
1471 	}
1472 
1473 	dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n",
1474 		 hweight_long(tdma->chan_mask));
1475 
1476 	return 0;
1477 }
1478 
1479 static int tegra_dma_remove(struct platform_device *pdev)
1480 {
1481 	struct tegra_dma *tdma = platform_get_drvdata(pdev);
1482 
1483 	of_dma_controller_free(pdev->dev.of_node);
1484 	dma_async_device_unregister(&tdma->dma_dev);
1485 
1486 	return 0;
1487 }
1488 
1489 static int __maybe_unused tegra_dma_pm_suspend(struct device *dev)
1490 {
1491 	struct tegra_dma *tdma = dev_get_drvdata(dev);
1492 	unsigned int i;
1493 
1494 	for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1495 		struct tegra_dma_channel *tdc = &tdma->channels[i];
1496 
1497 		if (!(tdma->chan_mask & BIT(i)))
1498 			continue;
1499 
1500 		if (tdc->dma_desc) {
1501 			dev_err(tdma->dev, "channel %u busy\n", i);
1502 			return -EBUSY;
1503 		}
1504 	}
1505 
1506 	return 0;
1507 }
1508 
1509 static int __maybe_unused tegra_dma_pm_resume(struct device *dev)
1510 {
1511 	struct tegra_dma *tdma = dev_get_drvdata(dev);
1512 	unsigned int i;
1513 
1514 	reset_control_reset(tdma->rst);
1515 
1516 	for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1517 		struct tegra_dma_channel *tdc = &tdma->channels[i];
1518 
1519 		if (!(tdma->chan_mask & BIT(i)))
1520 			continue;
1521 
1522 		tegra_dma_program_sid(tdc, tdc->stream_id);
1523 	}
1524 
1525 	return 0;
1526 }
1527 
1528 static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1529 	SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)
1530 };
1531 
1532 static struct platform_driver tegra_dma_driver = {
1533 	.driver = {
1534 		.name	= "tegra-gpcdma",
1535 		.pm	= &tegra_dma_dev_pm_ops,
1536 		.of_match_table = tegra_dma_of_match,
1537 	},
1538 	.probe		= tegra_dma_probe,
1539 	.remove		= tegra_dma_remove,
1540 };
1541 
1542 module_platform_driver(tegra_dma_driver);
1543 
1544 MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver");
1545 MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>");
1546 MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>");
1547 MODULE_LICENSE("GPL");
1548