1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4  * Synopsys DesignWare eDMA v0 core
5  *
6  * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
7  */
8 
9 #include <linux/bitfield.h>
10 
11 #include "dw-edma-core.h"
12 #include "dw-edma-v0-core.h"
13 #include "dw-edma-v0-regs.h"
14 #include "dw-edma-v0-debugfs.h"
15 
16 enum dw_edma_control {
17 	DW_EDMA_V0_CB					= BIT(0),
18 	DW_EDMA_V0_TCB					= BIT(1),
19 	DW_EDMA_V0_LLP					= BIT(2),
20 	DW_EDMA_V0_LIE					= BIT(3),
21 	DW_EDMA_V0_RIE					= BIT(4),
22 	DW_EDMA_V0_CCS					= BIT(8),
23 	DW_EDMA_V0_LLE					= BIT(9),
24 };
25 
26 static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
27 {
28 	return dw->rg_region.vaddr;
29 }
30 
31 #define SET_32(dw, name, value)				\
32 	writel(value, &(__dw_regs(dw)->name))
33 
34 #define GET_32(dw, name)				\
35 	readl(&(__dw_regs(dw)->name))
36 
37 #define SET_RW_32(dw, dir, name, value)			\
38 	do {						\
39 		if ((dir) == EDMA_DIR_WRITE)		\
40 			SET_32(dw, wr_##name, value);	\
41 		else					\
42 			SET_32(dw, rd_##name, value);	\
43 	} while (0)
44 
45 #define GET_RW_32(dw, dir, name)			\
46 	((dir) == EDMA_DIR_WRITE			\
47 	  ? GET_32(dw, wr_##name)			\
48 	  : GET_32(dw, rd_##name))
49 
50 #define SET_BOTH_32(dw, name, value)			\
51 	do {						\
52 		SET_32(dw, wr_##name, value);		\
53 		SET_32(dw, rd_##name, value);		\
54 	} while (0)
55 
56 #ifdef CONFIG_64BIT
57 
58 #define SET_64(dw, name, value)				\
59 	writeq(value, &(__dw_regs(dw)->name))
60 
61 #define GET_64(dw, name)				\
62 	readq(&(__dw_regs(dw)->name))
63 
64 #define SET_RW_64(dw, dir, name, value)			\
65 	do {						\
66 		if ((dir) == EDMA_DIR_WRITE)		\
67 			SET_64(dw, wr_##name, value);	\
68 		else					\
69 			SET_64(dw, rd_##name, value);	\
70 	} while (0)
71 
72 #define GET_RW_64(dw, dir, name)			\
73 	((dir) == EDMA_DIR_WRITE			\
74 	  ? GET_64(dw, wr_##name)			\
75 	  : GET_64(dw, rd_##name))
76 
77 #define SET_BOTH_64(dw, name, value)			\
78 	do {						\
79 		SET_64(dw, wr_##name, value);		\
80 		SET_64(dw, rd_##name, value);		\
81 	} while (0)
82 
83 #endif /* CONFIG_64BIT */
84 
85 #define SET_COMPAT(dw, name, value)			\
86 	writel(value, &(__dw_regs(dw)->type.unroll.name))
87 
88 #define SET_RW_COMPAT(dw, dir, name, value)		\
89 	do {						\
90 		if ((dir) == EDMA_DIR_WRITE)		\
91 			SET_COMPAT(dw, wr_##name, value); \
92 		else					\
93 			SET_COMPAT(dw, rd_##name, value); \
94 	} while (0)
95 
96 static inline struct dw_edma_v0_ch_regs __iomem *
97 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
98 {
99 	if (dw->mf == EDMA_MF_EDMA_LEGACY)
100 		return &(__dw_regs(dw)->type.legacy.ch);
101 
102 	if (dir == EDMA_DIR_WRITE)
103 		return &__dw_regs(dw)->type.unroll.ch[ch].wr;
104 
105 	return &__dw_regs(dw)->type.unroll.ch[ch].rd;
106 }
107 
108 static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
109 			     u32 value, void __iomem *addr)
110 {
111 	if (dw->mf == EDMA_MF_EDMA_LEGACY) {
112 		u32 viewport_sel;
113 		unsigned long flags;
114 
115 		raw_spin_lock_irqsave(&dw->lock, flags);
116 
117 		viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
118 		if (dir == EDMA_DIR_READ)
119 			viewport_sel |= BIT(31);
120 
121 		writel(viewport_sel,
122 		       &(__dw_regs(dw)->type.legacy.viewport_sel));
123 		writel(value, addr);
124 
125 		raw_spin_unlock_irqrestore(&dw->lock, flags);
126 	} else {
127 		writel(value, addr);
128 	}
129 }
130 
131 static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
132 			   const void __iomem *addr)
133 {
134 	u32 value;
135 
136 	if (dw->mf == EDMA_MF_EDMA_LEGACY) {
137 		u32 viewport_sel;
138 		unsigned long flags;
139 
140 		raw_spin_lock_irqsave(&dw->lock, flags);
141 
142 		viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
143 		if (dir == EDMA_DIR_READ)
144 			viewport_sel |= BIT(31);
145 
146 		writel(viewport_sel,
147 		       &(__dw_regs(dw)->type.legacy.viewport_sel));
148 		value = readl(addr);
149 
150 		raw_spin_unlock_irqrestore(&dw->lock, flags);
151 	} else {
152 		value = readl(addr);
153 	}
154 
155 	return value;
156 }
157 
158 #define SET_CH_32(dw, dir, ch, name, value) \
159 	writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
160 
161 #define GET_CH_32(dw, dir, ch, name) \
162 	readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
163 
164 #define SET_LL_32(ll, value) \
165 	writel(value, ll)
166 
167 #ifdef CONFIG_64BIT
168 
169 static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
170 			     u64 value, void __iomem *addr)
171 {
172 	if (dw->mf == EDMA_MF_EDMA_LEGACY) {
173 		u32 viewport_sel;
174 		unsigned long flags;
175 
176 		raw_spin_lock_irqsave(&dw->lock, flags);
177 
178 		viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
179 		if (dir == EDMA_DIR_READ)
180 			viewport_sel |= BIT(31);
181 
182 		writel(viewport_sel,
183 		       &(__dw_regs(dw)->type.legacy.viewport_sel));
184 		writeq(value, addr);
185 
186 		raw_spin_unlock_irqrestore(&dw->lock, flags);
187 	} else {
188 		writeq(value, addr);
189 	}
190 }
191 
192 static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
193 			   const void __iomem *addr)
194 {
195 	u32 value;
196 
197 	if (dw->mf == EDMA_MF_EDMA_LEGACY) {
198 		u32 viewport_sel;
199 		unsigned long flags;
200 
201 		raw_spin_lock_irqsave(&dw->lock, flags);
202 
203 		viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
204 		if (dir == EDMA_DIR_READ)
205 			viewport_sel |= BIT(31);
206 
207 		writel(viewport_sel,
208 		       &(__dw_regs(dw)->type.legacy.viewport_sel));
209 		value = readq(addr);
210 
211 		raw_spin_unlock_irqrestore(&dw->lock, flags);
212 	} else {
213 		value = readq(addr);
214 	}
215 
216 	return value;
217 }
218 
219 #define SET_CH_64(dw, dir, ch, name, value) \
220 	writeq_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
221 
222 #define GET_CH_64(dw, dir, ch, name) \
223 	readq_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
224 
225 #define SET_LL_64(ll, value) \
226 	writeq(value, ll)
227 
228 #endif /* CONFIG_64BIT */
229 
230 /* eDMA management callbacks */
231 void dw_edma_v0_core_off(struct dw_edma *dw)
232 {
233 	SET_BOTH_32(dw, int_mask,
234 		    EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
235 	SET_BOTH_32(dw, int_clear,
236 		    EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
237 	SET_BOTH_32(dw, engine_en, 0);
238 }
239 
240 u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
241 {
242 	u32 num_ch;
243 
244 	if (dir == EDMA_DIR_WRITE)
245 		num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK,
246 				   GET_32(dw, ctrl));
247 	else
248 		num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK,
249 				   GET_32(dw, ctrl));
250 
251 	if (num_ch > EDMA_V0_MAX_NR_CH)
252 		num_ch = EDMA_V0_MAX_NR_CH;
253 
254 	return (u16)num_ch;
255 }
256 
257 enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
258 {
259 	struct dw_edma *dw = chan->chip->dw;
260 	u32 tmp;
261 
262 	tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,
263 			GET_CH_32(dw, chan->dir, chan->id, ch_control1));
264 
265 	if (tmp == 1)
266 		return DMA_IN_PROGRESS;
267 	else if (tmp == 3)
268 		return DMA_COMPLETE;
269 	else
270 		return DMA_ERROR;
271 }
272 
273 void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
274 {
275 	struct dw_edma *dw = chan->chip->dw;
276 
277 	SET_RW_32(dw, chan->dir, int_clear,
278 		  FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
279 }
280 
281 void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
282 {
283 	struct dw_edma *dw = chan->chip->dw;
284 
285 	SET_RW_32(dw, chan->dir, int_clear,
286 		  FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
287 }
288 
289 u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
290 {
291 	return FIELD_GET(EDMA_V0_DONE_INT_MASK,
292 			 GET_RW_32(dw, dir, int_status));
293 }
294 
295 u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
296 {
297 	return FIELD_GET(EDMA_V0_ABORT_INT_MASK,
298 			 GET_RW_32(dw, dir, int_status));
299 }
300 
301 static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
302 {
303 	struct dw_edma_burst *child;
304 	struct dw_edma_v0_lli __iomem *lli;
305 	struct dw_edma_v0_llp __iomem *llp;
306 	u32 control = 0, i = 0;
307 	int j;
308 
309 	lli = chunk->ll_region.vaddr;
310 
311 	if (chunk->cb)
312 		control = DW_EDMA_V0_CB;
313 
314 	j = chunk->bursts_alloc;
315 	list_for_each_entry(child, &chunk->burst->list, list) {
316 		j--;
317 		if (!j)
318 			control |= (DW_EDMA_V0_LIE | DW_EDMA_V0_RIE);
319 
320 		/* Channel control */
321 		SET_LL_32(&lli[i].control, control);
322 		/* Transfer size */
323 		SET_LL_32(&lli[i].transfer_size, child->sz);
324 		/* SAR */
325 		#ifdef CONFIG_64BIT
326 			SET_LL_64(&lli[i].sar.reg, child->sar);
327 		#else /* CONFIG_64BIT */
328 			SET_LL_32(&lli[i].sar.lsb, lower_32_bits(child->sar));
329 			SET_LL_32(&lli[i].sar.msb, upper_32_bits(child->sar));
330 		#endif /* CONFIG_64BIT */
331 		/* DAR */
332 		#ifdef CONFIG_64BIT
333 			SET_LL_64(&lli[i].dar.reg, child->dar);
334 		#else /* CONFIG_64BIT */
335 			SET_LL_32(&lli[i].dar.lsb, lower_32_bits(child->dar));
336 			SET_LL_32(&lli[i].dar.msb, upper_32_bits(child->dar));
337 		#endif /* CONFIG_64BIT */
338 		i++;
339 	}
340 
341 	llp = (void __iomem *)&lli[i];
342 	control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
343 	if (!chunk->cb)
344 		control |= DW_EDMA_V0_CB;
345 
346 	/* Channel control */
347 	SET_LL_32(&llp->control, control);
348 	/* Linked list */
349 	#ifdef CONFIG_64BIT
350 		SET_LL_64(&llp->llp.reg, chunk->ll_region.paddr);
351 	#else /* CONFIG_64BIT */
352 		SET_LL_32(&llp->llp.lsb, lower_32_bits(chunk->ll_region.paddr));
353 		SET_LL_32(&llp->llp.msb, upper_32_bits(chunk->ll_region.paddr));
354 	#endif /* CONFIG_64BIT */
355 }
356 
357 void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
358 {
359 	struct dw_edma_chan *chan = chunk->chan;
360 	struct dw_edma *dw = chan->chip->dw;
361 	u32 tmp;
362 
363 	dw_edma_v0_core_write_chunk(chunk);
364 
365 	if (first) {
366 		/* Enable engine */
367 		SET_RW_32(dw, chan->dir, engine_en, BIT(0));
368 		if (dw->mf == EDMA_MF_HDMA_COMPAT) {
369 			switch (chan->id) {
370 			case 0:
371 				SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
372 					      BIT(0));
373 				break;
374 			case 1:
375 				SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
376 					      BIT(0));
377 				break;
378 			case 2:
379 				SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
380 					      BIT(0));
381 				break;
382 			case 3:
383 				SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
384 					      BIT(0));
385 				break;
386 			case 4:
387 				SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
388 					      BIT(0));
389 				break;
390 			case 5:
391 				SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
392 					      BIT(0));
393 				break;
394 			case 6:
395 				SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
396 					      BIT(0));
397 				break;
398 			case 7:
399 				SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
400 					      BIT(0));
401 				break;
402 			}
403 		}
404 		/* Interrupt unmask - done, abort */
405 		tmp = GET_RW_32(dw, chan->dir, int_mask);
406 		tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
407 		tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
408 		SET_RW_32(dw, chan->dir, int_mask, tmp);
409 		/* Linked list error */
410 		tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
411 		tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
412 		SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
413 		/* Channel control */
414 		SET_CH_32(dw, chan->dir, chan->id, ch_control1,
415 			  (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
416 		/* Linked list */
417 
418 		#ifdef CONFIG_64BIT
419 		/* llp is not aligned on 64bit -> keep 32bit accesses */
420 		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
421 			  lower_32_bits(chunk->ll_region.paddr));
422 		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
423 			  upper_32_bits(chunk->ll_region.paddr));
424 		#else /* CONFIG_64BIT */
425 		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
426 			  lower_32_bits(chunk->ll_region.paddr));
427 		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
428 			  upper_32_bits(chunk->ll_region.paddr));
429 		#endif /* CONFIG_64BIT */
430 	}
431 	/* Doorbell */
432 	SET_RW_32(dw, chan->dir, doorbell,
433 		  FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
434 }
435 
436 int dw_edma_v0_core_device_config(struct dw_edma_chan *chan)
437 {
438 	struct dw_edma *dw = chan->chip->dw;
439 	u32 tmp = 0;
440 
441 	/* MSI done addr - low, high */
442 	SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo);
443 	SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi);
444 	/* MSI abort addr - low, high */
445 	SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo);
446 	SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi);
447 	/* MSI data - low, high */
448 	switch (chan->id) {
449 	case 0:
450 	case 1:
451 		tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data);
452 		break;
453 
454 	case 2:
455 	case 3:
456 		tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data);
457 		break;
458 
459 	case 4:
460 	case 5:
461 		tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data);
462 		break;
463 
464 	case 6:
465 	case 7:
466 		tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data);
467 		break;
468 	}
469 
470 	if (chan->id & BIT(0)) {
471 		/* Channel odd {1, 3, 5, 7} */
472 		tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK;
473 		tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK,
474 				  chan->msi.data);
475 	} else {
476 		/* Channel even {0, 2, 4, 6} */
477 		tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK;
478 		tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK,
479 				  chan->msi.data);
480 	}
481 
482 	switch (chan->id) {
483 	case 0:
484 	case 1:
485 		SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp);
486 		break;
487 
488 	case 2:
489 	case 3:
490 		SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp);
491 		break;
492 
493 	case 4:
494 	case 5:
495 		SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp);
496 		break;
497 
498 	case 6:
499 	case 7:
500 		SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp);
501 		break;
502 	}
503 
504 	return 0;
505 }
506 
507 /* eDMA debugfs callbacks */
508 void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip)
509 {
510 	dw_edma_v0_debugfs_on(chip);
511 }
512 
513 void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip)
514 {
515 	dw_edma_v0_debugfs_off(chip);
516 }
517