xref: /openbmc/linux/drivers/dma/mv_xor.c (revision 890bcd49)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * offload engine driver for the Marvell XOR engine
4  * Copyright (C) 2007, 2008, Marvell International Ltd.
5  */
6 
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/spinlock.h>
12 #include <linux/interrupt.h>
13 #include <linux/of_device.h>
14 #include <linux/platform_device.h>
15 #include <linux/memory.h>
16 #include <linux/clk.h>
17 #include <linux/of.h>
18 #include <linux/of_irq.h>
19 #include <linux/irqdomain.h>
20 #include <linux/cpumask.h>
21 #include <linux/platform_data/dma-mv_xor.h>
22 
23 #include "dmaengine.h"
24 #include "mv_xor.h"
25 
26 enum mv_xor_type {
27 	XOR_ORION,
28 	XOR_ARMADA_38X,
29 	XOR_ARMADA_37XX,
30 };
31 
32 enum mv_xor_mode {
33 	XOR_MODE_IN_REG,
34 	XOR_MODE_IN_DESC,
35 };
36 
37 static void mv_xor_issue_pending(struct dma_chan *chan);
38 
39 #define to_mv_xor_chan(chan)		\
40 	container_of(chan, struct mv_xor_chan, dmachan)
41 
42 #define to_mv_xor_slot(tx)		\
43 	container_of(tx, struct mv_xor_desc_slot, async_tx)
44 
45 #define mv_chan_to_devp(chan)           \
46 	((chan)->dmadev.dev)
47 
mv_desc_init(struct mv_xor_desc_slot * desc,dma_addr_t addr,u32 byte_count,enum dma_ctrl_flags flags)48 static void mv_desc_init(struct mv_xor_desc_slot *desc,
49 			 dma_addr_t addr, u32 byte_count,
50 			 enum dma_ctrl_flags flags)
51 {
52 	struct mv_xor_desc *hw_desc = desc->hw_desc;
53 
54 	hw_desc->status = XOR_DESC_DMA_OWNED;
55 	hw_desc->phy_next_desc = 0;
56 	/* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
57 	hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
58 				XOR_DESC_EOD_INT_EN : 0;
59 	hw_desc->phy_dest_addr = addr;
60 	hw_desc->byte_count = byte_count;
61 }
62 
mv_desc_set_mode(struct mv_xor_desc_slot * desc)63 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
64 {
65 	struct mv_xor_desc *hw_desc = desc->hw_desc;
66 
67 	switch (desc->type) {
68 	case DMA_XOR:
69 	case DMA_INTERRUPT:
70 		hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
71 		break;
72 	case DMA_MEMCPY:
73 		hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
74 		break;
75 	default:
76 		BUG();
77 		return;
78 	}
79 }
80 
mv_desc_set_next_desc(struct mv_xor_desc_slot * desc,u32 next_desc_addr)81 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
82 				  u32 next_desc_addr)
83 {
84 	struct mv_xor_desc *hw_desc = desc->hw_desc;
85 	BUG_ON(hw_desc->phy_next_desc);
86 	hw_desc->phy_next_desc = next_desc_addr;
87 }
88 
mv_desc_set_src_addr(struct mv_xor_desc_slot * desc,int index,dma_addr_t addr)89 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
90 				 int index, dma_addr_t addr)
91 {
92 	struct mv_xor_desc *hw_desc = desc->hw_desc;
93 	hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
94 	if (desc->type == DMA_XOR)
95 		hw_desc->desc_command |= (1 << index);
96 }
97 
mv_chan_get_current_desc(struct mv_xor_chan * chan)98 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
99 {
100 	return readl_relaxed(XOR_CURR_DESC(chan));
101 }
102 
mv_chan_set_next_descriptor(struct mv_xor_chan * chan,u32 next_desc_addr)103 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
104 					u32 next_desc_addr)
105 {
106 	writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
107 }
108 
mv_chan_unmask_interrupts(struct mv_xor_chan * chan)109 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
110 {
111 	u32 val = readl_relaxed(XOR_INTR_MASK(chan));
112 	val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
113 	writel_relaxed(val, XOR_INTR_MASK(chan));
114 }
115 
mv_chan_get_intr_cause(struct mv_xor_chan * chan)116 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
117 {
118 	u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
119 	intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
120 	return intr_cause;
121 }
122 
mv_chan_clear_eoc_cause(struct mv_xor_chan * chan)123 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
124 {
125 	u32 val;
126 
127 	val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
128 	val = ~(val << (chan->idx * 16));
129 	dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
130 	writel_relaxed(val, XOR_INTR_CAUSE(chan));
131 }
132 
mv_chan_clear_err_status(struct mv_xor_chan * chan)133 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
134 {
135 	u32 val = 0xFFFF0000 >> (chan->idx * 16);
136 	writel_relaxed(val, XOR_INTR_CAUSE(chan));
137 }
138 
mv_chan_set_mode(struct mv_xor_chan * chan,u32 op_mode)139 static void mv_chan_set_mode(struct mv_xor_chan *chan,
140 			     u32 op_mode)
141 {
142 	u32 config = readl_relaxed(XOR_CONFIG(chan));
143 
144 	config &= ~0x7;
145 	config |= op_mode;
146 
147 #if defined(__BIG_ENDIAN)
148 	config |= XOR_DESCRIPTOR_SWAP;
149 #else
150 	config &= ~XOR_DESCRIPTOR_SWAP;
151 #endif
152 
153 	writel_relaxed(config, XOR_CONFIG(chan));
154 }
155 
mv_chan_activate(struct mv_xor_chan * chan)156 static void mv_chan_activate(struct mv_xor_chan *chan)
157 {
158 	dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
159 
160 	/* writel ensures all descriptors are flushed before activation */
161 	writel(BIT(0), XOR_ACTIVATION(chan));
162 }
163 
mv_chan_is_busy(struct mv_xor_chan * chan)164 static char mv_chan_is_busy(struct mv_xor_chan *chan)
165 {
166 	u32 state = readl_relaxed(XOR_ACTIVATION(chan));
167 
168 	state = (state >> 4) & 0x3;
169 
170 	return (state == 1) ? 1 : 0;
171 }
172 
173 /*
174  * mv_chan_start_new_chain - program the engine to operate on new
175  * chain headed by sw_desc
176  * Caller must hold &mv_chan->lock while calling this function
177  */
mv_chan_start_new_chain(struct mv_xor_chan * mv_chan,struct mv_xor_desc_slot * sw_desc)178 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
179 				    struct mv_xor_desc_slot *sw_desc)
180 {
181 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
182 		__func__, __LINE__, sw_desc);
183 
184 	/* set the hardware chain */
185 	mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
186 
187 	mv_chan->pending++;
188 	mv_xor_issue_pending(&mv_chan->dmachan);
189 }
190 
191 static dma_cookie_t
mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot * desc,struct mv_xor_chan * mv_chan,dma_cookie_t cookie)192 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
193 				struct mv_xor_chan *mv_chan,
194 				dma_cookie_t cookie)
195 {
196 	BUG_ON(desc->async_tx.cookie < 0);
197 
198 	if (desc->async_tx.cookie > 0) {
199 		cookie = desc->async_tx.cookie;
200 
201 		dma_descriptor_unmap(&desc->async_tx);
202 		/* call the callback (must not sleep or submit new
203 		 * operations to this channel)
204 		 */
205 		dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
206 	}
207 
208 	/* run dependent operations */
209 	dma_run_dependencies(&desc->async_tx);
210 
211 	return cookie;
212 }
213 
214 static int
mv_chan_clean_completed_slots(struct mv_xor_chan * mv_chan)215 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
216 {
217 	struct mv_xor_desc_slot *iter, *_iter;
218 
219 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
220 	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
221 				 node) {
222 
223 		if (async_tx_test_ack(&iter->async_tx)) {
224 			list_move_tail(&iter->node, &mv_chan->free_slots);
225 			if (!list_empty(&iter->sg_tx_list)) {
226 				list_splice_tail_init(&iter->sg_tx_list,
227 							&mv_chan->free_slots);
228 			}
229 		}
230 	}
231 	return 0;
232 }
233 
234 static int
mv_desc_clean_slot(struct mv_xor_desc_slot * desc,struct mv_xor_chan * mv_chan)235 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
236 		   struct mv_xor_chan *mv_chan)
237 {
238 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
239 		__func__, __LINE__, desc, desc->async_tx.flags);
240 
241 	/* the client is allowed to attach dependent operations
242 	 * until 'ack' is set
243 	 */
244 	if (!async_tx_test_ack(&desc->async_tx)) {
245 		/* move this slot to the completed_slots */
246 		list_move_tail(&desc->node, &mv_chan->completed_slots);
247 		if (!list_empty(&desc->sg_tx_list)) {
248 			list_splice_tail_init(&desc->sg_tx_list,
249 					      &mv_chan->completed_slots);
250 		}
251 	} else {
252 		list_move_tail(&desc->node, &mv_chan->free_slots);
253 		if (!list_empty(&desc->sg_tx_list)) {
254 			list_splice_tail_init(&desc->sg_tx_list,
255 					      &mv_chan->free_slots);
256 		}
257 	}
258 
259 	return 0;
260 }
261 
262 /* This function must be called with the mv_xor_chan spinlock held */
mv_chan_slot_cleanup(struct mv_xor_chan * mv_chan)263 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
264 {
265 	struct mv_xor_desc_slot *iter, *_iter;
266 	dma_cookie_t cookie = 0;
267 	int busy = mv_chan_is_busy(mv_chan);
268 	u32 current_desc = mv_chan_get_current_desc(mv_chan);
269 	int current_cleaned = 0;
270 	struct mv_xor_desc *hw_desc;
271 
272 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
273 	dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
274 	mv_chan_clean_completed_slots(mv_chan);
275 
276 	/* free completed slots from the chain starting with
277 	 * the oldest descriptor
278 	 */
279 
280 	list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
281 				 node) {
282 
283 		/* clean finished descriptors */
284 		hw_desc = iter->hw_desc;
285 		if (hw_desc->status & XOR_DESC_SUCCESS) {
286 			cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
287 								 cookie);
288 
289 			/* done processing desc, clean slot */
290 			mv_desc_clean_slot(iter, mv_chan);
291 
292 			/* break if we did cleaned the current */
293 			if (iter->async_tx.phys == current_desc) {
294 				current_cleaned = 1;
295 				break;
296 			}
297 		} else {
298 			if (iter->async_tx.phys == current_desc) {
299 				current_cleaned = 0;
300 				break;
301 			}
302 		}
303 	}
304 
305 	if ((busy == 0) && !list_empty(&mv_chan->chain)) {
306 		if (current_cleaned) {
307 			/*
308 			 * current descriptor cleaned and removed, run
309 			 * from list head
310 			 */
311 			iter = list_entry(mv_chan->chain.next,
312 					  struct mv_xor_desc_slot,
313 					  node);
314 			mv_chan_start_new_chain(mv_chan, iter);
315 		} else {
316 			if (!list_is_last(&iter->node, &mv_chan->chain)) {
317 				/*
318 				 * descriptors are still waiting after
319 				 * current, trigger them
320 				 */
321 				iter = list_entry(iter->node.next,
322 						  struct mv_xor_desc_slot,
323 						  node);
324 				mv_chan_start_new_chain(mv_chan, iter);
325 			} else {
326 				/*
327 				 * some descriptors are still waiting
328 				 * to be cleaned
329 				 */
330 				tasklet_schedule(&mv_chan->irq_tasklet);
331 			}
332 		}
333 	}
334 
335 	if (cookie > 0)
336 		mv_chan->dmachan.completed_cookie = cookie;
337 }
338 
mv_xor_tasklet(struct tasklet_struct * t)339 static void mv_xor_tasklet(struct tasklet_struct *t)
340 {
341 	struct mv_xor_chan *chan = from_tasklet(chan, t, irq_tasklet);
342 
343 	spin_lock(&chan->lock);
344 	mv_chan_slot_cleanup(chan);
345 	spin_unlock(&chan->lock);
346 }
347 
348 static struct mv_xor_desc_slot *
mv_chan_alloc_slot(struct mv_xor_chan * mv_chan)349 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
350 {
351 	struct mv_xor_desc_slot *iter;
352 
353 	spin_lock_bh(&mv_chan->lock);
354 
355 	if (!list_empty(&mv_chan->free_slots)) {
356 		iter = list_first_entry(&mv_chan->free_slots,
357 					struct mv_xor_desc_slot,
358 					node);
359 
360 		list_move_tail(&iter->node, &mv_chan->allocated_slots);
361 
362 		spin_unlock_bh(&mv_chan->lock);
363 
364 		/* pre-ack descriptor */
365 		async_tx_ack(&iter->async_tx);
366 		iter->async_tx.cookie = -EBUSY;
367 
368 		return iter;
369 
370 	}
371 
372 	spin_unlock_bh(&mv_chan->lock);
373 
374 	/* try to free some slots if the allocation fails */
375 	tasklet_schedule(&mv_chan->irq_tasklet);
376 
377 	return NULL;
378 }
379 
380 /************************ DMA engine API functions ****************************/
381 static dma_cookie_t
mv_xor_tx_submit(struct dma_async_tx_descriptor * tx)382 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
383 {
384 	struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
385 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
386 	struct mv_xor_desc_slot *old_chain_tail;
387 	dma_cookie_t cookie;
388 	int new_hw_chain = 1;
389 
390 	dev_dbg(mv_chan_to_devp(mv_chan),
391 		"%s sw_desc %p: async_tx %p\n",
392 		__func__, sw_desc, &sw_desc->async_tx);
393 
394 	spin_lock_bh(&mv_chan->lock);
395 	cookie = dma_cookie_assign(tx);
396 
397 	if (list_empty(&mv_chan->chain))
398 		list_move_tail(&sw_desc->node, &mv_chan->chain);
399 	else {
400 		new_hw_chain = 0;
401 
402 		old_chain_tail = list_entry(mv_chan->chain.prev,
403 					    struct mv_xor_desc_slot,
404 					    node);
405 		list_move_tail(&sw_desc->node, &mv_chan->chain);
406 
407 		dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
408 			&old_chain_tail->async_tx.phys);
409 
410 		/* fix up the hardware chain */
411 		mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
412 
413 		/* if the channel is not busy */
414 		if (!mv_chan_is_busy(mv_chan)) {
415 			u32 current_desc = mv_chan_get_current_desc(mv_chan);
416 			/*
417 			 * and the curren desc is the end of the chain before
418 			 * the append, then we need to start the channel
419 			 */
420 			if (current_desc == old_chain_tail->async_tx.phys)
421 				new_hw_chain = 1;
422 		}
423 	}
424 
425 	if (new_hw_chain)
426 		mv_chan_start_new_chain(mv_chan, sw_desc);
427 
428 	spin_unlock_bh(&mv_chan->lock);
429 
430 	return cookie;
431 }
432 
433 /* returns the number of allocated descriptors */
mv_xor_alloc_chan_resources(struct dma_chan * chan)434 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
435 {
436 	void *virt_desc;
437 	dma_addr_t dma_desc;
438 	int idx;
439 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
440 	struct mv_xor_desc_slot *slot = NULL;
441 	int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
442 
443 	/* Allocate descriptor slots */
444 	idx = mv_chan->slots_allocated;
445 	while (idx < num_descs_in_pool) {
446 		slot = kzalloc(sizeof(*slot), GFP_KERNEL);
447 		if (!slot) {
448 			dev_info(mv_chan_to_devp(mv_chan),
449 				 "channel only initialized %d descriptor slots",
450 				 idx);
451 			break;
452 		}
453 		virt_desc = mv_chan->dma_desc_pool_virt;
454 		slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
455 
456 		dma_async_tx_descriptor_init(&slot->async_tx, chan);
457 		slot->async_tx.tx_submit = mv_xor_tx_submit;
458 		INIT_LIST_HEAD(&slot->node);
459 		INIT_LIST_HEAD(&slot->sg_tx_list);
460 		dma_desc = mv_chan->dma_desc_pool;
461 		slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
462 		slot->idx = idx++;
463 
464 		spin_lock_bh(&mv_chan->lock);
465 		mv_chan->slots_allocated = idx;
466 		list_add_tail(&slot->node, &mv_chan->free_slots);
467 		spin_unlock_bh(&mv_chan->lock);
468 	}
469 
470 	dev_dbg(mv_chan_to_devp(mv_chan),
471 		"allocated %d descriptor slots\n",
472 		mv_chan->slots_allocated);
473 
474 	return mv_chan->slots_allocated ? : -ENOMEM;
475 }
476 
477 /*
478  * Check if source or destination is an PCIe/IO address (non-SDRAM) and add
479  * a new MBus window if necessary. Use a cache for these check so that
480  * the MMIO mapped registers don't have to be accessed for this check
481  * to speed up this process.
482  */
mv_xor_add_io_win(struct mv_xor_chan * mv_chan,u32 addr)483 static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)
484 {
485 	struct mv_xor_device *xordev = mv_chan->xordev;
486 	void __iomem *base = mv_chan->mmr_high_base;
487 	u32 win_enable;
488 	u32 size;
489 	u8 target, attr;
490 	int ret;
491 	int i;
492 
493 	/* Nothing needs to get done for the Armada 3700 */
494 	if (xordev->xor_type == XOR_ARMADA_37XX)
495 		return 0;
496 
497 	/*
498 	 * Loop over the cached windows to check, if the requested area
499 	 * is already mapped. If this the case, nothing needs to be done
500 	 * and we can return.
501 	 */
502 	for (i = 0; i < WINDOW_COUNT; i++) {
503 		if (addr >= xordev->win_start[i] &&
504 		    addr <= xordev->win_end[i]) {
505 			/* Window is already mapped */
506 			return 0;
507 		}
508 	}
509 
510 	/*
511 	 * The window is not mapped, so we need to create the new mapping
512 	 */
513 
514 	/* If no IO window is found that addr has to be located in SDRAM */
515 	ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);
516 	if (ret < 0)
517 		return 0;
518 
519 	/*
520 	 * Mask the base addr 'addr' according to 'size' read back from the
521 	 * MBus window. Otherwise we might end up with an address located
522 	 * somewhere in the middle of this area here.
523 	 */
524 	size -= 1;
525 	addr &= ~size;
526 
527 	/*
528 	 * Reading one of both enabled register is enough, as they are always
529 	 * programmed to the identical values
530 	 */
531 	win_enable = readl(base + WINDOW_BAR_ENABLE(0));
532 
533 	/* Set 'i' to the first free window to write the new values to */
534 	i = ffs(~win_enable) - 1;
535 	if (i >= WINDOW_COUNT)
536 		return -ENOMEM;
537 
538 	writel((addr & 0xffff0000) | (attr << 8) | target,
539 	       base + WINDOW_BASE(i));
540 	writel(size & 0xffff0000, base + WINDOW_SIZE(i));
541 
542 	/* Fill the caching variables for later use */
543 	xordev->win_start[i] = addr;
544 	xordev->win_end[i] = addr + size;
545 
546 	win_enable |= (1 << i);
547 	win_enable |= 3 << (16 + (2 * i));
548 	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
549 	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
550 
551 	return 0;
552 }
553 
554 static struct dma_async_tx_descriptor *
mv_xor_prep_dma_xor(struct dma_chan * chan,dma_addr_t dest,dma_addr_t * src,unsigned int src_cnt,size_t len,unsigned long flags)555 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
556 		    unsigned int src_cnt, size_t len, unsigned long flags)
557 {
558 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
559 	struct mv_xor_desc_slot *sw_desc;
560 	int ret;
561 
562 	if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
563 		return NULL;
564 
565 	BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
566 
567 	dev_dbg(mv_chan_to_devp(mv_chan),
568 		"%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
569 		__func__, src_cnt, len, &dest, flags);
570 
571 	/* Check if a new window needs to get added for 'dest' */
572 	ret = mv_xor_add_io_win(mv_chan, dest);
573 	if (ret)
574 		return NULL;
575 
576 	sw_desc = mv_chan_alloc_slot(mv_chan);
577 	if (sw_desc) {
578 		sw_desc->type = DMA_XOR;
579 		sw_desc->async_tx.flags = flags;
580 		mv_desc_init(sw_desc, dest, len, flags);
581 		if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
582 			mv_desc_set_mode(sw_desc);
583 		while (src_cnt--) {
584 			/* Check if a new window needs to get added for 'src' */
585 			ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
586 			if (ret)
587 				return NULL;
588 			mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
589 		}
590 	}
591 
592 	dev_dbg(mv_chan_to_devp(mv_chan),
593 		"%s sw_desc %p async_tx %p \n",
594 		__func__, sw_desc, &sw_desc->async_tx);
595 	return sw_desc ? &sw_desc->async_tx : NULL;
596 }
597 
598 static struct dma_async_tx_descriptor *
mv_xor_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)599 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
600 		size_t len, unsigned long flags)
601 {
602 	/*
603 	 * A MEMCPY operation is identical to an XOR operation with only
604 	 * a single source address.
605 	 */
606 	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
607 }
608 
609 static struct dma_async_tx_descriptor *
mv_xor_prep_dma_interrupt(struct dma_chan * chan,unsigned long flags)610 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
611 {
612 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
613 	dma_addr_t src, dest;
614 	size_t len;
615 
616 	src = mv_chan->dummy_src_addr;
617 	dest = mv_chan->dummy_dst_addr;
618 	len = MV_XOR_MIN_BYTE_COUNT;
619 
620 	/*
621 	 * We implement the DMA_INTERRUPT operation as a minimum sized
622 	 * XOR operation with a single dummy source address.
623 	 */
624 	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
625 }
626 
mv_xor_free_chan_resources(struct dma_chan * chan)627 static void mv_xor_free_chan_resources(struct dma_chan *chan)
628 {
629 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
630 	struct mv_xor_desc_slot *iter, *_iter;
631 	int in_use_descs = 0;
632 
633 	spin_lock_bh(&mv_chan->lock);
634 
635 	mv_chan_slot_cleanup(mv_chan);
636 
637 	list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
638 					node) {
639 		in_use_descs++;
640 		list_move_tail(&iter->node, &mv_chan->free_slots);
641 	}
642 	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
643 				 node) {
644 		in_use_descs++;
645 		list_move_tail(&iter->node, &mv_chan->free_slots);
646 	}
647 	list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
648 				 node) {
649 		in_use_descs++;
650 		list_move_tail(&iter->node, &mv_chan->free_slots);
651 	}
652 	list_for_each_entry_safe_reverse(
653 		iter, _iter, &mv_chan->free_slots, node) {
654 		list_del(&iter->node);
655 		kfree(iter);
656 		mv_chan->slots_allocated--;
657 	}
658 
659 	dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
660 		__func__, mv_chan->slots_allocated);
661 	spin_unlock_bh(&mv_chan->lock);
662 
663 	if (in_use_descs)
664 		dev_err(mv_chan_to_devp(mv_chan),
665 			"freeing %d in use descriptors!\n", in_use_descs);
666 }
667 
668 /**
669  * mv_xor_status - poll the status of an XOR transaction
670  * @chan: XOR channel handle
671  * @cookie: XOR transaction identifier
672  * @txstate: XOR transactions state holder (or NULL)
673  */
mv_xor_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)674 static enum dma_status mv_xor_status(struct dma_chan *chan,
675 					  dma_cookie_t cookie,
676 					  struct dma_tx_state *txstate)
677 {
678 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
679 	enum dma_status ret;
680 
681 	ret = dma_cookie_status(chan, cookie, txstate);
682 	if (ret == DMA_COMPLETE)
683 		return ret;
684 
685 	spin_lock_bh(&mv_chan->lock);
686 	mv_chan_slot_cleanup(mv_chan);
687 	spin_unlock_bh(&mv_chan->lock);
688 
689 	return dma_cookie_status(chan, cookie, txstate);
690 }
691 
mv_chan_dump_regs(struct mv_xor_chan * chan)692 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
693 {
694 	u32 val;
695 
696 	val = readl_relaxed(XOR_CONFIG(chan));
697 	dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
698 
699 	val = readl_relaxed(XOR_ACTIVATION(chan));
700 	dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
701 
702 	val = readl_relaxed(XOR_INTR_CAUSE(chan));
703 	dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
704 
705 	val = readl_relaxed(XOR_INTR_MASK(chan));
706 	dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
707 
708 	val = readl_relaxed(XOR_ERROR_CAUSE(chan));
709 	dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
710 
711 	val = readl_relaxed(XOR_ERROR_ADDR(chan));
712 	dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
713 }
714 
mv_chan_err_interrupt_handler(struct mv_xor_chan * chan,u32 intr_cause)715 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
716 					  u32 intr_cause)
717 {
718 	if (intr_cause & XOR_INT_ERR_DECODE) {
719 		dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
720 		return;
721 	}
722 
723 	dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
724 		chan->idx, intr_cause);
725 
726 	mv_chan_dump_regs(chan);
727 	WARN_ON(1);
728 }
729 
mv_xor_interrupt_handler(int irq,void * data)730 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
731 {
732 	struct mv_xor_chan *chan = data;
733 	u32 intr_cause = mv_chan_get_intr_cause(chan);
734 
735 	dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
736 
737 	if (intr_cause & XOR_INTR_ERRORS)
738 		mv_chan_err_interrupt_handler(chan, intr_cause);
739 
740 	tasklet_schedule(&chan->irq_tasklet);
741 
742 	mv_chan_clear_eoc_cause(chan);
743 
744 	return IRQ_HANDLED;
745 }
746 
mv_xor_issue_pending(struct dma_chan * chan)747 static void mv_xor_issue_pending(struct dma_chan *chan)
748 {
749 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
750 
751 	if (mv_chan->pending >= MV_XOR_THRESHOLD) {
752 		mv_chan->pending = 0;
753 		mv_chan_activate(mv_chan);
754 	}
755 }
756 
757 /*
758  * Perform a transaction to verify the HW works.
759  */
760 
mv_chan_memcpy_self_test(struct mv_xor_chan * mv_chan)761 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
762 {
763 	int i, ret;
764 	void *src, *dest;
765 	dma_addr_t src_dma, dest_dma;
766 	struct dma_chan *dma_chan;
767 	dma_cookie_t cookie;
768 	struct dma_async_tx_descriptor *tx;
769 	struct dmaengine_unmap_data *unmap;
770 	int err = 0;
771 
772 	src = kmalloc(PAGE_SIZE, GFP_KERNEL);
773 	if (!src)
774 		return -ENOMEM;
775 
776 	dest = kzalloc(PAGE_SIZE, GFP_KERNEL);
777 	if (!dest) {
778 		kfree(src);
779 		return -ENOMEM;
780 	}
781 
782 	/* Fill in src buffer */
783 	for (i = 0; i < PAGE_SIZE; i++)
784 		((u8 *) src)[i] = (u8)i;
785 
786 	dma_chan = &mv_chan->dmachan;
787 	if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
788 		err = -ENODEV;
789 		goto out;
790 	}
791 
792 	unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
793 	if (!unmap) {
794 		err = -ENOMEM;
795 		goto free_resources;
796 	}
797 
798 	src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),
799 			       offset_in_page(src), PAGE_SIZE,
800 			       DMA_TO_DEVICE);
801 	unmap->addr[0] = src_dma;
802 
803 	ret = dma_mapping_error(dma_chan->device->dev, src_dma);
804 	if (ret) {
805 		err = -ENOMEM;
806 		goto free_resources;
807 	}
808 	unmap->to_cnt = 1;
809 
810 	dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),
811 				offset_in_page(dest), PAGE_SIZE,
812 				DMA_FROM_DEVICE);
813 	unmap->addr[1] = dest_dma;
814 
815 	ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
816 	if (ret) {
817 		err = -ENOMEM;
818 		goto free_resources;
819 	}
820 	unmap->from_cnt = 1;
821 	unmap->len = PAGE_SIZE;
822 
823 	tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
824 				    PAGE_SIZE, 0);
825 	if (!tx) {
826 		dev_err(dma_chan->device->dev,
827 			"Self-test cannot prepare operation, disabling\n");
828 		err = -ENODEV;
829 		goto free_resources;
830 	}
831 
832 	cookie = mv_xor_tx_submit(tx);
833 	if (dma_submit_error(cookie)) {
834 		dev_err(dma_chan->device->dev,
835 			"Self-test submit error, disabling\n");
836 		err = -ENODEV;
837 		goto free_resources;
838 	}
839 
840 	mv_xor_issue_pending(dma_chan);
841 	async_tx_ack(tx);
842 	msleep(1);
843 
844 	if (mv_xor_status(dma_chan, cookie, NULL) !=
845 	    DMA_COMPLETE) {
846 		dev_err(dma_chan->device->dev,
847 			"Self-test copy timed out, disabling\n");
848 		err = -ENODEV;
849 		goto free_resources;
850 	}
851 
852 	dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
853 				PAGE_SIZE, DMA_FROM_DEVICE);
854 	if (memcmp(src, dest, PAGE_SIZE)) {
855 		dev_err(dma_chan->device->dev,
856 			"Self-test copy failed compare, disabling\n");
857 		err = -ENODEV;
858 		goto free_resources;
859 	}
860 
861 free_resources:
862 	dmaengine_unmap_put(unmap);
863 	mv_xor_free_chan_resources(dma_chan);
864 out:
865 	kfree(src);
866 	kfree(dest);
867 	return err;
868 }
869 
870 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
871 static int
mv_chan_xor_self_test(struct mv_xor_chan * mv_chan)872 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
873 {
874 	int i, src_idx, ret;
875 	struct page *dest;
876 	struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
877 	dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
878 	dma_addr_t dest_dma;
879 	struct dma_async_tx_descriptor *tx;
880 	struct dmaengine_unmap_data *unmap;
881 	struct dma_chan *dma_chan;
882 	dma_cookie_t cookie;
883 	u8 cmp_byte = 0;
884 	u32 cmp_word;
885 	int err = 0;
886 	int src_count = MV_XOR_NUM_SRC_TEST;
887 
888 	for (src_idx = 0; src_idx < src_count; src_idx++) {
889 		xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
890 		if (!xor_srcs[src_idx]) {
891 			while (src_idx--)
892 				__free_page(xor_srcs[src_idx]);
893 			return -ENOMEM;
894 		}
895 	}
896 
897 	dest = alloc_page(GFP_KERNEL);
898 	if (!dest) {
899 		while (src_idx--)
900 			__free_page(xor_srcs[src_idx]);
901 		return -ENOMEM;
902 	}
903 
904 	/* Fill in src buffers */
905 	for (src_idx = 0; src_idx < src_count; src_idx++) {
906 		u8 *ptr = page_address(xor_srcs[src_idx]);
907 		for (i = 0; i < PAGE_SIZE; i++)
908 			ptr[i] = (1 << src_idx);
909 	}
910 
911 	for (src_idx = 0; src_idx < src_count; src_idx++)
912 		cmp_byte ^= (u8) (1 << src_idx);
913 
914 	cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
915 		(cmp_byte << 8) | cmp_byte;
916 
917 	memset(page_address(dest), 0, PAGE_SIZE);
918 
919 	dma_chan = &mv_chan->dmachan;
920 	if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
921 		err = -ENODEV;
922 		goto out;
923 	}
924 
925 	unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
926 					 GFP_KERNEL);
927 	if (!unmap) {
928 		err = -ENOMEM;
929 		goto free_resources;
930 	}
931 
932 	/* test xor */
933 	for (i = 0; i < src_count; i++) {
934 		unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
935 					      0, PAGE_SIZE, DMA_TO_DEVICE);
936 		dma_srcs[i] = unmap->addr[i];
937 		ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
938 		if (ret) {
939 			err = -ENOMEM;
940 			goto free_resources;
941 		}
942 		unmap->to_cnt++;
943 	}
944 
945 	unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
946 				      DMA_FROM_DEVICE);
947 	dest_dma = unmap->addr[src_count];
948 	ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
949 	if (ret) {
950 		err = -ENOMEM;
951 		goto free_resources;
952 	}
953 	unmap->from_cnt = 1;
954 	unmap->len = PAGE_SIZE;
955 
956 	tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
957 				 src_count, PAGE_SIZE, 0);
958 	if (!tx) {
959 		dev_err(dma_chan->device->dev,
960 			"Self-test cannot prepare operation, disabling\n");
961 		err = -ENODEV;
962 		goto free_resources;
963 	}
964 
965 	cookie = mv_xor_tx_submit(tx);
966 	if (dma_submit_error(cookie)) {
967 		dev_err(dma_chan->device->dev,
968 			"Self-test submit error, disabling\n");
969 		err = -ENODEV;
970 		goto free_resources;
971 	}
972 
973 	mv_xor_issue_pending(dma_chan);
974 	async_tx_ack(tx);
975 	msleep(8);
976 
977 	if (mv_xor_status(dma_chan, cookie, NULL) !=
978 	    DMA_COMPLETE) {
979 		dev_err(dma_chan->device->dev,
980 			"Self-test xor timed out, disabling\n");
981 		err = -ENODEV;
982 		goto free_resources;
983 	}
984 
985 	dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
986 				PAGE_SIZE, DMA_FROM_DEVICE);
987 	for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
988 		u32 *ptr = page_address(dest);
989 		if (ptr[i] != cmp_word) {
990 			dev_err(dma_chan->device->dev,
991 				"Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
992 				i, ptr[i], cmp_word);
993 			err = -ENODEV;
994 			goto free_resources;
995 		}
996 	}
997 
998 free_resources:
999 	dmaengine_unmap_put(unmap);
1000 	mv_xor_free_chan_resources(dma_chan);
1001 out:
1002 	src_idx = src_count;
1003 	while (src_idx--)
1004 		__free_page(xor_srcs[src_idx]);
1005 	__free_page(dest);
1006 	return err;
1007 }
1008 
mv_xor_channel_remove(struct mv_xor_chan * mv_chan)1009 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
1010 {
1011 	struct dma_chan *chan, *_chan;
1012 	struct device *dev = mv_chan->dmadev.dev;
1013 
1014 	dma_async_device_unregister(&mv_chan->dmadev);
1015 
1016 	dma_free_coherent(dev, MV_XOR_POOL_SIZE,
1017 			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1018 	dma_unmap_single(dev, mv_chan->dummy_src_addr,
1019 			 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1020 	dma_unmap_single(dev, mv_chan->dummy_dst_addr,
1021 			 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1022 
1023 	list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
1024 				 device_node) {
1025 		list_del(&chan->device_node);
1026 	}
1027 
1028 	free_irq(mv_chan->irq, mv_chan);
1029 
1030 	return 0;
1031 }
1032 
1033 static struct mv_xor_chan *
mv_xor_channel_add(struct mv_xor_device * xordev,struct platform_device * pdev,int idx,dma_cap_mask_t cap_mask,int irq)1034 mv_xor_channel_add(struct mv_xor_device *xordev,
1035 		   struct platform_device *pdev,
1036 		   int idx, dma_cap_mask_t cap_mask, int irq)
1037 {
1038 	int ret = 0;
1039 	struct mv_xor_chan *mv_chan;
1040 	struct dma_device *dma_dev;
1041 
1042 	mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
1043 	if (!mv_chan)
1044 		return ERR_PTR(-ENOMEM);
1045 
1046 	mv_chan->idx = idx;
1047 	mv_chan->irq = irq;
1048 	if (xordev->xor_type == XOR_ORION)
1049 		mv_chan->op_in_desc = XOR_MODE_IN_REG;
1050 	else
1051 		mv_chan->op_in_desc = XOR_MODE_IN_DESC;
1052 
1053 	dma_dev = &mv_chan->dmadev;
1054 	dma_dev->dev = &pdev->dev;
1055 	mv_chan->xordev = xordev;
1056 
1057 	/*
1058 	 * These source and destination dummy buffers are used to implement
1059 	 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
1060 	 * Hence, we only need to map the buffers at initialization-time.
1061 	 */
1062 	mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
1063 		mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1064 	mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
1065 		mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1066 
1067 	/* allocate coherent memory for hardware descriptors
1068 	 * note: writecombine gives slightly better performance, but
1069 	 * requires that we explicitly flush the writes
1070 	 */
1071 	mv_chan->dma_desc_pool_virt =
1072 	  dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
1073 		       GFP_KERNEL);
1074 	if (!mv_chan->dma_desc_pool_virt)
1075 		return ERR_PTR(-ENOMEM);
1076 
1077 	/* discover transaction capabilites from the platform data */
1078 	dma_dev->cap_mask = cap_mask;
1079 
1080 	INIT_LIST_HEAD(&dma_dev->channels);
1081 
1082 	/* set base routines */
1083 	dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1084 	dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1085 	dma_dev->device_tx_status = mv_xor_status;
1086 	dma_dev->device_issue_pending = mv_xor_issue_pending;
1087 
1088 	/* set prep routines based on capability */
1089 	if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1090 		dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1091 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1092 		dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1093 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1094 		dma_dev->max_xor = 8;
1095 		dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1096 	}
1097 
1098 	mv_chan->mmr_base = xordev->xor_base;
1099 	mv_chan->mmr_high_base = xordev->xor_high_base;
1100 	tasklet_setup(&mv_chan->irq_tasklet, mv_xor_tasklet);
1101 
1102 	/* clear errors before enabling interrupts */
1103 	mv_chan_clear_err_status(mv_chan);
1104 
1105 	ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1106 			  0, dev_name(&pdev->dev), mv_chan);
1107 	if (ret)
1108 		goto err_free_dma;
1109 
1110 	mv_chan_unmask_interrupts(mv_chan);
1111 
1112 	if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1113 		mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
1114 	else
1115 		mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
1116 
1117 	spin_lock_init(&mv_chan->lock);
1118 	INIT_LIST_HEAD(&mv_chan->chain);
1119 	INIT_LIST_HEAD(&mv_chan->completed_slots);
1120 	INIT_LIST_HEAD(&mv_chan->free_slots);
1121 	INIT_LIST_HEAD(&mv_chan->allocated_slots);
1122 	mv_chan->dmachan.device = dma_dev;
1123 	dma_cookie_init(&mv_chan->dmachan);
1124 
1125 	list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1126 
1127 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1128 		ret = mv_chan_memcpy_self_test(mv_chan);
1129 		dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1130 		if (ret)
1131 			goto err_free_irq;
1132 	}
1133 
1134 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1135 		ret = mv_chan_xor_self_test(mv_chan);
1136 		dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1137 		if (ret)
1138 			goto err_free_irq;
1139 	}
1140 
1141 	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1142 		 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1143 		 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1144 		 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1145 		 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1146 
1147 	ret = dma_async_device_register(dma_dev);
1148 	if (ret)
1149 		goto err_free_irq;
1150 
1151 	return mv_chan;
1152 
1153 err_free_irq:
1154 	free_irq(mv_chan->irq, mv_chan);
1155 err_free_dma:
1156 	dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1157 			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1158 	return ERR_PTR(ret);
1159 }
1160 
1161 static void
mv_xor_conf_mbus_windows(struct mv_xor_device * xordev,const struct mbus_dram_target_info * dram)1162 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1163 			 const struct mbus_dram_target_info *dram)
1164 {
1165 	void __iomem *base = xordev->xor_high_base;
1166 	u32 win_enable = 0;
1167 	int i;
1168 
1169 	for (i = 0; i < 8; i++) {
1170 		writel(0, base + WINDOW_BASE(i));
1171 		writel(0, base + WINDOW_SIZE(i));
1172 		if (i < 4)
1173 			writel(0, base + WINDOW_REMAP_HIGH(i));
1174 	}
1175 
1176 	for (i = 0; i < dram->num_cs; i++) {
1177 		const struct mbus_dram_window *cs = dram->cs + i;
1178 
1179 		writel((cs->base & 0xffff0000) |
1180 		       (cs->mbus_attr << 8) |
1181 		       dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1182 		writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1183 
1184 		/* Fill the caching variables for later use */
1185 		xordev->win_start[i] = cs->base;
1186 		xordev->win_end[i] = cs->base + cs->size - 1;
1187 
1188 		win_enable |= (1 << i);
1189 		win_enable |= 3 << (16 + (2 * i));
1190 	}
1191 
1192 	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1193 	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1194 	writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1195 	writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1196 }
1197 
1198 static void
mv_xor_conf_mbus_windows_a3700(struct mv_xor_device * xordev)1199 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1200 {
1201 	void __iomem *base = xordev->xor_high_base;
1202 	u32 win_enable = 0;
1203 	int i;
1204 
1205 	for (i = 0; i < 8; i++) {
1206 		writel(0, base + WINDOW_BASE(i));
1207 		writel(0, base + WINDOW_SIZE(i));
1208 		if (i < 4)
1209 			writel(0, base + WINDOW_REMAP_HIGH(i));
1210 	}
1211 	/*
1212 	 * For Armada3700 open default 4GB Mbus window. The dram
1213 	 * related configuration are done at AXIS level.
1214 	 */
1215 	writel(0xffff0000, base + WINDOW_SIZE(0));
1216 	win_enable |= 1;
1217 	win_enable |= 3 << 16;
1218 
1219 	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1220 	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1221 	writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1222 	writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1223 }
1224 
1225 /*
1226  * Since this XOR driver is basically used only for RAID5, we don't
1227  * need to care about synchronizing ->suspend with DMA activity,
1228  * because the DMA engine will naturally be quiet due to the block
1229  * devices being suspended.
1230  */
mv_xor_suspend(struct platform_device * pdev,pm_message_t state)1231 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1232 {
1233 	struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1234 	int i;
1235 
1236 	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1237 		struct mv_xor_chan *mv_chan = xordev->channels[i];
1238 
1239 		if (!mv_chan)
1240 			continue;
1241 
1242 		mv_chan->saved_config_reg =
1243 			readl_relaxed(XOR_CONFIG(mv_chan));
1244 		mv_chan->saved_int_mask_reg =
1245 			readl_relaxed(XOR_INTR_MASK(mv_chan));
1246 	}
1247 
1248 	return 0;
1249 }
1250 
mv_xor_resume(struct platform_device * dev)1251 static int mv_xor_resume(struct platform_device *dev)
1252 {
1253 	struct mv_xor_device *xordev = platform_get_drvdata(dev);
1254 	const struct mbus_dram_target_info *dram;
1255 	int i;
1256 
1257 	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1258 		struct mv_xor_chan *mv_chan = xordev->channels[i];
1259 
1260 		if (!mv_chan)
1261 			continue;
1262 
1263 		writel_relaxed(mv_chan->saved_config_reg,
1264 			       XOR_CONFIG(mv_chan));
1265 		writel_relaxed(mv_chan->saved_int_mask_reg,
1266 			       XOR_INTR_MASK(mv_chan));
1267 	}
1268 
1269 	if (xordev->xor_type == XOR_ARMADA_37XX) {
1270 		mv_xor_conf_mbus_windows_a3700(xordev);
1271 		return 0;
1272 	}
1273 
1274 	dram = mv_mbus_dram_info();
1275 	if (dram)
1276 		mv_xor_conf_mbus_windows(xordev, dram);
1277 
1278 	return 0;
1279 }
1280 
1281 static const struct of_device_id mv_xor_dt_ids[] = {
1282 	{ .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1283 	{ .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
1284 	{ .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
1285 	{},
1286 };
1287 
1288 static unsigned int mv_xor_engine_count;
1289 
mv_xor_probe(struct platform_device * pdev)1290 static int mv_xor_probe(struct platform_device *pdev)
1291 {
1292 	const struct mbus_dram_target_info *dram;
1293 	struct mv_xor_device *xordev;
1294 	struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1295 	struct resource *res;
1296 	unsigned int max_engines, max_channels;
1297 	int i, ret;
1298 
1299 	dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1300 
1301 	xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1302 	if (!xordev)
1303 		return -ENOMEM;
1304 
1305 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1306 	if (!res)
1307 		return -ENODEV;
1308 
1309 	xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1310 					resource_size(res));
1311 	if (!xordev->xor_base)
1312 		return -EBUSY;
1313 
1314 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1315 	if (!res)
1316 		return -ENODEV;
1317 
1318 	xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1319 					     resource_size(res));
1320 	if (!xordev->xor_high_base)
1321 		return -EBUSY;
1322 
1323 	platform_set_drvdata(pdev, xordev);
1324 
1325 
1326 	/*
1327 	 * We need to know which type of XOR device we use before
1328 	 * setting up. In non-dt case it can only be the legacy one.
1329 	 */
1330 	xordev->xor_type = XOR_ORION;
1331 	if (pdev->dev.of_node) {
1332 		const struct of_device_id *of_id =
1333 			of_match_device(mv_xor_dt_ids,
1334 					&pdev->dev);
1335 
1336 		xordev->xor_type = (uintptr_t)of_id->data;
1337 	}
1338 
1339 	/*
1340 	 * (Re-)program MBUS remapping windows if we are asked to.
1341 	 */
1342 	if (xordev->xor_type == XOR_ARMADA_37XX) {
1343 		mv_xor_conf_mbus_windows_a3700(xordev);
1344 	} else {
1345 		dram = mv_mbus_dram_info();
1346 		if (dram)
1347 			mv_xor_conf_mbus_windows(xordev, dram);
1348 	}
1349 
1350 	/* Not all platforms can gate the clock, so it is not
1351 	 * an error if the clock does not exists.
1352 	 */
1353 	xordev->clk = clk_get(&pdev->dev, NULL);
1354 	if (!IS_ERR(xordev->clk))
1355 		clk_prepare_enable(xordev->clk);
1356 
1357 	/*
1358 	 * We don't want to have more than one channel per CPU in
1359 	 * order for async_tx to perform well. So we limit the number
1360 	 * of engines and channels so that we take into account this
1361 	 * constraint. Note that we also want to use channels from
1362 	 * separate engines when possible.  For dual-CPU Armada 3700
1363 	 * SoC with single XOR engine allow using its both channels.
1364 	 */
1365 	max_engines = num_present_cpus();
1366 	if (xordev->xor_type == XOR_ARMADA_37XX)
1367 		max_channels =	num_present_cpus();
1368 	else
1369 		max_channels = min_t(unsigned int,
1370 				     MV_XOR_MAX_CHANNELS,
1371 				     DIV_ROUND_UP(num_present_cpus(), 2));
1372 
1373 	if (mv_xor_engine_count >= max_engines)
1374 		return 0;
1375 
1376 	if (pdev->dev.of_node) {
1377 		struct device_node *np;
1378 		int i = 0;
1379 
1380 		for_each_child_of_node(pdev->dev.of_node, np) {
1381 			struct mv_xor_chan *chan;
1382 			dma_cap_mask_t cap_mask;
1383 			int irq;
1384 
1385 			if (i >= max_channels)
1386 				continue;
1387 
1388 			dma_cap_zero(cap_mask);
1389 			dma_cap_set(DMA_MEMCPY, cap_mask);
1390 			dma_cap_set(DMA_XOR, cap_mask);
1391 			dma_cap_set(DMA_INTERRUPT, cap_mask);
1392 
1393 			irq = irq_of_parse_and_map(np, 0);
1394 			if (!irq) {
1395 				ret = -ENODEV;
1396 				goto err_channel_add;
1397 			}
1398 
1399 			chan = mv_xor_channel_add(xordev, pdev, i,
1400 						  cap_mask, irq);
1401 			if (IS_ERR(chan)) {
1402 				ret = PTR_ERR(chan);
1403 				irq_dispose_mapping(irq);
1404 				goto err_channel_add;
1405 			}
1406 
1407 			xordev->channels[i] = chan;
1408 			i++;
1409 		}
1410 	} else if (pdata && pdata->channels) {
1411 		for (i = 0; i < max_channels; i++) {
1412 			struct mv_xor_channel_data *cd;
1413 			struct mv_xor_chan *chan;
1414 			int irq;
1415 
1416 			cd = &pdata->channels[i];
1417 			irq = platform_get_irq(pdev, i);
1418 			if (irq < 0) {
1419 				ret = irq;
1420 				goto err_channel_add;
1421 			}
1422 
1423 			chan = mv_xor_channel_add(xordev, pdev, i,
1424 						  cd->cap_mask, irq);
1425 			if (IS_ERR(chan)) {
1426 				ret = PTR_ERR(chan);
1427 				goto err_channel_add;
1428 			}
1429 
1430 			xordev->channels[i] = chan;
1431 		}
1432 	}
1433 
1434 	return 0;
1435 
1436 err_channel_add:
1437 	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1438 		if (xordev->channels[i]) {
1439 			mv_xor_channel_remove(xordev->channels[i]);
1440 			if (pdev->dev.of_node)
1441 				irq_dispose_mapping(xordev->channels[i]->irq);
1442 		}
1443 
1444 	if (!IS_ERR(xordev->clk)) {
1445 		clk_disable_unprepare(xordev->clk);
1446 		clk_put(xordev->clk);
1447 	}
1448 
1449 	return ret;
1450 }
1451 
1452 static struct platform_driver mv_xor_driver = {
1453 	.probe		= mv_xor_probe,
1454 	.suspend        = mv_xor_suspend,
1455 	.resume         = mv_xor_resume,
1456 	.driver		= {
1457 		.name	        = MV_XOR_NAME,
1458 		.of_match_table = mv_xor_dt_ids,
1459 	},
1460 };
1461 
1462 builtin_platform_driver(mv_xor_driver);
1463 
1464 /*
1465 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1466 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1467 MODULE_LICENSE("GPL");
1468 */
1469