xref: /openbmc/linux/drivers/dma/mv_xor.c (revision ca55b2fe)
1 /*
2  * offload engine driver for the Marvell XOR engine
3  * Copyright (C) 2007, 2008, Marvell International Ltd.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  */
14 
15 #include <linux/init.h>
16 #include <linux/slab.h>
17 #include <linux/delay.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/of_device.h>
22 #include <linux/platform_device.h>
23 #include <linux/memory.h>
24 #include <linux/clk.h>
25 #include <linux/of.h>
26 #include <linux/of_irq.h>
27 #include <linux/irqdomain.h>
28 #include <linux/cpumask.h>
29 #include <linux/platform_data/dma-mv_xor.h>
30 
31 #include "dmaengine.h"
32 #include "mv_xor.h"
33 
34 enum mv_xor_mode {
35 	XOR_MODE_IN_REG,
36 	XOR_MODE_IN_DESC,
37 };
38 
39 static void mv_xor_issue_pending(struct dma_chan *chan);
40 
41 #define to_mv_xor_chan(chan)		\
42 	container_of(chan, struct mv_xor_chan, dmachan)
43 
44 #define to_mv_xor_slot(tx)		\
45 	container_of(tx, struct mv_xor_desc_slot, async_tx)
46 
47 #define mv_chan_to_devp(chan)           \
48 	((chan)->dmadev.dev)
49 
50 static void mv_desc_init(struct mv_xor_desc_slot *desc,
51 			 dma_addr_t addr, u32 byte_count,
52 			 enum dma_ctrl_flags flags)
53 {
54 	struct mv_xor_desc *hw_desc = desc->hw_desc;
55 
56 	hw_desc->status = XOR_DESC_DMA_OWNED;
57 	hw_desc->phy_next_desc = 0;
58 	/* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
59 	hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
60 				XOR_DESC_EOD_INT_EN : 0;
61 	hw_desc->phy_dest_addr = addr;
62 	hw_desc->byte_count = byte_count;
63 }
64 
65 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
66 {
67 	struct mv_xor_desc *hw_desc = desc->hw_desc;
68 
69 	switch (desc->type) {
70 	case DMA_XOR:
71 	case DMA_INTERRUPT:
72 		hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
73 		break;
74 	case DMA_MEMCPY:
75 		hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
76 		break;
77 	default:
78 		BUG();
79 		return;
80 	}
81 }
82 
83 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
84 				  u32 next_desc_addr)
85 {
86 	struct mv_xor_desc *hw_desc = desc->hw_desc;
87 	BUG_ON(hw_desc->phy_next_desc);
88 	hw_desc->phy_next_desc = next_desc_addr;
89 }
90 
91 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
92 				 int index, dma_addr_t addr)
93 {
94 	struct mv_xor_desc *hw_desc = desc->hw_desc;
95 	hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
96 	if (desc->type == DMA_XOR)
97 		hw_desc->desc_command |= (1 << index);
98 }
99 
100 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
101 {
102 	return readl_relaxed(XOR_CURR_DESC(chan));
103 }
104 
105 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
106 					u32 next_desc_addr)
107 {
108 	writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
109 }
110 
111 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
112 {
113 	u32 val = readl_relaxed(XOR_INTR_MASK(chan));
114 	val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
115 	writel_relaxed(val, XOR_INTR_MASK(chan));
116 }
117 
118 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
119 {
120 	u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
121 	intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
122 	return intr_cause;
123 }
124 
125 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
126 {
127 	u32 val;
128 
129 	val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
130 	val = ~(val << (chan->idx * 16));
131 	dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
132 	writel_relaxed(val, XOR_INTR_CAUSE(chan));
133 }
134 
135 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
136 {
137 	u32 val = 0xFFFF0000 >> (chan->idx * 16);
138 	writel_relaxed(val, XOR_INTR_CAUSE(chan));
139 }
140 
141 static void mv_chan_set_mode(struct mv_xor_chan *chan,
142 			     enum dma_transaction_type type)
143 {
144 	u32 op_mode;
145 	u32 config = readl_relaxed(XOR_CONFIG(chan));
146 
147 	switch (type) {
148 	case DMA_XOR:
149 		op_mode = XOR_OPERATION_MODE_XOR;
150 		break;
151 	case DMA_MEMCPY:
152 		op_mode = XOR_OPERATION_MODE_MEMCPY;
153 		break;
154 	default:
155 		dev_err(mv_chan_to_devp(chan),
156 			"error: unsupported operation %d\n",
157 			type);
158 		BUG();
159 		return;
160 	}
161 
162 	config &= ~0x7;
163 	config |= op_mode;
164 
165 #if defined(__BIG_ENDIAN)
166 	config |= XOR_DESCRIPTOR_SWAP;
167 #else
168 	config &= ~XOR_DESCRIPTOR_SWAP;
169 #endif
170 
171 	writel_relaxed(config, XOR_CONFIG(chan));
172 	chan->current_type = type;
173 }
174 
175 static void mv_chan_set_mode_to_desc(struct mv_xor_chan *chan)
176 {
177 	u32 op_mode;
178 	u32 config = readl_relaxed(XOR_CONFIG(chan));
179 
180 	op_mode = XOR_OPERATION_MODE_IN_DESC;
181 
182 	config &= ~0x7;
183 	config |= op_mode;
184 
185 #if defined(__BIG_ENDIAN)
186 	config |= XOR_DESCRIPTOR_SWAP;
187 #else
188 	config &= ~XOR_DESCRIPTOR_SWAP;
189 #endif
190 
191 	writel_relaxed(config, XOR_CONFIG(chan));
192 }
193 
194 static void mv_chan_activate(struct mv_xor_chan *chan)
195 {
196 	dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
197 
198 	/* writel ensures all descriptors are flushed before activation */
199 	writel(BIT(0), XOR_ACTIVATION(chan));
200 }
201 
202 static char mv_chan_is_busy(struct mv_xor_chan *chan)
203 {
204 	u32 state = readl_relaxed(XOR_ACTIVATION(chan));
205 
206 	state = (state >> 4) & 0x3;
207 
208 	return (state == 1) ? 1 : 0;
209 }
210 
211 /*
212  * mv_chan_start_new_chain - program the engine to operate on new
213  * chain headed by sw_desc
214  * Caller must hold &mv_chan->lock while calling this function
215  */
216 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
217 				    struct mv_xor_desc_slot *sw_desc)
218 {
219 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
220 		__func__, __LINE__, sw_desc);
221 
222 	/* set the hardware chain */
223 	mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
224 
225 	mv_chan->pending++;
226 	mv_xor_issue_pending(&mv_chan->dmachan);
227 }
228 
229 static dma_cookie_t
230 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
231 				struct mv_xor_chan *mv_chan,
232 				dma_cookie_t cookie)
233 {
234 	BUG_ON(desc->async_tx.cookie < 0);
235 
236 	if (desc->async_tx.cookie > 0) {
237 		cookie = desc->async_tx.cookie;
238 
239 		/* call the callback (must not sleep or submit new
240 		 * operations to this channel)
241 		 */
242 		if (desc->async_tx.callback)
243 			desc->async_tx.callback(
244 				desc->async_tx.callback_param);
245 
246 		dma_descriptor_unmap(&desc->async_tx);
247 	}
248 
249 	/* run dependent operations */
250 	dma_run_dependencies(&desc->async_tx);
251 
252 	return cookie;
253 }
254 
255 static int
256 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
257 {
258 	struct mv_xor_desc_slot *iter, *_iter;
259 
260 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
261 	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
262 				 node) {
263 
264 		if (async_tx_test_ack(&iter->async_tx))
265 			list_move_tail(&iter->node, &mv_chan->free_slots);
266 	}
267 	return 0;
268 }
269 
270 static int
271 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
272 		   struct mv_xor_chan *mv_chan)
273 {
274 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
275 		__func__, __LINE__, desc, desc->async_tx.flags);
276 
277 	/* the client is allowed to attach dependent operations
278 	 * until 'ack' is set
279 	 */
280 	if (!async_tx_test_ack(&desc->async_tx))
281 		/* move this slot to the completed_slots */
282 		list_move_tail(&desc->node, &mv_chan->completed_slots);
283 	else
284 		list_move_tail(&desc->node, &mv_chan->free_slots);
285 
286 	return 0;
287 }
288 
289 /* This function must be called with the mv_xor_chan spinlock held */
290 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
291 {
292 	struct mv_xor_desc_slot *iter, *_iter;
293 	dma_cookie_t cookie = 0;
294 	int busy = mv_chan_is_busy(mv_chan);
295 	u32 current_desc = mv_chan_get_current_desc(mv_chan);
296 	int current_cleaned = 0;
297 	struct mv_xor_desc *hw_desc;
298 
299 	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
300 	dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
301 	mv_chan_clean_completed_slots(mv_chan);
302 
303 	/* free completed slots from the chain starting with
304 	 * the oldest descriptor
305 	 */
306 
307 	list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
308 				 node) {
309 
310 		/* clean finished descriptors */
311 		hw_desc = iter->hw_desc;
312 		if (hw_desc->status & XOR_DESC_SUCCESS) {
313 			cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
314 								 cookie);
315 
316 			/* done processing desc, clean slot */
317 			mv_desc_clean_slot(iter, mv_chan);
318 
319 			/* break if we did cleaned the current */
320 			if (iter->async_tx.phys == current_desc) {
321 				current_cleaned = 1;
322 				break;
323 			}
324 		} else {
325 			if (iter->async_tx.phys == current_desc) {
326 				current_cleaned = 0;
327 				break;
328 			}
329 		}
330 	}
331 
332 	if ((busy == 0) && !list_empty(&mv_chan->chain)) {
333 		if (current_cleaned) {
334 			/*
335 			 * current descriptor cleaned and removed, run
336 			 * from list head
337 			 */
338 			iter = list_entry(mv_chan->chain.next,
339 					  struct mv_xor_desc_slot,
340 					  node);
341 			mv_chan_start_new_chain(mv_chan, iter);
342 		} else {
343 			if (!list_is_last(&iter->node, &mv_chan->chain)) {
344 				/*
345 				 * descriptors are still waiting after
346 				 * current, trigger them
347 				 */
348 				iter = list_entry(iter->node.next,
349 						  struct mv_xor_desc_slot,
350 						  node);
351 				mv_chan_start_new_chain(mv_chan, iter);
352 			} else {
353 				/*
354 				 * some descriptors are still waiting
355 				 * to be cleaned
356 				 */
357 				tasklet_schedule(&mv_chan->irq_tasklet);
358 			}
359 		}
360 	}
361 
362 	if (cookie > 0)
363 		mv_chan->dmachan.completed_cookie = cookie;
364 }
365 
366 static void mv_xor_tasklet(unsigned long data)
367 {
368 	struct mv_xor_chan *chan = (struct mv_xor_chan *) data;
369 
370 	spin_lock_bh(&chan->lock);
371 	mv_chan_slot_cleanup(chan);
372 	spin_unlock_bh(&chan->lock);
373 }
374 
375 static struct mv_xor_desc_slot *
376 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
377 {
378 	struct mv_xor_desc_slot *iter;
379 
380 	spin_lock_bh(&mv_chan->lock);
381 
382 	if (!list_empty(&mv_chan->free_slots)) {
383 		iter = list_first_entry(&mv_chan->free_slots,
384 					struct mv_xor_desc_slot,
385 					node);
386 
387 		list_move_tail(&iter->node, &mv_chan->allocated_slots);
388 
389 		spin_unlock_bh(&mv_chan->lock);
390 
391 		/* pre-ack descriptor */
392 		async_tx_ack(&iter->async_tx);
393 		iter->async_tx.cookie = -EBUSY;
394 
395 		return iter;
396 
397 	}
398 
399 	spin_unlock_bh(&mv_chan->lock);
400 
401 	/* try to free some slots if the allocation fails */
402 	tasklet_schedule(&mv_chan->irq_tasklet);
403 
404 	return NULL;
405 }
406 
407 /************************ DMA engine API functions ****************************/
408 static dma_cookie_t
409 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
410 {
411 	struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
412 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
413 	struct mv_xor_desc_slot *old_chain_tail;
414 	dma_cookie_t cookie;
415 	int new_hw_chain = 1;
416 
417 	dev_dbg(mv_chan_to_devp(mv_chan),
418 		"%s sw_desc %p: async_tx %p\n",
419 		__func__, sw_desc, &sw_desc->async_tx);
420 
421 	spin_lock_bh(&mv_chan->lock);
422 	cookie = dma_cookie_assign(tx);
423 
424 	if (list_empty(&mv_chan->chain))
425 		list_move_tail(&sw_desc->node, &mv_chan->chain);
426 	else {
427 		new_hw_chain = 0;
428 
429 		old_chain_tail = list_entry(mv_chan->chain.prev,
430 					    struct mv_xor_desc_slot,
431 					    node);
432 		list_move_tail(&sw_desc->node, &mv_chan->chain);
433 
434 		dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
435 			&old_chain_tail->async_tx.phys);
436 
437 		/* fix up the hardware chain */
438 		mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
439 
440 		/* if the channel is not busy */
441 		if (!mv_chan_is_busy(mv_chan)) {
442 			u32 current_desc = mv_chan_get_current_desc(mv_chan);
443 			/*
444 			 * and the curren desc is the end of the chain before
445 			 * the append, then we need to start the channel
446 			 */
447 			if (current_desc == old_chain_tail->async_tx.phys)
448 				new_hw_chain = 1;
449 		}
450 	}
451 
452 	if (new_hw_chain)
453 		mv_chan_start_new_chain(mv_chan, sw_desc);
454 
455 	spin_unlock_bh(&mv_chan->lock);
456 
457 	return cookie;
458 }
459 
460 /* returns the number of allocated descriptors */
461 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
462 {
463 	void *virt_desc;
464 	dma_addr_t dma_desc;
465 	int idx;
466 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
467 	struct mv_xor_desc_slot *slot = NULL;
468 	int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
469 
470 	/* Allocate descriptor slots */
471 	idx = mv_chan->slots_allocated;
472 	while (idx < num_descs_in_pool) {
473 		slot = kzalloc(sizeof(*slot), GFP_KERNEL);
474 		if (!slot) {
475 			dev_info(mv_chan_to_devp(mv_chan),
476 				 "channel only initialized %d descriptor slots",
477 				 idx);
478 			break;
479 		}
480 		virt_desc = mv_chan->dma_desc_pool_virt;
481 		slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
482 
483 		dma_async_tx_descriptor_init(&slot->async_tx, chan);
484 		slot->async_tx.tx_submit = mv_xor_tx_submit;
485 		INIT_LIST_HEAD(&slot->node);
486 		dma_desc = mv_chan->dma_desc_pool;
487 		slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
488 		slot->idx = idx++;
489 
490 		spin_lock_bh(&mv_chan->lock);
491 		mv_chan->slots_allocated = idx;
492 		list_add_tail(&slot->node, &mv_chan->free_slots);
493 		spin_unlock_bh(&mv_chan->lock);
494 	}
495 
496 	dev_dbg(mv_chan_to_devp(mv_chan),
497 		"allocated %d descriptor slots\n",
498 		mv_chan->slots_allocated);
499 
500 	return mv_chan->slots_allocated ? : -ENOMEM;
501 }
502 
503 static struct dma_async_tx_descriptor *
504 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
505 		    unsigned int src_cnt, size_t len, unsigned long flags)
506 {
507 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
508 	struct mv_xor_desc_slot *sw_desc;
509 
510 	if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
511 		return NULL;
512 
513 	BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
514 
515 	dev_dbg(mv_chan_to_devp(mv_chan),
516 		"%s src_cnt: %d len: %u dest %pad flags: %ld\n",
517 		__func__, src_cnt, len, &dest, flags);
518 
519 	sw_desc = mv_chan_alloc_slot(mv_chan);
520 	if (sw_desc) {
521 		sw_desc->type = DMA_XOR;
522 		sw_desc->async_tx.flags = flags;
523 		mv_desc_init(sw_desc, dest, len, flags);
524 		if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
525 			mv_desc_set_mode(sw_desc);
526 		while (src_cnt--)
527 			mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
528 	}
529 
530 	dev_dbg(mv_chan_to_devp(mv_chan),
531 		"%s sw_desc %p async_tx %p \n",
532 		__func__, sw_desc, &sw_desc->async_tx);
533 	return sw_desc ? &sw_desc->async_tx : NULL;
534 }
535 
536 static struct dma_async_tx_descriptor *
537 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
538 		size_t len, unsigned long flags)
539 {
540 	/*
541 	 * A MEMCPY operation is identical to an XOR operation with only
542 	 * a single source address.
543 	 */
544 	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
545 }
546 
547 static struct dma_async_tx_descriptor *
548 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
549 {
550 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
551 	dma_addr_t src, dest;
552 	size_t len;
553 
554 	src = mv_chan->dummy_src_addr;
555 	dest = mv_chan->dummy_dst_addr;
556 	len = MV_XOR_MIN_BYTE_COUNT;
557 
558 	/*
559 	 * We implement the DMA_INTERRUPT operation as a minimum sized
560 	 * XOR operation with a single dummy source address.
561 	 */
562 	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
563 }
564 
565 static void mv_xor_free_chan_resources(struct dma_chan *chan)
566 {
567 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
568 	struct mv_xor_desc_slot *iter, *_iter;
569 	int in_use_descs = 0;
570 
571 	spin_lock_bh(&mv_chan->lock);
572 
573 	mv_chan_slot_cleanup(mv_chan);
574 
575 	list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
576 					node) {
577 		in_use_descs++;
578 		list_move_tail(&iter->node, &mv_chan->free_slots);
579 	}
580 	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
581 				 node) {
582 		in_use_descs++;
583 		list_move_tail(&iter->node, &mv_chan->free_slots);
584 	}
585 	list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
586 				 node) {
587 		in_use_descs++;
588 		list_move_tail(&iter->node, &mv_chan->free_slots);
589 	}
590 	list_for_each_entry_safe_reverse(
591 		iter, _iter, &mv_chan->free_slots, node) {
592 		list_del(&iter->node);
593 		kfree(iter);
594 		mv_chan->slots_allocated--;
595 	}
596 
597 	dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
598 		__func__, mv_chan->slots_allocated);
599 	spin_unlock_bh(&mv_chan->lock);
600 
601 	if (in_use_descs)
602 		dev_err(mv_chan_to_devp(mv_chan),
603 			"freeing %d in use descriptors!\n", in_use_descs);
604 }
605 
606 /**
607  * mv_xor_status - poll the status of an XOR transaction
608  * @chan: XOR channel handle
609  * @cookie: XOR transaction identifier
610  * @txstate: XOR transactions state holder (or NULL)
611  */
612 static enum dma_status mv_xor_status(struct dma_chan *chan,
613 					  dma_cookie_t cookie,
614 					  struct dma_tx_state *txstate)
615 {
616 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
617 	enum dma_status ret;
618 
619 	ret = dma_cookie_status(chan, cookie, txstate);
620 	if (ret == DMA_COMPLETE)
621 		return ret;
622 
623 	spin_lock_bh(&mv_chan->lock);
624 	mv_chan_slot_cleanup(mv_chan);
625 	spin_unlock_bh(&mv_chan->lock);
626 
627 	return dma_cookie_status(chan, cookie, txstate);
628 }
629 
630 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
631 {
632 	u32 val;
633 
634 	val = readl_relaxed(XOR_CONFIG(chan));
635 	dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
636 
637 	val = readl_relaxed(XOR_ACTIVATION(chan));
638 	dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
639 
640 	val = readl_relaxed(XOR_INTR_CAUSE(chan));
641 	dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
642 
643 	val = readl_relaxed(XOR_INTR_MASK(chan));
644 	dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
645 
646 	val = readl_relaxed(XOR_ERROR_CAUSE(chan));
647 	dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
648 
649 	val = readl_relaxed(XOR_ERROR_ADDR(chan));
650 	dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
651 }
652 
653 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
654 					  u32 intr_cause)
655 {
656 	if (intr_cause & XOR_INT_ERR_DECODE) {
657 		dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
658 		return;
659 	}
660 
661 	dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
662 		chan->idx, intr_cause);
663 
664 	mv_chan_dump_regs(chan);
665 	WARN_ON(1);
666 }
667 
668 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
669 {
670 	struct mv_xor_chan *chan = data;
671 	u32 intr_cause = mv_chan_get_intr_cause(chan);
672 
673 	dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
674 
675 	if (intr_cause & XOR_INTR_ERRORS)
676 		mv_chan_err_interrupt_handler(chan, intr_cause);
677 
678 	tasklet_schedule(&chan->irq_tasklet);
679 
680 	mv_chan_clear_eoc_cause(chan);
681 
682 	return IRQ_HANDLED;
683 }
684 
685 static void mv_xor_issue_pending(struct dma_chan *chan)
686 {
687 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
688 
689 	if (mv_chan->pending >= MV_XOR_THRESHOLD) {
690 		mv_chan->pending = 0;
691 		mv_chan_activate(mv_chan);
692 	}
693 }
694 
695 /*
696  * Perform a transaction to verify the HW works.
697  */
698 
699 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
700 {
701 	int i, ret;
702 	void *src, *dest;
703 	dma_addr_t src_dma, dest_dma;
704 	struct dma_chan *dma_chan;
705 	dma_cookie_t cookie;
706 	struct dma_async_tx_descriptor *tx;
707 	struct dmaengine_unmap_data *unmap;
708 	int err = 0;
709 
710 	src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
711 	if (!src)
712 		return -ENOMEM;
713 
714 	dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
715 	if (!dest) {
716 		kfree(src);
717 		return -ENOMEM;
718 	}
719 
720 	/* Fill in src buffer */
721 	for (i = 0; i < PAGE_SIZE; i++)
722 		((u8 *) src)[i] = (u8)i;
723 
724 	dma_chan = &mv_chan->dmachan;
725 	if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
726 		err = -ENODEV;
727 		goto out;
728 	}
729 
730 	unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
731 	if (!unmap) {
732 		err = -ENOMEM;
733 		goto free_resources;
734 	}
735 
736 	src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 0,
737 				 PAGE_SIZE, DMA_TO_DEVICE);
738 	unmap->addr[0] = src_dma;
739 
740 	ret = dma_mapping_error(dma_chan->device->dev, src_dma);
741 	if (ret) {
742 		err = -ENOMEM;
743 		goto free_resources;
744 	}
745 	unmap->to_cnt = 1;
746 
747 	dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 0,
748 				  PAGE_SIZE, DMA_FROM_DEVICE);
749 	unmap->addr[1] = dest_dma;
750 
751 	ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
752 	if (ret) {
753 		err = -ENOMEM;
754 		goto free_resources;
755 	}
756 	unmap->from_cnt = 1;
757 	unmap->len = PAGE_SIZE;
758 
759 	tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
760 				    PAGE_SIZE, 0);
761 	if (!tx) {
762 		dev_err(dma_chan->device->dev,
763 			"Self-test cannot prepare operation, disabling\n");
764 		err = -ENODEV;
765 		goto free_resources;
766 	}
767 
768 	cookie = mv_xor_tx_submit(tx);
769 	if (dma_submit_error(cookie)) {
770 		dev_err(dma_chan->device->dev,
771 			"Self-test submit error, disabling\n");
772 		err = -ENODEV;
773 		goto free_resources;
774 	}
775 
776 	mv_xor_issue_pending(dma_chan);
777 	async_tx_ack(tx);
778 	msleep(1);
779 
780 	if (mv_xor_status(dma_chan, cookie, NULL) !=
781 	    DMA_COMPLETE) {
782 		dev_err(dma_chan->device->dev,
783 			"Self-test copy timed out, disabling\n");
784 		err = -ENODEV;
785 		goto free_resources;
786 	}
787 
788 	dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
789 				PAGE_SIZE, DMA_FROM_DEVICE);
790 	if (memcmp(src, dest, PAGE_SIZE)) {
791 		dev_err(dma_chan->device->dev,
792 			"Self-test copy failed compare, disabling\n");
793 		err = -ENODEV;
794 		goto free_resources;
795 	}
796 
797 free_resources:
798 	dmaengine_unmap_put(unmap);
799 	mv_xor_free_chan_resources(dma_chan);
800 out:
801 	kfree(src);
802 	kfree(dest);
803 	return err;
804 }
805 
806 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
807 static int
808 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
809 {
810 	int i, src_idx, ret;
811 	struct page *dest;
812 	struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
813 	dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
814 	dma_addr_t dest_dma;
815 	struct dma_async_tx_descriptor *tx;
816 	struct dmaengine_unmap_data *unmap;
817 	struct dma_chan *dma_chan;
818 	dma_cookie_t cookie;
819 	u8 cmp_byte = 0;
820 	u32 cmp_word;
821 	int err = 0;
822 	int src_count = MV_XOR_NUM_SRC_TEST;
823 
824 	for (src_idx = 0; src_idx < src_count; src_idx++) {
825 		xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
826 		if (!xor_srcs[src_idx]) {
827 			while (src_idx--)
828 				__free_page(xor_srcs[src_idx]);
829 			return -ENOMEM;
830 		}
831 	}
832 
833 	dest = alloc_page(GFP_KERNEL);
834 	if (!dest) {
835 		while (src_idx--)
836 			__free_page(xor_srcs[src_idx]);
837 		return -ENOMEM;
838 	}
839 
840 	/* Fill in src buffers */
841 	for (src_idx = 0; src_idx < src_count; src_idx++) {
842 		u8 *ptr = page_address(xor_srcs[src_idx]);
843 		for (i = 0; i < PAGE_SIZE; i++)
844 			ptr[i] = (1 << src_idx);
845 	}
846 
847 	for (src_idx = 0; src_idx < src_count; src_idx++)
848 		cmp_byte ^= (u8) (1 << src_idx);
849 
850 	cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
851 		(cmp_byte << 8) | cmp_byte;
852 
853 	memset(page_address(dest), 0, PAGE_SIZE);
854 
855 	dma_chan = &mv_chan->dmachan;
856 	if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
857 		err = -ENODEV;
858 		goto out;
859 	}
860 
861 	unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
862 					 GFP_KERNEL);
863 	if (!unmap) {
864 		err = -ENOMEM;
865 		goto free_resources;
866 	}
867 
868 	/* test xor */
869 	for (i = 0; i < src_count; i++) {
870 		unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
871 					      0, PAGE_SIZE, DMA_TO_DEVICE);
872 		dma_srcs[i] = unmap->addr[i];
873 		ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
874 		if (ret) {
875 			err = -ENOMEM;
876 			goto free_resources;
877 		}
878 		unmap->to_cnt++;
879 	}
880 
881 	unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
882 				      DMA_FROM_DEVICE);
883 	dest_dma = unmap->addr[src_count];
884 	ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
885 	if (ret) {
886 		err = -ENOMEM;
887 		goto free_resources;
888 	}
889 	unmap->from_cnt = 1;
890 	unmap->len = PAGE_SIZE;
891 
892 	tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
893 				 src_count, PAGE_SIZE, 0);
894 	if (!tx) {
895 		dev_err(dma_chan->device->dev,
896 			"Self-test cannot prepare operation, disabling\n");
897 		err = -ENODEV;
898 		goto free_resources;
899 	}
900 
901 	cookie = mv_xor_tx_submit(tx);
902 	if (dma_submit_error(cookie)) {
903 		dev_err(dma_chan->device->dev,
904 			"Self-test submit error, disabling\n");
905 		err = -ENODEV;
906 		goto free_resources;
907 	}
908 
909 	mv_xor_issue_pending(dma_chan);
910 	async_tx_ack(tx);
911 	msleep(8);
912 
913 	if (mv_xor_status(dma_chan, cookie, NULL) !=
914 	    DMA_COMPLETE) {
915 		dev_err(dma_chan->device->dev,
916 			"Self-test xor timed out, disabling\n");
917 		err = -ENODEV;
918 		goto free_resources;
919 	}
920 
921 	dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
922 				PAGE_SIZE, DMA_FROM_DEVICE);
923 	for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
924 		u32 *ptr = page_address(dest);
925 		if (ptr[i] != cmp_word) {
926 			dev_err(dma_chan->device->dev,
927 				"Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
928 				i, ptr[i], cmp_word);
929 			err = -ENODEV;
930 			goto free_resources;
931 		}
932 	}
933 
934 free_resources:
935 	dmaengine_unmap_put(unmap);
936 	mv_xor_free_chan_resources(dma_chan);
937 out:
938 	src_idx = src_count;
939 	while (src_idx--)
940 		__free_page(xor_srcs[src_idx]);
941 	__free_page(dest);
942 	return err;
943 }
944 
945 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
946 {
947 	struct dma_chan *chan, *_chan;
948 	struct device *dev = mv_chan->dmadev.dev;
949 
950 	dma_async_device_unregister(&mv_chan->dmadev);
951 
952 	dma_free_coherent(dev, MV_XOR_POOL_SIZE,
953 			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
954 	dma_unmap_single(dev, mv_chan->dummy_src_addr,
955 			 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
956 	dma_unmap_single(dev, mv_chan->dummy_dst_addr,
957 			 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
958 
959 	list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
960 				 device_node) {
961 		list_del(&chan->device_node);
962 	}
963 
964 	free_irq(mv_chan->irq, mv_chan);
965 
966 	return 0;
967 }
968 
969 static struct mv_xor_chan *
970 mv_xor_channel_add(struct mv_xor_device *xordev,
971 		   struct platform_device *pdev,
972 		   int idx, dma_cap_mask_t cap_mask, int irq, int op_in_desc)
973 {
974 	int ret = 0;
975 	struct mv_xor_chan *mv_chan;
976 	struct dma_device *dma_dev;
977 
978 	mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
979 	if (!mv_chan)
980 		return ERR_PTR(-ENOMEM);
981 
982 	mv_chan->idx = idx;
983 	mv_chan->irq = irq;
984 	mv_chan->op_in_desc = op_in_desc;
985 
986 	dma_dev = &mv_chan->dmadev;
987 
988 	/*
989 	 * These source and destination dummy buffers are used to implement
990 	 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
991 	 * Hence, we only need to map the buffers at initialization-time.
992 	 */
993 	mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
994 		mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
995 	mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
996 		mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
997 
998 	/* allocate coherent memory for hardware descriptors
999 	 * note: writecombine gives slightly better performance, but
1000 	 * requires that we explicitly flush the writes
1001 	 */
1002 	mv_chan->dma_desc_pool_virt =
1003 	  dma_alloc_writecombine(&pdev->dev, MV_XOR_POOL_SIZE,
1004 				 &mv_chan->dma_desc_pool, GFP_KERNEL);
1005 	if (!mv_chan->dma_desc_pool_virt)
1006 		return ERR_PTR(-ENOMEM);
1007 
1008 	/* discover transaction capabilites from the platform data */
1009 	dma_dev->cap_mask = cap_mask;
1010 
1011 	INIT_LIST_HEAD(&dma_dev->channels);
1012 
1013 	/* set base routines */
1014 	dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1015 	dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1016 	dma_dev->device_tx_status = mv_xor_status;
1017 	dma_dev->device_issue_pending = mv_xor_issue_pending;
1018 	dma_dev->dev = &pdev->dev;
1019 
1020 	/* set prep routines based on capability */
1021 	if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1022 		dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1023 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1024 		dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1025 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1026 		dma_dev->max_xor = 8;
1027 		dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1028 	}
1029 
1030 	mv_chan->mmr_base = xordev->xor_base;
1031 	mv_chan->mmr_high_base = xordev->xor_high_base;
1032 	tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1033 		     mv_chan);
1034 
1035 	/* clear errors before enabling interrupts */
1036 	mv_chan_clear_err_status(mv_chan);
1037 
1038 	ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1039 			  0, dev_name(&pdev->dev), mv_chan);
1040 	if (ret)
1041 		goto err_free_dma;
1042 
1043 	mv_chan_unmask_interrupts(mv_chan);
1044 
1045 	if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1046 		mv_chan_set_mode_to_desc(mv_chan);
1047 	else
1048 		mv_chan_set_mode(mv_chan, DMA_XOR);
1049 
1050 	spin_lock_init(&mv_chan->lock);
1051 	INIT_LIST_HEAD(&mv_chan->chain);
1052 	INIT_LIST_HEAD(&mv_chan->completed_slots);
1053 	INIT_LIST_HEAD(&mv_chan->free_slots);
1054 	INIT_LIST_HEAD(&mv_chan->allocated_slots);
1055 	mv_chan->dmachan.device = dma_dev;
1056 	dma_cookie_init(&mv_chan->dmachan);
1057 
1058 	list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1059 
1060 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1061 		ret = mv_chan_memcpy_self_test(mv_chan);
1062 		dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1063 		if (ret)
1064 			goto err_free_irq;
1065 	}
1066 
1067 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1068 		ret = mv_chan_xor_self_test(mv_chan);
1069 		dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1070 		if (ret)
1071 			goto err_free_irq;
1072 	}
1073 
1074 	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1075 		 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1076 		 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1077 		 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1078 		 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1079 
1080 	dma_async_device_register(dma_dev);
1081 	return mv_chan;
1082 
1083 err_free_irq:
1084 	free_irq(mv_chan->irq, mv_chan);
1085  err_free_dma:
1086 	dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1087 			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1088 	return ERR_PTR(ret);
1089 }
1090 
1091 static void
1092 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1093 			 const struct mbus_dram_target_info *dram)
1094 {
1095 	void __iomem *base = xordev->xor_high_base;
1096 	u32 win_enable = 0;
1097 	int i;
1098 
1099 	for (i = 0; i < 8; i++) {
1100 		writel(0, base + WINDOW_BASE(i));
1101 		writel(0, base + WINDOW_SIZE(i));
1102 		if (i < 4)
1103 			writel(0, base + WINDOW_REMAP_HIGH(i));
1104 	}
1105 
1106 	for (i = 0; i < dram->num_cs; i++) {
1107 		const struct mbus_dram_window *cs = dram->cs + i;
1108 
1109 		writel((cs->base & 0xffff0000) |
1110 		       (cs->mbus_attr << 8) |
1111 		       dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1112 		writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1113 
1114 		win_enable |= (1 << i);
1115 		win_enable |= 3 << (16 + (2 * i));
1116 	}
1117 
1118 	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1119 	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1120 	writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1121 	writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1122 }
1123 
1124 static const struct of_device_id mv_xor_dt_ids[] = {
1125 	{ .compatible = "marvell,orion-xor", .data = (void *)XOR_MODE_IN_REG },
1126 	{ .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC },
1127 	{},
1128 };
1129 
1130 static unsigned int mv_xor_engine_count;
1131 
1132 static int mv_xor_probe(struct platform_device *pdev)
1133 {
1134 	const struct mbus_dram_target_info *dram;
1135 	struct mv_xor_device *xordev;
1136 	struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1137 	struct resource *res;
1138 	unsigned int max_engines, max_channels;
1139 	int i, ret;
1140 	int op_in_desc;
1141 
1142 	dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1143 
1144 	xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1145 	if (!xordev)
1146 		return -ENOMEM;
1147 
1148 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1149 	if (!res)
1150 		return -ENODEV;
1151 
1152 	xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1153 					resource_size(res));
1154 	if (!xordev->xor_base)
1155 		return -EBUSY;
1156 
1157 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1158 	if (!res)
1159 		return -ENODEV;
1160 
1161 	xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1162 					     resource_size(res));
1163 	if (!xordev->xor_high_base)
1164 		return -EBUSY;
1165 
1166 	platform_set_drvdata(pdev, xordev);
1167 
1168 	/*
1169 	 * (Re-)program MBUS remapping windows if we are asked to.
1170 	 */
1171 	dram = mv_mbus_dram_info();
1172 	if (dram)
1173 		mv_xor_conf_mbus_windows(xordev, dram);
1174 
1175 	/* Not all platforms can gate the clock, so it is not
1176 	 * an error if the clock does not exists.
1177 	 */
1178 	xordev->clk = clk_get(&pdev->dev, NULL);
1179 	if (!IS_ERR(xordev->clk))
1180 		clk_prepare_enable(xordev->clk);
1181 
1182 	/*
1183 	 * We don't want to have more than one channel per CPU in
1184 	 * order for async_tx to perform well. So we limit the number
1185 	 * of engines and channels so that we take into account this
1186 	 * constraint. Note that we also want to use channels from
1187 	 * separate engines when possible.
1188 	 */
1189 	max_engines = num_present_cpus();
1190 	max_channels = min_t(unsigned int,
1191 			     MV_XOR_MAX_CHANNELS,
1192 			     DIV_ROUND_UP(num_present_cpus(), 2));
1193 
1194 	if (mv_xor_engine_count >= max_engines)
1195 		return 0;
1196 
1197 	if (pdev->dev.of_node) {
1198 		struct device_node *np;
1199 		int i = 0;
1200 		const struct of_device_id *of_id =
1201 			of_match_device(mv_xor_dt_ids,
1202 					&pdev->dev);
1203 
1204 		for_each_child_of_node(pdev->dev.of_node, np) {
1205 			struct mv_xor_chan *chan;
1206 			dma_cap_mask_t cap_mask;
1207 			int irq;
1208 			op_in_desc = (int)of_id->data;
1209 
1210 			if (i >= max_channels)
1211 				continue;
1212 
1213 			dma_cap_zero(cap_mask);
1214 			dma_cap_set(DMA_MEMCPY, cap_mask);
1215 			dma_cap_set(DMA_XOR, cap_mask);
1216 			dma_cap_set(DMA_INTERRUPT, cap_mask);
1217 
1218 			irq = irq_of_parse_and_map(np, 0);
1219 			if (!irq) {
1220 				ret = -ENODEV;
1221 				goto err_channel_add;
1222 			}
1223 
1224 			chan = mv_xor_channel_add(xordev, pdev, i,
1225 						  cap_mask, irq, op_in_desc);
1226 			if (IS_ERR(chan)) {
1227 				ret = PTR_ERR(chan);
1228 				irq_dispose_mapping(irq);
1229 				goto err_channel_add;
1230 			}
1231 
1232 			xordev->channels[i] = chan;
1233 			i++;
1234 		}
1235 	} else if (pdata && pdata->channels) {
1236 		for (i = 0; i < max_channels; i++) {
1237 			struct mv_xor_channel_data *cd;
1238 			struct mv_xor_chan *chan;
1239 			int irq;
1240 
1241 			cd = &pdata->channels[i];
1242 			if (!cd) {
1243 				ret = -ENODEV;
1244 				goto err_channel_add;
1245 			}
1246 
1247 			irq = platform_get_irq(pdev, i);
1248 			if (irq < 0) {
1249 				ret = irq;
1250 				goto err_channel_add;
1251 			}
1252 
1253 			chan = mv_xor_channel_add(xordev, pdev, i,
1254 						  cd->cap_mask, irq,
1255 						  XOR_MODE_IN_REG);
1256 			if (IS_ERR(chan)) {
1257 				ret = PTR_ERR(chan);
1258 				goto err_channel_add;
1259 			}
1260 
1261 			xordev->channels[i] = chan;
1262 		}
1263 	}
1264 
1265 	return 0;
1266 
1267 err_channel_add:
1268 	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1269 		if (xordev->channels[i]) {
1270 			mv_xor_channel_remove(xordev->channels[i]);
1271 			if (pdev->dev.of_node)
1272 				irq_dispose_mapping(xordev->channels[i]->irq);
1273 		}
1274 
1275 	if (!IS_ERR(xordev->clk)) {
1276 		clk_disable_unprepare(xordev->clk);
1277 		clk_put(xordev->clk);
1278 	}
1279 
1280 	return ret;
1281 }
1282 
1283 static struct platform_driver mv_xor_driver = {
1284 	.probe		= mv_xor_probe,
1285 	.driver		= {
1286 		.name	        = MV_XOR_NAME,
1287 		.of_match_table = of_match_ptr(mv_xor_dt_ids),
1288 	},
1289 };
1290 
1291 
1292 static int __init mv_xor_init(void)
1293 {
1294 	return platform_driver_register(&mv_xor_driver);
1295 }
1296 device_initcall(mv_xor_init);
1297 
1298 /*
1299 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1300 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1301 MODULE_LICENSE("GPL");
1302 */
1303