xref: /openbmc/linux/drivers/dma/ppc4xx/adma.c (revision 897500c7)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2006-2009 DENX Software Engineering.
4  *
5  * Author: Yuri Tikhonov <yur@emcraft.com>
6  *
7  * Further porting to arch/powerpc by
8  * 	Anatolij Gustschin <agust@denx.de>
9  */
10 
11 /*
12  * This driver supports the asynchrounous DMA copy and RAID engines available
13  * on the AMCC PPC440SPe Processors.
14  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
15  * ADMA driver written by D.Williams.
16  */
17 
18 #include <linux/init.h>
19 #include <linux/module.h>
20 #include <linux/async_tx.h>
21 #include <linux/delay.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/spinlock.h>
24 #include <linux/interrupt.h>
25 #include <linux/slab.h>
26 #include <linux/uaccess.h>
27 #include <linux/proc_fs.h>
28 #include <linux/of.h>
29 #include <linux/of_address.h>
30 #include <linux/of_irq.h>
31 #include <linux/platform_device.h>
32 #include <asm/dcr.h>
33 #include <asm/dcr-regs.h>
34 #include "adma.h"
35 #include "../dmaengine.h"
36 
37 enum ppc_adma_init_code {
38 	PPC_ADMA_INIT_OK = 0,
39 	PPC_ADMA_INIT_MEMRES,
40 	PPC_ADMA_INIT_MEMREG,
41 	PPC_ADMA_INIT_ALLOC,
42 	PPC_ADMA_INIT_COHERENT,
43 	PPC_ADMA_INIT_CHANNEL,
44 	PPC_ADMA_INIT_IRQ1,
45 	PPC_ADMA_INIT_IRQ2,
46 	PPC_ADMA_INIT_REGISTER
47 };
48 
49 static char *ppc_adma_errors[] = {
50 	[PPC_ADMA_INIT_OK] = "ok",
51 	[PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
52 	[PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
53 	[PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
54 				"structure",
55 	[PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
56 				   "hardware descriptors",
57 	[PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
58 	[PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
59 	[PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
60 	[PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
61 };
62 
63 static enum ppc_adma_init_code
64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
65 
66 struct ppc_dma_chan_ref {
67 	struct dma_chan *chan;
68 	struct list_head node;
69 };
70 
71 /* The list of channels exported by ppc440spe ADMA */
72 static struct list_head
73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
74 
75 /* This flag is set when want to refetch the xor chain in the interrupt
76  * handler
77  */
78 static u32 do_xor_refetch;
79 
80 /* Pointer to DMA0, DMA1 CP/CS FIFO */
81 static void *ppc440spe_dma_fifo_buf;
82 
83 /* Pointers to last submitted to DMA0, DMA1 CDBs */
84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
86 
87 /* Pointer to last linked and submitted xor CB */
88 static struct ppc440spe_adma_desc_slot *xor_last_linked;
89 static struct ppc440spe_adma_desc_slot *xor_last_submit;
90 
91 /* This array is used in data-check operations for storing a pattern */
92 static char ppc440spe_qword[16];
93 
94 static atomic_t ppc440spe_adma_err_irq_ref;
95 static dcr_host_t ppc440spe_mq_dcr_host;
96 static unsigned int ppc440spe_mq_dcr_len;
97 
98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
99  * the block size in transactions, then we do not allow to activate more than
100  * only one RXOR transactions simultaneously. So use this var to store
101  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
102  * set) or not (PPC440SPE_RXOR_RUN is clear).
103  */
104 static unsigned long ppc440spe_rxor_state;
105 
106 /* These are used in enable & check routines
107  */
108 static u32 ppc440spe_r6_enabled;
109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
110 static struct completion ppc440spe_r6_test_comp;
111 
112 static int ppc440spe_adma_dma2rxor_prep_src(
113 		struct ppc440spe_adma_desc_slot *desc,
114 		struct ppc440spe_rxor *cursor, int index,
115 		int src_cnt, u32 addr);
116 static void ppc440spe_adma_dma2rxor_set_src(
117 		struct ppc440spe_adma_desc_slot *desc,
118 		int index, dma_addr_t addr);
119 static void ppc440spe_adma_dma2rxor_set_mult(
120 		struct ppc440spe_adma_desc_slot *desc,
121 		int index, u8 mult);
122 
123 #ifdef ADMA_LL_DEBUG
124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
125 #else
126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
127 #endif
128 
print_cb(struct ppc440spe_adma_chan * chan,void * block)129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
130 {
131 	struct dma_cdb *cdb;
132 	struct xor_cb *cb;
133 	int i;
134 
135 	switch (chan->device->id) {
136 	case 0:
137 	case 1:
138 		cdb = block;
139 
140 		pr_debug("CDB at %p [%d]:\n"
141 			"\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
142 			"\t sg1u 0x%08x sg1l 0x%08x\n"
143 			"\t sg2u 0x%08x sg2l 0x%08x\n"
144 			"\t sg3u 0x%08x sg3l 0x%08x\n",
145 			cdb, chan->device->id,
146 			cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
147 			le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
148 			le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
149 			le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
150 		);
151 		break;
152 	case 2:
153 		cb = block;
154 
155 		pr_debug("CB at %p [%d]:\n"
156 			"\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
157 			"\t cbtah 0x%08x cbtal 0x%08x\n"
158 			"\t cblah 0x%08x cblal 0x%08x\n",
159 			cb, chan->device->id,
160 			cb->cbc, cb->cbbc, cb->cbs,
161 			cb->cbtah, cb->cbtal,
162 			cb->cblah, cb->cblal);
163 		for (i = 0; i < 16; i++) {
164 			if (i && !cb->ops[i].h && !cb->ops[i].l)
165 				continue;
166 			pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
167 				i, cb->ops[i].h, cb->ops[i].l);
168 		}
169 		break;
170 	}
171 }
172 
print_cb_list(struct ppc440spe_adma_chan * chan,struct ppc440spe_adma_desc_slot * iter)173 static void print_cb_list(struct ppc440spe_adma_chan *chan,
174 			  struct ppc440spe_adma_desc_slot *iter)
175 {
176 	for (; iter; iter = iter->hw_next)
177 		print_cb(chan, iter->hw_desc);
178 }
179 
prep_dma_xor_dbg(int id,dma_addr_t dst,dma_addr_t * src,unsigned int src_cnt)180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
181 			     unsigned int src_cnt)
182 {
183 	int i;
184 
185 	pr_debug("\n%s(%d):\nsrc: ", __func__, id);
186 	for (i = 0; i < src_cnt; i++)
187 		pr_debug("\t0x%016llx ", src[i]);
188 	pr_debug("dst:\n\t0x%016llx\n", dst);
189 }
190 
prep_dma_pq_dbg(int id,dma_addr_t * dst,dma_addr_t * src,unsigned int src_cnt)191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
192 			    unsigned int src_cnt)
193 {
194 	int i;
195 
196 	pr_debug("\n%s(%d):\nsrc: ", __func__, id);
197 	for (i = 0; i < src_cnt; i++)
198 		pr_debug("\t0x%016llx ", src[i]);
199 	pr_debug("dst: ");
200 	for (i = 0; i < 2; i++)
201 		pr_debug("\t0x%016llx ", dst[i]);
202 }
203 
prep_dma_pqzero_sum_dbg(int id,dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf)204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
205 				    unsigned int src_cnt,
206 				    const unsigned char *scf)
207 {
208 	int i;
209 
210 	pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
211 	if (scf) {
212 		for (i = 0; i < src_cnt; i++)
213 			pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
214 	} else {
215 		for (i = 0; i < src_cnt; i++)
216 			pr_debug("\t0x%016llx(no) ", src[i]);
217 	}
218 
219 	pr_debug("dst: ");
220 	for (i = 0; i < 2; i++)
221 		pr_debug("\t0x%016llx ", src[src_cnt + i]);
222 }
223 
224 /******************************************************************************
225  * Command (Descriptor) Blocks low-level routines
226  ******************************************************************************/
227 /**
228  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
229  * pseudo operation
230  */
ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan)231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
232 					  struct ppc440spe_adma_chan *chan)
233 {
234 	struct xor_cb *p;
235 
236 	switch (chan->device->id) {
237 	case PPC440SPE_XOR_ID:
238 		p = desc->hw_desc;
239 		memset(desc->hw_desc, 0, sizeof(struct xor_cb));
240 		/* NOP with Command Block Complete Enable */
241 		p->cbc = XOR_CBCR_CBCE_BIT;
242 		break;
243 	case PPC440SPE_DMA0_ID:
244 	case PPC440SPE_DMA1_ID:
245 		memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
246 		/* NOP with interrupt */
247 		set_bit(PPC440SPE_DESC_INT, &desc->flags);
248 		break;
249 	default:
250 		printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
251 				__func__);
252 		break;
253 	}
254 }
255 
256 /**
257  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
258  * pseudo operation
259  */
ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot * desc)260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
261 {
262 	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
263 	desc->hw_next = NULL;
264 	desc->src_cnt = 0;
265 	desc->dst_cnt = 1;
266 }
267 
268 /**
269  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
270  */
ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot * desc,int src_cnt,unsigned long flags)271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
272 					 int src_cnt, unsigned long flags)
273 {
274 	struct xor_cb *hw_desc = desc->hw_desc;
275 
276 	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
277 	desc->hw_next = NULL;
278 	desc->src_cnt = src_cnt;
279 	desc->dst_cnt = 1;
280 
281 	hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
282 	if (flags & DMA_PREP_INTERRUPT)
283 		/* Enable interrupt on completion */
284 		hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
285 }
286 
287 /**
288  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
289  * operation in DMA2 controller
290  */
ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot * desc,int dst_cnt,int src_cnt,unsigned long flags)291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
292 		int dst_cnt, int src_cnt, unsigned long flags)
293 {
294 	struct xor_cb *hw_desc = desc->hw_desc;
295 
296 	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
297 	desc->hw_next = NULL;
298 	desc->src_cnt = src_cnt;
299 	desc->dst_cnt = dst_cnt;
300 	memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
301 	desc->descs_per_op = 0;
302 
303 	hw_desc->cbc = XOR_CBCR_TGT_BIT;
304 	if (flags & DMA_PREP_INTERRUPT)
305 		/* Enable interrupt on completion */
306 		hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
307 }
308 
309 #define DMA_CTRL_FLAGS_LAST	DMA_PREP_FENCE
310 #define DMA_PREP_ZERO_P		(DMA_CTRL_FLAGS_LAST << 1)
311 #define DMA_PREP_ZERO_Q		(DMA_PREP_ZERO_P << 1)
312 
313 /**
314  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
315  * with DMA0/1
316  */
ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot * desc,int dst_cnt,int src_cnt,unsigned long flags,unsigned long op)317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
318 				int dst_cnt, int src_cnt, unsigned long flags,
319 				unsigned long op)
320 {
321 	struct dma_cdb *hw_desc;
322 	struct ppc440spe_adma_desc_slot *iter;
323 	u8 dopc;
324 
325 	/* Common initialization of a PQ descriptors chain */
326 	set_bits(op, &desc->flags);
327 	desc->src_cnt = src_cnt;
328 	desc->dst_cnt = dst_cnt;
329 
330 	/* WXOR MULTICAST if both P and Q are being computed
331 	 * MV_SG1_SG2 if Q only
332 	 */
333 	dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
334 		DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
335 
336 	list_for_each_entry(iter, &desc->group_list, chain_node) {
337 		hw_desc = iter->hw_desc;
338 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
339 
340 		if (likely(!list_is_last(&iter->chain_node,
341 				&desc->group_list))) {
342 			/* set 'next' pointer */
343 			iter->hw_next = list_entry(iter->chain_node.next,
344 				struct ppc440spe_adma_desc_slot, chain_node);
345 			clear_bit(PPC440SPE_DESC_INT, &iter->flags);
346 		} else {
347 			/* this is the last descriptor.
348 			 * this slot will be pasted from ADMA level
349 			 * each time it wants to configure parameters
350 			 * of the transaction (src, dst, ...)
351 			 */
352 			iter->hw_next = NULL;
353 			if (flags & DMA_PREP_INTERRUPT)
354 				set_bit(PPC440SPE_DESC_INT, &iter->flags);
355 			else
356 				clear_bit(PPC440SPE_DESC_INT, &iter->flags);
357 		}
358 	}
359 
360 	/* Set OPS depending on WXOR/RXOR type of operation */
361 	if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
362 		/* This is a WXOR only chain:
363 		 * - first descriptors are for zeroing destinations
364 		 *   if PPC440SPE_ZERO_P/Q set;
365 		 * - descriptors remained are for GF-XOR operations.
366 		 */
367 		iter = list_first_entry(&desc->group_list,
368 					struct ppc440spe_adma_desc_slot,
369 					chain_node);
370 
371 		if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
372 			hw_desc = iter->hw_desc;
373 			hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
374 			iter = list_first_entry(&iter->chain_node,
375 					struct ppc440spe_adma_desc_slot,
376 					chain_node);
377 		}
378 
379 		if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
380 			hw_desc = iter->hw_desc;
381 			hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
382 			iter = list_first_entry(&iter->chain_node,
383 					struct ppc440spe_adma_desc_slot,
384 					chain_node);
385 		}
386 
387 		list_for_each_entry_from(iter, &desc->group_list, chain_node) {
388 			hw_desc = iter->hw_desc;
389 			hw_desc->opc = dopc;
390 		}
391 	} else {
392 		/* This is either RXOR-only or mixed RXOR/WXOR */
393 
394 		/* The first 1 or 2 slots in chain are always RXOR,
395 		 * if need to calculate P & Q, then there are two
396 		 * RXOR slots; if only P or only Q, then there is one
397 		 */
398 		iter = list_first_entry(&desc->group_list,
399 					struct ppc440spe_adma_desc_slot,
400 					chain_node);
401 		hw_desc = iter->hw_desc;
402 		hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
403 
404 		if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
405 			iter = list_first_entry(&iter->chain_node,
406 						struct ppc440spe_adma_desc_slot,
407 						chain_node);
408 			hw_desc = iter->hw_desc;
409 			hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
410 		}
411 
412 		/* The remaining descs (if any) are WXORs */
413 		if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
414 			iter = list_first_entry(&iter->chain_node,
415 						struct ppc440spe_adma_desc_slot,
416 						chain_node);
417 			list_for_each_entry_from(iter, &desc->group_list,
418 						chain_node) {
419 				hw_desc = iter->hw_desc;
420 				hw_desc->opc = dopc;
421 			}
422 		}
423 	}
424 }
425 
426 /**
427  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
428  * for PQ_ZERO_SUM operation
429  */
ppc440spe_desc_init_dma01pqzero_sum(struct ppc440spe_adma_desc_slot * desc,int dst_cnt,int src_cnt)430 static void ppc440spe_desc_init_dma01pqzero_sum(
431 				struct ppc440spe_adma_desc_slot *desc,
432 				int dst_cnt, int src_cnt)
433 {
434 	struct dma_cdb *hw_desc;
435 	struct ppc440spe_adma_desc_slot *iter;
436 	int i = 0;
437 	u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
438 				   DMA_CDB_OPC_MV_SG1_SG2;
439 	/*
440 	 * Initialize starting from 2nd or 3rd descriptor dependent
441 	 * on dst_cnt. First one or two slots are for cloning P
442 	 * and/or Q to chan->pdest and/or chan->qdest as we have
443 	 * to preserve original P/Q.
444 	 */
445 	iter = list_first_entry(&desc->group_list,
446 				struct ppc440spe_adma_desc_slot, chain_node);
447 	iter = list_entry(iter->chain_node.next,
448 			  struct ppc440spe_adma_desc_slot, chain_node);
449 
450 	if (dst_cnt > 1) {
451 		iter = list_entry(iter->chain_node.next,
452 				  struct ppc440spe_adma_desc_slot, chain_node);
453 	}
454 	/* initialize each source descriptor in chain */
455 	list_for_each_entry_from(iter, &desc->group_list, chain_node) {
456 		hw_desc = iter->hw_desc;
457 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
458 		iter->src_cnt = 0;
459 		iter->dst_cnt = 0;
460 
461 		/* This is a ZERO_SUM operation:
462 		 * - <src_cnt> descriptors starting from 2nd or 3rd
463 		 *   descriptor are for GF-XOR operations;
464 		 * - remaining <dst_cnt> descriptors are for checking the result
465 		 */
466 		if (i++ < src_cnt)
467 			/* MV_SG1_SG2 if only Q is being verified
468 			 * MULTICAST if both P and Q are being verified
469 			 */
470 			hw_desc->opc = dopc;
471 		else
472 			/* DMA_CDB_OPC_DCHECK128 operation */
473 			hw_desc->opc = DMA_CDB_OPC_DCHECK128;
474 
475 		if (likely(!list_is_last(&iter->chain_node,
476 					 &desc->group_list))) {
477 			/* set 'next' pointer */
478 			iter->hw_next = list_entry(iter->chain_node.next,
479 						struct ppc440spe_adma_desc_slot,
480 						chain_node);
481 		} else {
482 			/* this is the last descriptor.
483 			 * this slot will be pasted from ADMA level
484 			 * each time it wants to configure parameters
485 			 * of the transaction (src, dst, ...)
486 			 */
487 			iter->hw_next = NULL;
488 			/* always enable interrupt generation since we get
489 			 * the status of pqzero from the handler
490 			 */
491 			set_bit(PPC440SPE_DESC_INT, &iter->flags);
492 		}
493 	}
494 	desc->src_cnt = src_cnt;
495 	desc->dst_cnt = dst_cnt;
496 }
497 
498 /**
499  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
500  */
ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot * desc,unsigned long flags)501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
502 					unsigned long flags)
503 {
504 	struct dma_cdb *hw_desc = desc->hw_desc;
505 
506 	memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
507 	desc->hw_next = NULL;
508 	desc->src_cnt = 1;
509 	desc->dst_cnt = 1;
510 
511 	if (flags & DMA_PREP_INTERRUPT)
512 		set_bit(PPC440SPE_DESC_INT, &desc->flags);
513 	else
514 		clear_bit(PPC440SPE_DESC_INT, &desc->flags);
515 
516 	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
517 }
518 
519 /**
520  * ppc440spe_desc_set_src_addr - set source address into the descriptor
521  */
ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan,int src_idx,dma_addr_t addrh,dma_addr_t addrl)522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
523 					struct ppc440spe_adma_chan *chan,
524 					int src_idx, dma_addr_t addrh,
525 					dma_addr_t addrl)
526 {
527 	struct dma_cdb *dma_hw_desc;
528 	struct xor_cb *xor_hw_desc;
529 	phys_addr_t addr64, tmplow, tmphi;
530 
531 	switch (chan->device->id) {
532 	case PPC440SPE_DMA0_ID:
533 	case PPC440SPE_DMA1_ID:
534 		if (!addrh) {
535 			addr64 = addrl;
536 			tmphi = (addr64 >> 32);
537 			tmplow = (addr64 & 0xFFFFFFFF);
538 		} else {
539 			tmphi = addrh;
540 			tmplow = addrl;
541 		}
542 		dma_hw_desc = desc->hw_desc;
543 		dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
544 		dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
545 		break;
546 	case PPC440SPE_XOR_ID:
547 		xor_hw_desc = desc->hw_desc;
548 		xor_hw_desc->ops[src_idx].l = addrl;
549 		xor_hw_desc->ops[src_idx].h |= addrh;
550 		break;
551 	}
552 }
553 
554 /**
555  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
556  */
ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan,u32 mult_index,int sg_index,unsigned char mult_value)557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
558 			struct ppc440spe_adma_chan *chan, u32 mult_index,
559 			int sg_index, unsigned char mult_value)
560 {
561 	struct dma_cdb *dma_hw_desc;
562 	u32 *psgu;
563 
564 	switch (chan->device->id) {
565 	case PPC440SPE_DMA0_ID:
566 	case PPC440SPE_DMA1_ID:
567 		dma_hw_desc = desc->hw_desc;
568 
569 		switch (sg_index) {
570 		/* for RXOR operations set multiplier
571 		 * into source cued address
572 		 */
573 		case DMA_CDB_SG_SRC:
574 			psgu = &dma_hw_desc->sg1u;
575 			break;
576 		/* for WXOR operations set multiplier
577 		 * into destination cued address(es)
578 		 */
579 		case DMA_CDB_SG_DST1:
580 			psgu = &dma_hw_desc->sg2u;
581 			break;
582 		case DMA_CDB_SG_DST2:
583 			psgu = &dma_hw_desc->sg3u;
584 			break;
585 		default:
586 			BUG();
587 		}
588 
589 		*psgu |= cpu_to_le32(mult_value << mult_index);
590 		break;
591 	case PPC440SPE_XOR_ID:
592 		break;
593 	default:
594 		BUG();
595 	}
596 }
597 
598 /**
599  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
600  */
ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan,dma_addr_t addrh,dma_addr_t addrl,u32 dst_idx)601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
602 				struct ppc440spe_adma_chan *chan,
603 				dma_addr_t addrh, dma_addr_t addrl,
604 				u32 dst_idx)
605 {
606 	struct dma_cdb *dma_hw_desc;
607 	struct xor_cb *xor_hw_desc;
608 	phys_addr_t addr64, tmphi, tmplow;
609 	u32 *psgu, *psgl;
610 
611 	switch (chan->device->id) {
612 	case PPC440SPE_DMA0_ID:
613 	case PPC440SPE_DMA1_ID:
614 		if (!addrh) {
615 			addr64 = addrl;
616 			tmphi = (addr64 >> 32);
617 			tmplow = (addr64 & 0xFFFFFFFF);
618 		} else {
619 			tmphi = addrh;
620 			tmplow = addrl;
621 		}
622 		dma_hw_desc = desc->hw_desc;
623 
624 		psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
625 		psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
626 
627 		*psgl = cpu_to_le32((u32)tmplow);
628 		*psgu |= cpu_to_le32((u32)tmphi);
629 		break;
630 	case PPC440SPE_XOR_ID:
631 		xor_hw_desc = desc->hw_desc;
632 		xor_hw_desc->cbtal = addrl;
633 		xor_hw_desc->cbtah |= addrh;
634 		break;
635 	}
636 }
637 
638 /**
639  * ppc440spe_desc_set_byte_count - set number of data bytes involved
640  * into the operation
641  */
ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan,u32 byte_count)642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
643 				struct ppc440spe_adma_chan *chan,
644 				u32 byte_count)
645 {
646 	struct dma_cdb *dma_hw_desc;
647 	struct xor_cb *xor_hw_desc;
648 
649 	switch (chan->device->id) {
650 	case PPC440SPE_DMA0_ID:
651 	case PPC440SPE_DMA1_ID:
652 		dma_hw_desc = desc->hw_desc;
653 		dma_hw_desc->cnt = cpu_to_le32(byte_count);
654 		break;
655 	case PPC440SPE_XOR_ID:
656 		xor_hw_desc = desc->hw_desc;
657 		xor_hw_desc->cbbc = byte_count;
658 		break;
659 	}
660 }
661 
662 /**
663  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
664  */
ppc440spe_desc_set_rxor_block_size(u32 byte_count)665 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
666 {
667 	/* assume that byte_count is aligned on the 512-boundary;
668 	 * thus write it directly to the register (bits 23:31 are
669 	 * reserved there).
670 	 */
671 	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
672 }
673 
674 /**
675  * ppc440spe_desc_set_dcheck - set CHECK pattern
676  */
ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan,u8 * qword)677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
678 				struct ppc440spe_adma_chan *chan, u8 *qword)
679 {
680 	struct dma_cdb *dma_hw_desc;
681 
682 	switch (chan->device->id) {
683 	case PPC440SPE_DMA0_ID:
684 	case PPC440SPE_DMA1_ID:
685 		dma_hw_desc = desc->hw_desc;
686 		iowrite32(qword[0], &dma_hw_desc->sg3l);
687 		iowrite32(qword[4], &dma_hw_desc->sg3u);
688 		iowrite32(qword[8], &dma_hw_desc->sg2l);
689 		iowrite32(qword[12], &dma_hw_desc->sg2u);
690 		break;
691 	default:
692 		BUG();
693 	}
694 }
695 
696 /**
697  * ppc440spe_xor_set_link - set link address in xor CB
698  */
ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot * prev_desc,struct ppc440spe_adma_desc_slot * next_desc)699 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
700 				struct ppc440spe_adma_desc_slot *next_desc)
701 {
702 	struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
703 
704 	if (unlikely(!next_desc || !(next_desc->phys))) {
705 		printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
706 			__func__, next_desc,
707 			next_desc ? next_desc->phys : 0);
708 		BUG();
709 	}
710 
711 	xor_hw_desc->cbs = 0;
712 	xor_hw_desc->cblal = next_desc->phys;
713 	xor_hw_desc->cblah = 0;
714 	xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
715 }
716 
717 /**
718  * ppc440spe_desc_set_link - set the address of descriptor following this
719  * descriptor in chain
720  */
ppc440spe_desc_set_link(struct ppc440spe_adma_chan * chan,struct ppc440spe_adma_desc_slot * prev_desc,struct ppc440spe_adma_desc_slot * next_desc)721 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
722 				struct ppc440spe_adma_desc_slot *prev_desc,
723 				struct ppc440spe_adma_desc_slot *next_desc)
724 {
725 	unsigned long flags;
726 	struct ppc440spe_adma_desc_slot *tail = next_desc;
727 
728 	if (unlikely(!prev_desc || !next_desc ||
729 		(prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
730 		/* If previous next is overwritten something is wrong.
731 		 * though we may refetch from append to initiate list
732 		 * processing; in this case - it's ok.
733 		 */
734 		printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
735 			"prev->hw_next=0x%p\n", __func__, prev_desc,
736 			next_desc, prev_desc ? prev_desc->hw_next : 0);
737 		BUG();
738 	}
739 
740 	local_irq_save(flags);
741 
742 	/* do s/w chaining both for DMA and XOR descriptors */
743 	prev_desc->hw_next = next_desc;
744 
745 	switch (chan->device->id) {
746 	case PPC440SPE_DMA0_ID:
747 	case PPC440SPE_DMA1_ID:
748 		break;
749 	case PPC440SPE_XOR_ID:
750 		/* bind descriptor to the chain */
751 		while (tail->hw_next)
752 			tail = tail->hw_next;
753 		xor_last_linked = tail;
754 
755 		if (prev_desc == xor_last_submit)
756 			/* do not link to the last submitted CB */
757 			break;
758 		ppc440spe_xor_set_link(prev_desc, next_desc);
759 		break;
760 	}
761 
762 	local_irq_restore(flags);
763 }
764 
765 /**
766  * ppc440spe_desc_get_link - get the address of the descriptor that
767  * follows this one
768  */
ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan)769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
770 					struct ppc440spe_adma_chan *chan)
771 {
772 	if (!desc->hw_next)
773 		return 0;
774 
775 	return desc->hw_next->phys;
776 }
777 
778 /**
779  * ppc440spe_desc_is_aligned - check alignment
780  */
ppc440spe_desc_is_aligned(struct ppc440spe_adma_desc_slot * desc,int num_slots)781 static inline int ppc440spe_desc_is_aligned(
782 	struct ppc440spe_adma_desc_slot *desc, int num_slots)
783 {
784 	return (desc->idx & (num_slots - 1)) ? 0 : 1;
785 }
786 
787 /**
788  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
789  * XOR operation
790  */
ppc440spe_chan_xor_slot_count(size_t len,int src_cnt,int * slots_per_op)791 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
792 			int *slots_per_op)
793 {
794 	int slot_cnt;
795 
796 	/* each XOR descriptor provides up to 16 source operands */
797 	slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
798 
799 	if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
800 		return slot_cnt;
801 
802 	printk(KERN_ERR "%s: len %d > max %d !!\n",
803 		__func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
804 	BUG();
805 	return slot_cnt;
806 }
807 
808 /**
809  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
810  * DMA2 PQ operation
811  */
ppc440spe_dma2_pq_slot_count(dma_addr_t * srcs,int src_cnt,size_t len)812 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
813 		int src_cnt, size_t len)
814 {
815 	signed long long order = 0;
816 	int state = 0;
817 	int addr_count = 0;
818 	int i;
819 	for (i = 1; i < src_cnt; i++) {
820 		dma_addr_t cur_addr = srcs[i];
821 		dma_addr_t old_addr = srcs[i-1];
822 		switch (state) {
823 		case 0:
824 			if (cur_addr == old_addr + len) {
825 				/* direct RXOR */
826 				order = 1;
827 				state = 1;
828 				if (i == src_cnt-1)
829 					addr_count++;
830 			} else if (old_addr == cur_addr + len) {
831 				/* reverse RXOR */
832 				order = -1;
833 				state = 1;
834 				if (i == src_cnt-1)
835 					addr_count++;
836 			} else {
837 				state = 3;
838 			}
839 			break;
840 		case 1:
841 			if (i == src_cnt-2 || (order == -1
842 				&& cur_addr != old_addr - len)) {
843 				order = 0;
844 				state = 0;
845 				addr_count++;
846 			} else if (cur_addr == old_addr + len*order) {
847 				state = 2;
848 				if (i == src_cnt-1)
849 					addr_count++;
850 			} else if (cur_addr == old_addr + 2*len) {
851 				state = 2;
852 				if (i == src_cnt-1)
853 					addr_count++;
854 			} else if (cur_addr == old_addr + 3*len) {
855 				state = 2;
856 				if (i == src_cnt-1)
857 					addr_count++;
858 			} else {
859 				order = 0;
860 				state = 0;
861 				addr_count++;
862 			}
863 			break;
864 		case 2:
865 			order = 0;
866 			state = 0;
867 			addr_count++;
868 				break;
869 		}
870 		if (state == 3)
871 			break;
872 	}
873 	if (src_cnt <= 1 || (state != 1 && state != 2)) {
874 		pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
875 			__func__, src_cnt, state, addr_count, order);
876 		for (i = 0; i < src_cnt; i++)
877 			pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
878 		BUG();
879 	}
880 
881 	return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
882 }
883 
884 
885 /******************************************************************************
886  * ADMA channel low-level routines
887  ******************************************************************************/
888 
889 static u32
890 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
891 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
892 
893 /**
894  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
895  */
ppc440spe_adma_device_clear_eot_status(struct ppc440spe_adma_chan * chan)896 static void ppc440spe_adma_device_clear_eot_status(
897 					struct ppc440spe_adma_chan *chan)
898 {
899 	struct dma_regs *dma_reg;
900 	struct xor_regs *xor_reg;
901 	u8 *p = chan->device->dma_desc_pool_virt;
902 	struct dma_cdb *cdb;
903 	u32 rv, i;
904 
905 	switch (chan->device->id) {
906 	case PPC440SPE_DMA0_ID:
907 	case PPC440SPE_DMA1_ID:
908 		/* read FIFO to ack */
909 		dma_reg = chan->device->dma_reg;
910 		while ((rv = ioread32(&dma_reg->csfpl))) {
911 			i = rv & DMA_CDB_ADDR_MSK;
912 			cdb = (struct dma_cdb *)&p[i -
913 			    (u32)chan->device->dma_desc_pool];
914 
915 			/* Clear opcode to ack. This is necessary for
916 			 * ZeroSum operations only
917 			 */
918 			cdb->opc = 0;
919 
920 			if (test_bit(PPC440SPE_RXOR_RUN,
921 			    &ppc440spe_rxor_state)) {
922 				/* probably this is a completed RXOR op,
923 				 * get pointer to CDB using the fact that
924 				 * physical and virtual addresses of CDB
925 				 * in pools have the same offsets
926 				 */
927 				if (le32_to_cpu(cdb->sg1u) &
928 				    DMA_CUED_XOR_BASE) {
929 					/* this is a RXOR */
930 					clear_bit(PPC440SPE_RXOR_RUN,
931 						  &ppc440spe_rxor_state);
932 				}
933 			}
934 
935 			if (rv & DMA_CDB_STATUS_MSK) {
936 				/* ZeroSum check failed
937 				 */
938 				struct ppc440spe_adma_desc_slot *iter;
939 				dma_addr_t phys = rv & ~DMA_CDB_MSK;
940 
941 				/*
942 				 * Update the status of corresponding
943 				 * descriptor.
944 				 */
945 				list_for_each_entry(iter, &chan->chain,
946 				    chain_node) {
947 					if (iter->phys == phys)
948 						break;
949 				}
950 				/*
951 				 * if cannot find the corresponding
952 				 * slot it's a bug
953 				 */
954 				BUG_ON(&iter->chain_node == &chan->chain);
955 
956 				if (iter->xor_check_result) {
957 					if (test_bit(PPC440SPE_DESC_PCHECK,
958 						     &iter->flags)) {
959 						*iter->xor_check_result |=
960 							SUM_CHECK_P_RESULT;
961 					} else
962 					if (test_bit(PPC440SPE_DESC_QCHECK,
963 						     &iter->flags)) {
964 						*iter->xor_check_result |=
965 							SUM_CHECK_Q_RESULT;
966 					} else
967 						BUG();
968 				}
969 			}
970 		}
971 
972 		rv = ioread32(&dma_reg->dsts);
973 		if (rv) {
974 			pr_err("DMA%d err status: 0x%x\n",
975 			       chan->device->id, rv);
976 			/* write back to clear */
977 			iowrite32(rv, &dma_reg->dsts);
978 		}
979 		break;
980 	case PPC440SPE_XOR_ID:
981 		/* reset status bits to ack */
982 		xor_reg = chan->device->xor_reg;
983 		rv = ioread32be(&xor_reg->sr);
984 		iowrite32be(rv, &xor_reg->sr);
985 
986 		if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
987 			if (rv & XOR_IE_RPTIE_BIT) {
988 				/* Read PLB Timeout Error.
989 				 * Try to resubmit the CB
990 				 */
991 				u32 val = ioread32be(&xor_reg->ccbalr);
992 
993 				iowrite32be(val, &xor_reg->cblalr);
994 
995 				val = ioread32be(&xor_reg->crsr);
996 				iowrite32be(val | XOR_CRSR_XAE_BIT,
997 					    &xor_reg->crsr);
998 			} else
999 				pr_err("XOR ERR 0x%x status\n", rv);
1000 			break;
1001 		}
1002 
1003 		/*  if the XORcore is idle, but there are unprocessed CBs
1004 		 * then refetch the s/w chain here
1005 		 */
1006 		if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1007 		    do_xor_refetch)
1008 			ppc440spe_chan_append(chan);
1009 		break;
1010 	}
1011 }
1012 
1013 /**
1014  * ppc440spe_chan_is_busy - get the channel status
1015  */
ppc440spe_chan_is_busy(struct ppc440spe_adma_chan * chan)1016 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1017 {
1018 	struct dma_regs *dma_reg;
1019 	struct xor_regs *xor_reg;
1020 	int busy = 0;
1021 
1022 	switch (chan->device->id) {
1023 	case PPC440SPE_DMA0_ID:
1024 	case PPC440SPE_DMA1_ID:
1025 		dma_reg = chan->device->dma_reg;
1026 		/*  if command FIFO's head and tail pointers are equal and
1027 		 * status tail is the same as command, then channel is free
1028 		 */
1029 		if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1030 		    ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1031 			busy = 1;
1032 		break;
1033 	case PPC440SPE_XOR_ID:
1034 		/* use the special status bit for the XORcore
1035 		 */
1036 		xor_reg = chan->device->xor_reg;
1037 		busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1038 		break;
1039 	}
1040 
1041 	return busy;
1042 }
1043 
1044 /**
1045  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1046  */
ppc440spe_chan_set_first_xor_descriptor(struct ppc440spe_adma_chan * chan,struct ppc440spe_adma_desc_slot * next_desc)1047 static void ppc440spe_chan_set_first_xor_descriptor(
1048 				struct ppc440spe_adma_chan *chan,
1049 				struct ppc440spe_adma_desc_slot *next_desc)
1050 {
1051 	struct xor_regs *xor_reg = chan->device->xor_reg;
1052 
1053 	if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1054 		printk(KERN_INFO "%s: Warn: XORcore is running "
1055 			"when try to set the first CDB!\n",
1056 			__func__);
1057 
1058 	xor_last_submit = xor_last_linked = next_desc;
1059 
1060 	iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1061 
1062 	iowrite32be(next_desc->phys, &xor_reg->cblalr);
1063 	iowrite32be(0, &xor_reg->cblahr);
1064 	iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1065 		    &xor_reg->cbcr);
1066 
1067 	chan->hw_chain_inited = 1;
1068 }
1069 
1070 /**
1071  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1072  * called with irqs disabled
1073  */
ppc440spe_dma_put_desc(struct ppc440spe_adma_chan * chan,struct ppc440spe_adma_desc_slot * desc)1074 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1075 		struct ppc440spe_adma_desc_slot *desc)
1076 {
1077 	u32 pcdb;
1078 	struct dma_regs *dma_reg = chan->device->dma_reg;
1079 
1080 	pcdb = desc->phys;
1081 	if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1082 		pcdb |= DMA_CDB_NO_INT;
1083 
1084 	chan_last_sub[chan->device->id] = desc;
1085 
1086 	ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1087 
1088 	iowrite32(pcdb, &dma_reg->cpfpl);
1089 }
1090 
1091 /**
1092  * ppc440spe_chan_append - update the h/w chain in the channel
1093  */
ppc440spe_chan_append(struct ppc440spe_adma_chan * chan)1094 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1095 {
1096 	struct xor_regs *xor_reg;
1097 	struct ppc440spe_adma_desc_slot *iter;
1098 	struct xor_cb *xcb;
1099 	u32 cur_desc;
1100 	unsigned long flags;
1101 
1102 	local_irq_save(flags);
1103 
1104 	switch (chan->device->id) {
1105 	case PPC440SPE_DMA0_ID:
1106 	case PPC440SPE_DMA1_ID:
1107 		cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1108 
1109 		if (likely(cur_desc)) {
1110 			iter = chan_last_sub[chan->device->id];
1111 			BUG_ON(!iter);
1112 		} else {
1113 			/* first peer */
1114 			iter = chan_first_cdb[chan->device->id];
1115 			BUG_ON(!iter);
1116 			ppc440spe_dma_put_desc(chan, iter);
1117 			chan->hw_chain_inited = 1;
1118 		}
1119 
1120 		/* is there something new to append */
1121 		if (!iter->hw_next)
1122 			break;
1123 
1124 		/* flush descriptors from the s/w queue to fifo */
1125 		list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1126 			ppc440spe_dma_put_desc(chan, iter);
1127 			if (!iter->hw_next)
1128 				break;
1129 		}
1130 		break;
1131 	case PPC440SPE_XOR_ID:
1132 		/* update h/w links and refetch */
1133 		if (!xor_last_submit->hw_next)
1134 			break;
1135 
1136 		xor_reg = chan->device->xor_reg;
1137 		/* the last linked CDB has to generate an interrupt
1138 		 * that we'd be able to append the next lists to h/w
1139 		 * regardless of the XOR engine state at the moment of
1140 		 * appending of these next lists
1141 		 */
1142 		xcb = xor_last_linked->hw_desc;
1143 		xcb->cbc |= XOR_CBCR_CBCE_BIT;
1144 
1145 		if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1146 			/* XORcore is idle. Refetch now */
1147 			do_xor_refetch = 0;
1148 			ppc440spe_xor_set_link(xor_last_submit,
1149 				xor_last_submit->hw_next);
1150 
1151 			ADMA_LL_DBG(print_cb_list(chan,
1152 				xor_last_submit->hw_next));
1153 
1154 			xor_last_submit = xor_last_linked;
1155 			iowrite32be(ioread32be(&xor_reg->crsr) |
1156 				    XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1157 				    &xor_reg->crsr);
1158 		} else {
1159 			/* XORcore is running. Refetch later in the handler */
1160 			do_xor_refetch = 1;
1161 		}
1162 
1163 		break;
1164 	}
1165 
1166 	local_irq_restore(flags);
1167 }
1168 
1169 /**
1170  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1171  */
1172 static u32
ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan * chan)1173 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1174 {
1175 	struct dma_regs *dma_reg;
1176 	struct xor_regs *xor_reg;
1177 
1178 	if (unlikely(!chan->hw_chain_inited))
1179 		/* h/w descriptor chain is not initialized yet */
1180 		return 0;
1181 
1182 	switch (chan->device->id) {
1183 	case PPC440SPE_DMA0_ID:
1184 	case PPC440SPE_DMA1_ID:
1185 		dma_reg = chan->device->dma_reg;
1186 		return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1187 	case PPC440SPE_XOR_ID:
1188 		xor_reg = chan->device->xor_reg;
1189 		return ioread32be(&xor_reg->ccbalr);
1190 	}
1191 	return 0;
1192 }
1193 
1194 /**
1195  * ppc440spe_chan_run - enable the channel
1196  */
ppc440spe_chan_run(struct ppc440spe_adma_chan * chan)1197 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1198 {
1199 	struct xor_regs *xor_reg;
1200 
1201 	switch (chan->device->id) {
1202 	case PPC440SPE_DMA0_ID:
1203 	case PPC440SPE_DMA1_ID:
1204 		/* DMAs are always enabled, do nothing */
1205 		break;
1206 	case PPC440SPE_XOR_ID:
1207 		/* drain write buffer */
1208 		xor_reg = chan->device->xor_reg;
1209 
1210 		/* fetch descriptor pointed to in <link> */
1211 		iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1212 			    &xor_reg->crsr);
1213 		break;
1214 	}
1215 }
1216 
1217 /******************************************************************************
1218  * ADMA device level
1219  ******************************************************************************/
1220 
1221 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1222 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1223 
1224 static dma_cookie_t
1225 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1226 
1227 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1228 				    dma_addr_t addr, int index);
1229 static void
1230 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1231 				  dma_addr_t addr, int index);
1232 
1233 static void
1234 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1235 			   dma_addr_t *paddr, unsigned long flags);
1236 static void
1237 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1238 			  dma_addr_t addr, int index);
1239 static void
1240 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1241 			       unsigned char mult, int index, int dst_pos);
1242 static void
1243 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1244 				   dma_addr_t paddr, dma_addr_t qaddr);
1245 
1246 static struct page *ppc440spe_rxor_srcs[32];
1247 
1248 /**
1249  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1250  */
ppc440spe_can_rxor(struct page ** srcs,int src_cnt,size_t len)1251 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1252 {
1253 	int i, order = 0, state = 0;
1254 	int idx = 0;
1255 
1256 	if (unlikely(!(src_cnt > 1)))
1257 		return 0;
1258 
1259 	BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1260 
1261 	/* Skip holes in the source list before checking */
1262 	for (i = 0; i < src_cnt; i++) {
1263 		if (!srcs[i])
1264 			continue;
1265 		ppc440spe_rxor_srcs[idx++] = srcs[i];
1266 	}
1267 	src_cnt = idx;
1268 
1269 	for (i = 1; i < src_cnt; i++) {
1270 		char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1271 		char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1272 
1273 		switch (state) {
1274 		case 0:
1275 			if (cur_addr == old_addr + len) {
1276 				/* direct RXOR */
1277 				order = 1;
1278 				state = 1;
1279 			} else if (old_addr == cur_addr + len) {
1280 				/* reverse RXOR */
1281 				order = -1;
1282 				state = 1;
1283 			} else
1284 				goto out;
1285 			break;
1286 		case 1:
1287 			if ((i == src_cnt - 2) ||
1288 			    (order == -1 && cur_addr != old_addr - len)) {
1289 				order = 0;
1290 				state = 0;
1291 			} else if ((cur_addr == old_addr + len * order) ||
1292 				   (cur_addr == old_addr + 2 * len) ||
1293 				   (cur_addr == old_addr + 3 * len)) {
1294 				state = 2;
1295 			} else {
1296 				order = 0;
1297 				state = 0;
1298 			}
1299 			break;
1300 		case 2:
1301 			order = 0;
1302 			state = 0;
1303 			break;
1304 		}
1305 	}
1306 
1307 out:
1308 	if (state == 1 || state == 2)
1309 		return 1;
1310 
1311 	return 0;
1312 }
1313 
1314 /**
1315  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1316  *	the operation given on this channel. It's assumed that 'chan' is
1317  *	capable to process 'cap' type of operation.
1318  * @chan: channel to use
1319  * @cap: type of transaction
1320  * @dst_lst: array of destination pointers
1321  * @dst_cnt: number of destination operands
1322  * @src_lst: array of source pointers
1323  * @src_cnt: number of source operands
1324  * @src_sz: size of each source operand
1325  */
ppc440spe_adma_estimate(struct dma_chan * chan,enum dma_transaction_type cap,struct page ** dst_lst,int dst_cnt,struct page ** src_lst,int src_cnt,size_t src_sz)1326 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1327 	enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1328 	struct page **src_lst, int src_cnt, size_t src_sz)
1329 {
1330 	int ef = 1;
1331 
1332 	if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1333 		/* If RAID-6 capabilities were not activated don't try
1334 		 * to use them
1335 		 */
1336 		if (unlikely(!ppc440spe_r6_enabled))
1337 			return -1;
1338 	}
1339 	/*  In the current implementation of ppc440spe ADMA driver it
1340 	 * makes sense to pick out only pq case, because it may be
1341 	 * processed:
1342 	 * (1) either using Biskup method on DMA2;
1343 	 * (2) or on DMA0/1.
1344 	 *  Thus we give a favour to (1) if the sources are suitable;
1345 	 * else let it be processed on one of the DMA0/1 engines.
1346 	 *  In the sum_product case where destination is also the
1347 	 * source process it on DMA0/1 only.
1348 	 */
1349 	if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1350 
1351 		if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1352 			ef = 0; /* sum_product case, process on DMA0/1 */
1353 		else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1354 			ef = 3; /* override (DMA0/1 + idle) */
1355 		else
1356 			ef = 0; /* can't process on DMA2 if !rxor */
1357 	}
1358 
1359 	/* channel idleness increases the priority */
1360 	if (likely(ef) &&
1361 	    !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1362 		ef++;
1363 
1364 	return ef;
1365 }
1366 
1367 struct dma_chan *
ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,struct page ** dst_lst,int dst_cnt,struct page ** src_lst,int src_cnt,size_t src_sz)1368 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1369 	struct page **dst_lst, int dst_cnt, struct page **src_lst,
1370 	int src_cnt, size_t src_sz)
1371 {
1372 	struct dma_chan *best_chan = NULL;
1373 	struct ppc_dma_chan_ref *ref;
1374 	int best_rank = -1;
1375 
1376 	if (unlikely(!src_sz))
1377 		return NULL;
1378 	if (src_sz > PAGE_SIZE) {
1379 		/*
1380 		 * should a user of the api ever pass > PAGE_SIZE requests
1381 		 * we sort out cases where temporary page-sized buffers
1382 		 * are used.
1383 		 */
1384 		switch (cap) {
1385 		case DMA_PQ:
1386 			if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1387 				return NULL;
1388 			if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1389 				return NULL;
1390 			break;
1391 		case DMA_PQ_VAL:
1392 		case DMA_XOR_VAL:
1393 			return NULL;
1394 		default:
1395 			break;
1396 		}
1397 	}
1398 
1399 	list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1400 		if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1401 			int rank;
1402 
1403 			rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1404 					dst_cnt, src_lst, src_cnt, src_sz);
1405 			if (rank > best_rank) {
1406 				best_rank = rank;
1407 				best_chan = ref->chan;
1408 			}
1409 		}
1410 	}
1411 
1412 	return best_chan;
1413 }
1414 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1415 
1416 /**
1417  * ppc440spe_get_group_entry - get group entry with index idx
1418  * @tdesc: is the last allocated slot in the group.
1419  */
1420 static struct ppc440spe_adma_desc_slot *
ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot * tdesc,u32 entry_idx)1421 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1422 {
1423 	struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1424 	int i = 0;
1425 
1426 	if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1427 		printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1428 			__func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1429 		BUG();
1430 	}
1431 
1432 	list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1433 		if (i++ == entry_idx)
1434 			break;
1435 	}
1436 	return iter;
1437 }
1438 
1439 /**
1440  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1441  * @slot: Slot to free
1442  * Caller must hold &ppc440spe_chan->lock while calling this function
1443  */
ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot * slot,struct ppc440spe_adma_chan * chan)1444 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1445 				      struct ppc440spe_adma_chan *chan)
1446 {
1447 	int stride = slot->slots_per_op;
1448 
1449 	while (stride--) {
1450 		slot->slots_per_op = 0;
1451 		slot = list_entry(slot->slot_node.next,
1452 				struct ppc440spe_adma_desc_slot,
1453 				slot_node);
1454 	}
1455 }
1456 
1457 /**
1458  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1459  * upon completion
1460  */
ppc440spe_adma_run_tx_complete_actions(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan,dma_cookie_t cookie)1461 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1462 		struct ppc440spe_adma_desc_slot *desc,
1463 		struct ppc440spe_adma_chan *chan,
1464 		dma_cookie_t cookie)
1465 {
1466 	BUG_ON(desc->async_tx.cookie < 0);
1467 	if (desc->async_tx.cookie > 0) {
1468 		cookie = desc->async_tx.cookie;
1469 		desc->async_tx.cookie = 0;
1470 
1471 		dma_descriptor_unmap(&desc->async_tx);
1472 		/* call the callback (must not sleep or submit new
1473 		 * operations to this channel)
1474 		 */
1475 		dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
1476 	}
1477 
1478 	/* run dependent operations */
1479 	dma_run_dependencies(&desc->async_tx);
1480 
1481 	return cookie;
1482 }
1483 
1484 /**
1485  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1486  */
ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_adma_chan * chan)1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1488 		struct ppc440spe_adma_chan *chan)
1489 {
1490 	/* the client is allowed to attach dependent operations
1491 	 * until 'ack' is set
1492 	 */
1493 	if (!async_tx_test_ack(&desc->async_tx))
1494 		return 0;
1495 
1496 	/* leave the last descriptor in the chain
1497 	 * so we can append to it
1498 	 */
1499 	if (list_is_last(&desc->chain_node, &chan->chain) ||
1500 	    desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1501 		return 1;
1502 
1503 	if (chan->device->id != PPC440SPE_XOR_ID) {
1504 		/* our DMA interrupt handler clears opc field of
1505 		 * each processed descriptor. For all types of
1506 		 * operations except for ZeroSum we do not actually
1507 		 * need ack from the interrupt handler. ZeroSum is a
1508 		 * special case since the result of this operation
1509 		 * is available from the handler only, so if we see
1510 		 * such type of descriptor (which is unprocessed yet)
1511 		 * then leave it in chain.
1512 		 */
1513 		struct dma_cdb *cdb = desc->hw_desc;
1514 		if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1515 			return 1;
1516 	}
1517 
1518 	dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1519 		desc->phys, desc->idx, desc->slots_per_op);
1520 
1521 	list_del(&desc->chain_node);
1522 	ppc440spe_adma_free_slots(desc, chan);
1523 	return 0;
1524 }
1525 
1526 /**
1527  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1528  *	which runs through the channel CDBs list until reach the descriptor
1529  *	currently processed. When routine determines that all CDBs of group
1530  *	are completed then corresponding callbacks (if any) are called and slots
1531  *	are freed.
1532  */
__ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan * chan)1533 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1534 {
1535 	struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1536 	dma_cookie_t cookie = 0;
1537 	u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1538 	int busy = ppc440spe_chan_is_busy(chan);
1539 	int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1540 
1541 	dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1542 		chan->device->id, __func__);
1543 
1544 	if (!current_desc) {
1545 		/*  There were no transactions yet, so
1546 		 * nothing to clean
1547 		 */
1548 		return;
1549 	}
1550 
1551 	/* free completed slots from the chain starting with
1552 	 * the oldest descriptor
1553 	 */
1554 	list_for_each_entry_safe(iter, _iter, &chan->chain,
1555 					chain_node) {
1556 		dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1557 		    "busy: %d this_desc: %#llx next_desc: %#x "
1558 		    "cur: %#x ack: %d\n",
1559 		    iter->async_tx.cookie, iter->idx, busy, iter->phys,
1560 		    ppc440spe_desc_get_link(iter, chan), current_desc,
1561 		    async_tx_test_ack(&iter->async_tx));
1562 		prefetch(_iter);
1563 		prefetch(&_iter->async_tx);
1564 
1565 		/* do not advance past the current descriptor loaded into the
1566 		 * hardware channel,subsequent descriptors are either in process
1567 		 * or have not been submitted
1568 		 */
1569 		if (seen_current)
1570 			break;
1571 
1572 		/* stop the search if we reach the current descriptor and the
1573 		 * channel is busy, or if it appears that the current descriptor
1574 		 * needs to be re-read (i.e. has been appended to)
1575 		 */
1576 		if (iter->phys == current_desc) {
1577 			BUG_ON(seen_current++);
1578 			if (busy || ppc440spe_desc_get_link(iter, chan)) {
1579 				/* not all descriptors of the group have
1580 				 * been completed; exit.
1581 				 */
1582 				break;
1583 			}
1584 		}
1585 
1586 		/* detect the start of a group transaction */
1587 		if (!slot_cnt && !slots_per_op) {
1588 			slot_cnt = iter->slot_cnt;
1589 			slots_per_op = iter->slots_per_op;
1590 			if (slot_cnt <= slots_per_op) {
1591 				slot_cnt = 0;
1592 				slots_per_op = 0;
1593 			}
1594 		}
1595 
1596 		if (slot_cnt) {
1597 			if (!group_start)
1598 				group_start = iter;
1599 			slot_cnt -= slots_per_op;
1600 		}
1601 
1602 		/* all the members of a group are complete */
1603 		if (slots_per_op != 0 && slot_cnt == 0) {
1604 			struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1605 			int end_of_chain = 0;
1606 
1607 			/* clean up the group */
1608 			slot_cnt = group_start->slot_cnt;
1609 			grp_iter = group_start;
1610 			list_for_each_entry_safe_from(grp_iter, _grp_iter,
1611 				&chan->chain, chain_node) {
1612 
1613 				cookie = ppc440spe_adma_run_tx_complete_actions(
1614 					grp_iter, chan, cookie);
1615 
1616 				slot_cnt -= slots_per_op;
1617 				end_of_chain = ppc440spe_adma_clean_slot(
1618 				    grp_iter, chan);
1619 				if (end_of_chain && slot_cnt) {
1620 					/* Should wait for ZeroSum completion */
1621 					if (cookie > 0)
1622 						chan->common.completed_cookie = cookie;
1623 					return;
1624 				}
1625 
1626 				if (slot_cnt == 0 || end_of_chain)
1627 					break;
1628 			}
1629 
1630 			/* the group should be complete at this point */
1631 			BUG_ON(slot_cnt);
1632 
1633 			slots_per_op = 0;
1634 			group_start = NULL;
1635 			if (end_of_chain)
1636 				break;
1637 			else
1638 				continue;
1639 		} else if (slots_per_op) /* wait for group completion */
1640 			continue;
1641 
1642 		cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1643 		    cookie);
1644 
1645 		if (ppc440spe_adma_clean_slot(iter, chan))
1646 			break;
1647 	}
1648 
1649 	BUG_ON(!seen_current);
1650 
1651 	if (cookie > 0) {
1652 		chan->common.completed_cookie = cookie;
1653 		pr_debug("\tcompleted cookie %d\n", cookie);
1654 	}
1655 
1656 }
1657 
1658 /**
1659  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1660  */
ppc440spe_adma_tasklet(struct tasklet_struct * t)1661 static void ppc440spe_adma_tasklet(struct tasklet_struct *t)
1662 {
1663 	struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet);
1664 
1665 	spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1666 	__ppc440spe_adma_slot_cleanup(chan);
1667 	spin_unlock(&chan->lock);
1668 }
1669 
1670 /**
1671  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1672  */
ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan * chan)1673 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1674 {
1675 	spin_lock_bh(&chan->lock);
1676 	__ppc440spe_adma_slot_cleanup(chan);
1677 	spin_unlock_bh(&chan->lock);
1678 }
1679 
1680 /**
1681  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1682  */
ppc440spe_adma_alloc_slots(struct ppc440spe_adma_chan * chan,int num_slots,int slots_per_op)1683 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1684 		struct ppc440spe_adma_chan *chan, int num_slots,
1685 		int slots_per_op)
1686 {
1687 	struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1688 	struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1689 	int slots_found, retry = 0;
1690 	LIST_HEAD(chain);
1691 
1692 
1693 	BUG_ON(!num_slots || !slots_per_op);
1694 	/* start search from the last allocated descrtiptor
1695 	 * if a contiguous allocation can not be found start searching
1696 	 * from the beginning of the list
1697 	 */
1698 retry:
1699 	slots_found = 0;
1700 	if (retry == 0)
1701 		iter = chan->last_used;
1702 	else
1703 		iter = list_entry(&chan->all_slots,
1704 				  struct ppc440spe_adma_desc_slot,
1705 				  slot_node);
1706 	list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1707 	    slot_node) {
1708 		prefetch(_iter);
1709 		prefetch(&_iter->async_tx);
1710 		if (iter->slots_per_op) {
1711 			slots_found = 0;
1712 			continue;
1713 		}
1714 
1715 		/* start the allocation if the slot is correctly aligned */
1716 		if (!slots_found++)
1717 			alloc_start = iter;
1718 
1719 		if (slots_found == num_slots) {
1720 			struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1721 			struct ppc440spe_adma_desc_slot *last_used = NULL;
1722 
1723 			iter = alloc_start;
1724 			while (num_slots) {
1725 				int i;
1726 				/* pre-ack all but the last descriptor */
1727 				if (num_slots != slots_per_op)
1728 					async_tx_ack(&iter->async_tx);
1729 
1730 				list_add_tail(&iter->chain_node, &chain);
1731 				alloc_tail = iter;
1732 				iter->async_tx.cookie = 0;
1733 				iter->hw_next = NULL;
1734 				iter->flags = 0;
1735 				iter->slot_cnt = num_slots;
1736 				iter->xor_check_result = NULL;
1737 				for (i = 0; i < slots_per_op; i++) {
1738 					iter->slots_per_op = slots_per_op - i;
1739 					last_used = iter;
1740 					iter = list_entry(iter->slot_node.next,
1741 						struct ppc440spe_adma_desc_slot,
1742 						slot_node);
1743 				}
1744 				num_slots -= slots_per_op;
1745 			}
1746 			alloc_tail->group_head = alloc_start;
1747 			alloc_tail->async_tx.cookie = -EBUSY;
1748 			list_splice(&chain, &alloc_tail->group_list);
1749 			chan->last_used = last_used;
1750 			return alloc_tail;
1751 		}
1752 	}
1753 	if (!retry++)
1754 		goto retry;
1755 
1756 	/* try to free some slots if the allocation fails */
1757 	tasklet_schedule(&chan->irq_tasklet);
1758 	return NULL;
1759 }
1760 
1761 /**
1762  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1763  */
ppc440spe_adma_alloc_chan_resources(struct dma_chan * chan)1764 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1765 {
1766 	struct ppc440spe_adma_chan *ppc440spe_chan;
1767 	struct ppc440spe_adma_desc_slot *slot = NULL;
1768 	char *hw_desc;
1769 	int i, db_sz;
1770 	int init;
1771 
1772 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1773 	init = ppc440spe_chan->slots_allocated ? 0 : 1;
1774 	chan->chan_id = ppc440spe_chan->device->id;
1775 
1776 	/* Allocate descriptor slots */
1777 	i = ppc440spe_chan->slots_allocated;
1778 	if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1779 		db_sz = sizeof(struct dma_cdb);
1780 	else
1781 		db_sz = sizeof(struct xor_cb);
1782 
1783 	for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1784 		slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1785 			       GFP_KERNEL);
1786 		if (!slot) {
1787 			printk(KERN_INFO "SPE ADMA Channel only initialized"
1788 				" %d descriptor slots", i--);
1789 			break;
1790 		}
1791 
1792 		hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1793 		slot->hw_desc = (void *) &hw_desc[i * db_sz];
1794 		dma_async_tx_descriptor_init(&slot->async_tx, chan);
1795 		slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1796 		INIT_LIST_HEAD(&slot->chain_node);
1797 		INIT_LIST_HEAD(&slot->slot_node);
1798 		INIT_LIST_HEAD(&slot->group_list);
1799 		slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1800 		slot->idx = i;
1801 
1802 		spin_lock_bh(&ppc440spe_chan->lock);
1803 		ppc440spe_chan->slots_allocated++;
1804 		list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1805 		spin_unlock_bh(&ppc440spe_chan->lock);
1806 	}
1807 
1808 	if (i && !ppc440spe_chan->last_used) {
1809 		ppc440spe_chan->last_used =
1810 			list_entry(ppc440spe_chan->all_slots.next,
1811 				struct ppc440spe_adma_desc_slot,
1812 				slot_node);
1813 	}
1814 
1815 	dev_dbg(ppc440spe_chan->device->common.dev,
1816 		"ppc440spe adma%d: allocated %d descriptor slots\n",
1817 		ppc440spe_chan->device->id, i);
1818 
1819 	/* initialize the channel and the chain with a null operation */
1820 	if (init) {
1821 		switch (ppc440spe_chan->device->id) {
1822 		case PPC440SPE_DMA0_ID:
1823 		case PPC440SPE_DMA1_ID:
1824 			ppc440spe_chan->hw_chain_inited = 0;
1825 			/* Use WXOR for self-testing */
1826 			if (!ppc440spe_r6_tchan)
1827 				ppc440spe_r6_tchan = ppc440spe_chan;
1828 			break;
1829 		case PPC440SPE_XOR_ID:
1830 			ppc440spe_chan_start_null_xor(ppc440spe_chan);
1831 			break;
1832 		default:
1833 			BUG();
1834 		}
1835 		ppc440spe_chan->needs_unmap = 1;
1836 	}
1837 
1838 	return (i > 0) ? i : -ENOMEM;
1839 }
1840 
1841 /**
1842  * ppc440spe_rxor_set_region_data -
1843  */
ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot * desc,u8 xor_arg_no,u32 mask)1844 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1845 	u8 xor_arg_no, u32 mask)
1846 {
1847 	struct xor_cb *xcb = desc->hw_desc;
1848 
1849 	xcb->ops[xor_arg_no].h |= mask;
1850 }
1851 
1852 /**
1853  * ppc440spe_rxor_set_src -
1854  */
ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot * desc,u8 xor_arg_no,dma_addr_t addr)1855 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1856 	u8 xor_arg_no, dma_addr_t addr)
1857 {
1858 	struct xor_cb *xcb = desc->hw_desc;
1859 
1860 	xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1861 	xcb->ops[xor_arg_no].l = addr;
1862 }
1863 
1864 /**
1865  * ppc440spe_rxor_set_mult -
1866  */
ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot * desc,u8 xor_arg_no,u8 idx,u8 mult)1867 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1868 	u8 xor_arg_no, u8 idx, u8 mult)
1869 {
1870 	struct xor_cb *xcb = desc->hw_desc;
1871 
1872 	xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1873 }
1874 
1875 /**
1876  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1877  *	has been achieved
1878  */
ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan * chan)1879 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1880 {
1881 	dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1882 		chan->device->id, chan->pending);
1883 
1884 	if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1885 		chan->pending = 0;
1886 		ppc440spe_chan_append(chan);
1887 	}
1888 }
1889 
1890 /**
1891  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1892  *	(it's not necessary that descriptors will be submitted to the h/w
1893  *	chains too right now)
1894  */
ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor * tx)1895 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1896 {
1897 	struct ppc440spe_adma_desc_slot *sw_desc;
1898 	struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1899 	struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1900 	int slot_cnt;
1901 	int slots_per_op;
1902 	dma_cookie_t cookie;
1903 
1904 	sw_desc = tx_to_ppc440spe_adma_slot(tx);
1905 
1906 	group_start = sw_desc->group_head;
1907 	slot_cnt = group_start->slot_cnt;
1908 	slots_per_op = group_start->slots_per_op;
1909 
1910 	spin_lock_bh(&chan->lock);
1911 	cookie = dma_cookie_assign(tx);
1912 
1913 	if (unlikely(list_empty(&chan->chain))) {
1914 		/* first peer */
1915 		list_splice_init(&sw_desc->group_list, &chan->chain);
1916 		chan_first_cdb[chan->device->id] = group_start;
1917 	} else {
1918 		/* isn't first peer, bind CDBs to chain */
1919 		old_chain_tail = list_entry(chan->chain.prev,
1920 					struct ppc440spe_adma_desc_slot,
1921 					chain_node);
1922 		list_splice_init(&sw_desc->group_list,
1923 		    &old_chain_tail->chain_node);
1924 		/* fix up the hardware chain */
1925 		ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1926 	}
1927 
1928 	/* increment the pending count by the number of operations */
1929 	chan->pending += slot_cnt / slots_per_op;
1930 	ppc440spe_adma_check_threshold(chan);
1931 	spin_unlock_bh(&chan->lock);
1932 
1933 	dev_dbg(chan->device->common.dev,
1934 		"ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1935 		chan->device->id, __func__,
1936 		sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1937 
1938 	return cookie;
1939 }
1940 
1941 /**
1942  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1943  */
ppc440spe_adma_prep_dma_interrupt(struct dma_chan * chan,unsigned long flags)1944 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1945 		struct dma_chan *chan, unsigned long flags)
1946 {
1947 	struct ppc440spe_adma_chan *ppc440spe_chan;
1948 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1949 	int slot_cnt, slots_per_op;
1950 
1951 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1952 
1953 	dev_dbg(ppc440spe_chan->device->common.dev,
1954 		"ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1955 		__func__);
1956 
1957 	spin_lock_bh(&ppc440spe_chan->lock);
1958 	slot_cnt = slots_per_op = 1;
1959 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1960 			slots_per_op);
1961 	if (sw_desc) {
1962 		group_start = sw_desc->group_head;
1963 		ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1964 		group_start->unmap_len = 0;
1965 		sw_desc->async_tx.flags = flags;
1966 	}
1967 	spin_unlock_bh(&ppc440spe_chan->lock);
1968 
1969 	return sw_desc ? &sw_desc->async_tx : NULL;
1970 }
1971 
1972 /**
1973  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1974  */
ppc440spe_adma_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dma_dest,dma_addr_t dma_src,size_t len,unsigned long flags)1975 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1976 		struct dma_chan *chan, dma_addr_t dma_dest,
1977 		dma_addr_t dma_src, size_t len, unsigned long flags)
1978 {
1979 	struct ppc440spe_adma_chan *ppc440spe_chan;
1980 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1981 	int slot_cnt, slots_per_op;
1982 
1983 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1984 
1985 	if (unlikely(!len))
1986 		return NULL;
1987 
1988 	BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1989 
1990 	spin_lock_bh(&ppc440spe_chan->lock);
1991 
1992 	dev_dbg(ppc440spe_chan->device->common.dev,
1993 		"ppc440spe adma%d: %s len: %u int_en %d\n",
1994 		ppc440spe_chan->device->id, __func__, len,
1995 		flags & DMA_PREP_INTERRUPT ? 1 : 0);
1996 	slot_cnt = slots_per_op = 1;
1997 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1998 		slots_per_op);
1999 	if (sw_desc) {
2000 		group_start = sw_desc->group_head;
2001 		ppc440spe_desc_init_memcpy(group_start, flags);
2002 		ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2003 		ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2004 		ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2005 		sw_desc->unmap_len = len;
2006 		sw_desc->async_tx.flags = flags;
2007 	}
2008 	spin_unlock_bh(&ppc440spe_chan->lock);
2009 
2010 	return sw_desc ? &sw_desc->async_tx : NULL;
2011 }
2012 
2013 /**
2014  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2015  */
ppc440spe_adma_prep_dma_xor(struct dma_chan * chan,dma_addr_t dma_dest,dma_addr_t * dma_src,u32 src_cnt,size_t len,unsigned long flags)2016 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2017 		struct dma_chan *chan, dma_addr_t dma_dest,
2018 		dma_addr_t *dma_src, u32 src_cnt, size_t len,
2019 		unsigned long flags)
2020 {
2021 	struct ppc440spe_adma_chan *ppc440spe_chan;
2022 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2023 	int slot_cnt, slots_per_op;
2024 
2025 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2026 
2027 	ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2028 				     dma_dest, dma_src, src_cnt));
2029 	if (unlikely(!len))
2030 		return NULL;
2031 	BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2032 
2033 	dev_dbg(ppc440spe_chan->device->common.dev,
2034 		"ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2035 		ppc440spe_chan->device->id, __func__, src_cnt, len,
2036 		flags & DMA_PREP_INTERRUPT ? 1 : 0);
2037 
2038 	spin_lock_bh(&ppc440spe_chan->lock);
2039 	slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2040 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2041 			slots_per_op);
2042 	if (sw_desc) {
2043 		group_start = sw_desc->group_head;
2044 		ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2045 		ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2046 		while (src_cnt--)
2047 			ppc440spe_adma_memcpy_xor_set_src(group_start,
2048 				dma_src[src_cnt], src_cnt);
2049 		ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2050 		sw_desc->unmap_len = len;
2051 		sw_desc->async_tx.flags = flags;
2052 	}
2053 	spin_unlock_bh(&ppc440spe_chan->lock);
2054 
2055 	return sw_desc ? &sw_desc->async_tx : NULL;
2056 }
2057 
2058 static inline void
2059 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2060 				int src_cnt);
2061 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2062 
2063 /**
2064  * ppc440spe_adma_init_dma2rxor_slot -
2065  */
ppc440spe_adma_init_dma2rxor_slot(struct ppc440spe_adma_desc_slot * desc,dma_addr_t * src,int src_cnt)2066 static void ppc440spe_adma_init_dma2rxor_slot(
2067 		struct ppc440spe_adma_desc_slot *desc,
2068 		dma_addr_t *src, int src_cnt)
2069 {
2070 	int i;
2071 
2072 	/* initialize CDB */
2073 	for (i = 0; i < src_cnt; i++) {
2074 		ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2075 						 desc->src_cnt, (u32)src[i]);
2076 	}
2077 }
2078 
2079 /**
2080  * ppc440spe_dma01_prep_mult -
2081  * for Q operation where destination is also the source
2082  */
ppc440spe_dma01_prep_mult(struct ppc440spe_adma_chan * ppc440spe_chan,dma_addr_t * dst,int dst_cnt,dma_addr_t * src,int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)2083 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2084 		struct ppc440spe_adma_chan *ppc440spe_chan,
2085 		dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2086 		const unsigned char *scf, size_t len, unsigned long flags)
2087 {
2088 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2089 	unsigned long op = 0;
2090 	int slot_cnt;
2091 
2092 	set_bit(PPC440SPE_DESC_WXOR, &op);
2093 	slot_cnt = 2;
2094 
2095 	spin_lock_bh(&ppc440spe_chan->lock);
2096 
2097 	/* use WXOR, each descriptor occupies one slot */
2098 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2099 	if (sw_desc) {
2100 		struct ppc440spe_adma_chan *chan;
2101 		struct ppc440spe_adma_desc_slot *iter;
2102 		struct dma_cdb *hw_desc;
2103 
2104 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2105 		set_bits(op, &sw_desc->flags);
2106 		sw_desc->src_cnt = src_cnt;
2107 		sw_desc->dst_cnt = dst_cnt;
2108 		/* First descriptor, zero data in the destination and copy it
2109 		 * to q page using MULTICAST transfer.
2110 		 */
2111 		iter = list_first_entry(&sw_desc->group_list,
2112 					struct ppc440spe_adma_desc_slot,
2113 					chain_node);
2114 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2115 		/* set 'next' pointer */
2116 		iter->hw_next = list_entry(iter->chain_node.next,
2117 					   struct ppc440spe_adma_desc_slot,
2118 					   chain_node);
2119 		clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2120 		hw_desc = iter->hw_desc;
2121 		hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2122 
2123 		ppc440spe_desc_set_dest_addr(iter, chan,
2124 					     DMA_CUED_XOR_BASE, dst[0], 0);
2125 		ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2126 		ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2127 					    src[0]);
2128 		ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2129 		iter->unmap_len = len;
2130 
2131 		/*
2132 		 * Second descriptor, multiply data from the q page
2133 		 * and store the result in real destination.
2134 		 */
2135 		iter = list_first_entry(&iter->chain_node,
2136 					struct ppc440spe_adma_desc_slot,
2137 					chain_node);
2138 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2139 		iter->hw_next = NULL;
2140 		if (flags & DMA_PREP_INTERRUPT)
2141 			set_bit(PPC440SPE_DESC_INT, &iter->flags);
2142 		else
2143 			clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2144 
2145 		hw_desc = iter->hw_desc;
2146 		hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2147 		ppc440spe_desc_set_src_addr(iter, chan, 0,
2148 					    DMA_CUED_XOR_HB, dst[1]);
2149 		ppc440spe_desc_set_dest_addr(iter, chan,
2150 					     DMA_CUED_XOR_BASE, dst[0], 0);
2151 
2152 		ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2153 					    DMA_CDB_SG_DST1, scf[0]);
2154 		ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2155 		iter->unmap_len = len;
2156 		sw_desc->async_tx.flags = flags;
2157 	}
2158 
2159 	spin_unlock_bh(&ppc440spe_chan->lock);
2160 
2161 	return sw_desc;
2162 }
2163 
2164 /**
2165  * ppc440spe_dma01_prep_sum_product -
2166  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2167  * the source.
2168  */
ppc440spe_dma01_prep_sum_product(struct ppc440spe_adma_chan * ppc440spe_chan,dma_addr_t * dst,dma_addr_t * src,int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)2169 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2170 		struct ppc440spe_adma_chan *ppc440spe_chan,
2171 		dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2172 		const unsigned char *scf, size_t len, unsigned long flags)
2173 {
2174 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2175 	unsigned long op = 0;
2176 	int slot_cnt;
2177 
2178 	set_bit(PPC440SPE_DESC_WXOR, &op);
2179 	slot_cnt = 3;
2180 
2181 	spin_lock_bh(&ppc440spe_chan->lock);
2182 
2183 	/* WXOR, each descriptor occupies one slot */
2184 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2185 	if (sw_desc) {
2186 		struct ppc440spe_adma_chan *chan;
2187 		struct ppc440spe_adma_desc_slot *iter;
2188 		struct dma_cdb *hw_desc;
2189 
2190 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2191 		set_bits(op, &sw_desc->flags);
2192 		sw_desc->src_cnt = src_cnt;
2193 		sw_desc->dst_cnt = 1;
2194 		/* 1st descriptor, src[1] data to q page and zero destination */
2195 		iter = list_first_entry(&sw_desc->group_list,
2196 					struct ppc440spe_adma_desc_slot,
2197 					chain_node);
2198 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2199 		iter->hw_next = list_entry(iter->chain_node.next,
2200 					   struct ppc440spe_adma_desc_slot,
2201 					   chain_node);
2202 		clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2203 		hw_desc = iter->hw_desc;
2204 		hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2205 
2206 		ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2207 					     *dst, 0);
2208 		ppc440spe_desc_set_dest_addr(iter, chan, 0,
2209 					     ppc440spe_chan->qdest, 1);
2210 		ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2211 					    src[1]);
2212 		ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2213 		iter->unmap_len = len;
2214 
2215 		/* 2nd descriptor, multiply src[1] data and store the
2216 		 * result in destination */
2217 		iter = list_first_entry(&iter->chain_node,
2218 					struct ppc440spe_adma_desc_slot,
2219 					chain_node);
2220 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2221 		/* set 'next' pointer */
2222 		iter->hw_next = list_entry(iter->chain_node.next,
2223 					   struct ppc440spe_adma_desc_slot,
2224 					   chain_node);
2225 		if (flags & DMA_PREP_INTERRUPT)
2226 			set_bit(PPC440SPE_DESC_INT, &iter->flags);
2227 		else
2228 			clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2229 
2230 		hw_desc = iter->hw_desc;
2231 		hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2232 		ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2233 					    ppc440spe_chan->qdest);
2234 		ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2235 					     *dst, 0);
2236 		ppc440spe_desc_set_src_mult(iter, chan,	DMA_CUED_MULT1_OFF,
2237 					    DMA_CDB_SG_DST1, scf[1]);
2238 		ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2239 		iter->unmap_len = len;
2240 
2241 		/*
2242 		 * 3rd descriptor, multiply src[0] data and xor it
2243 		 * with destination
2244 		 */
2245 		iter = list_first_entry(&iter->chain_node,
2246 					struct ppc440spe_adma_desc_slot,
2247 					chain_node);
2248 		memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2249 		iter->hw_next = NULL;
2250 		if (flags & DMA_PREP_INTERRUPT)
2251 			set_bit(PPC440SPE_DESC_INT, &iter->flags);
2252 		else
2253 			clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2254 
2255 		hw_desc = iter->hw_desc;
2256 		hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2257 		ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2258 					    src[0]);
2259 		ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2260 					     *dst, 0);
2261 		ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2262 					    DMA_CDB_SG_DST1, scf[0]);
2263 		ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2264 		iter->unmap_len = len;
2265 		sw_desc->async_tx.flags = flags;
2266 	}
2267 
2268 	spin_unlock_bh(&ppc440spe_chan->lock);
2269 
2270 	return sw_desc;
2271 }
2272 
ppc440spe_dma01_prep_pq(struct ppc440spe_adma_chan * ppc440spe_chan,dma_addr_t * dst,int dst_cnt,dma_addr_t * src,int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)2273 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2274 		struct ppc440spe_adma_chan *ppc440spe_chan,
2275 		dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2276 		const unsigned char *scf, size_t len, unsigned long flags)
2277 {
2278 	int slot_cnt;
2279 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2280 	unsigned long op = 0;
2281 	unsigned char mult = 1;
2282 
2283 	pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2284 		 __func__, dst_cnt, src_cnt, len);
2285 	/*  select operations WXOR/RXOR depending on the
2286 	 * source addresses of operators and the number
2287 	 * of destinations (RXOR support only Q-parity calculations)
2288 	 */
2289 	set_bit(PPC440SPE_DESC_WXOR, &op);
2290 	if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2291 		/* no active RXOR;
2292 		 * do RXOR if:
2293 		 * - there are more than 1 source,
2294 		 * - len is aligned on 512-byte boundary,
2295 		 * - source addresses fit to one of 4 possible regions.
2296 		 */
2297 		if (src_cnt > 1 &&
2298 		    !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2299 		    (src[0] + len) == src[1]) {
2300 			/* may do RXOR R1 R2 */
2301 			set_bit(PPC440SPE_DESC_RXOR, &op);
2302 			if (src_cnt != 2) {
2303 				/* may try to enhance region of RXOR */
2304 				if ((src[1] + len) == src[2]) {
2305 					/* do RXOR R1 R2 R3 */
2306 					set_bit(PPC440SPE_DESC_RXOR123,
2307 						&op);
2308 				} else if ((src[1] + len * 2) == src[2]) {
2309 					/* do RXOR R1 R2 R4 */
2310 					set_bit(PPC440SPE_DESC_RXOR124, &op);
2311 				} else if ((src[1] + len * 3) == src[2]) {
2312 					/* do RXOR R1 R2 R5 */
2313 					set_bit(PPC440SPE_DESC_RXOR125,
2314 						&op);
2315 				} else {
2316 					/* do RXOR R1 R2 */
2317 					set_bit(PPC440SPE_DESC_RXOR12,
2318 						&op);
2319 				}
2320 			} else {
2321 				/* do RXOR R1 R2 */
2322 				set_bit(PPC440SPE_DESC_RXOR12, &op);
2323 			}
2324 		}
2325 
2326 		if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2327 			/* can not do this operation with RXOR */
2328 			clear_bit(PPC440SPE_RXOR_RUN,
2329 				&ppc440spe_rxor_state);
2330 		} else {
2331 			/* can do; set block size right now */
2332 			ppc440spe_desc_set_rxor_block_size(len);
2333 		}
2334 	}
2335 
2336 	/* Number of necessary slots depends on operation type selected */
2337 	if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2338 		/*  This is a WXOR only chain. Need descriptors for each
2339 		 * source to GF-XOR them with WXOR, and need descriptors
2340 		 * for each destination to zero them with WXOR
2341 		 */
2342 		slot_cnt = src_cnt;
2343 
2344 		if (flags & DMA_PREP_ZERO_P) {
2345 			slot_cnt++;
2346 			set_bit(PPC440SPE_ZERO_P, &op);
2347 		}
2348 		if (flags & DMA_PREP_ZERO_Q) {
2349 			slot_cnt++;
2350 			set_bit(PPC440SPE_ZERO_Q, &op);
2351 		}
2352 	} else {
2353 		/*  Need 1/2 descriptor for RXOR operation, and
2354 		 * need (src_cnt - (2 or 3)) for WXOR of sources
2355 		 * remained (if any)
2356 		 */
2357 		slot_cnt = dst_cnt;
2358 
2359 		if (flags & DMA_PREP_ZERO_P)
2360 			set_bit(PPC440SPE_ZERO_P, &op);
2361 		if (flags & DMA_PREP_ZERO_Q)
2362 			set_bit(PPC440SPE_ZERO_Q, &op);
2363 
2364 		if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2365 			slot_cnt += src_cnt - 2;
2366 		else
2367 			slot_cnt += src_cnt - 3;
2368 
2369 		/*  Thus we have either RXOR only chain or
2370 		 * mixed RXOR/WXOR
2371 		 */
2372 		if (slot_cnt == dst_cnt)
2373 			/* RXOR only chain */
2374 			clear_bit(PPC440SPE_DESC_WXOR, &op);
2375 	}
2376 
2377 	spin_lock_bh(&ppc440spe_chan->lock);
2378 	/* for both RXOR/WXOR each descriptor occupies one slot */
2379 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2380 	if (sw_desc) {
2381 		ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2382 				flags, op);
2383 
2384 		/* setup dst/src/mult */
2385 		pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2386 			 __func__, dst[0], dst[1]);
2387 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2388 		while (src_cnt--) {
2389 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2390 						  src_cnt);
2391 
2392 			/* NOTE: "Multi = 0 is equivalent to = 1" as it
2393 			 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2394 			 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2395 			 * leads to zeroing source data after RXOR.
2396 			 * So, for P case set-up mult=1 explicitly.
2397 			 */
2398 			if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2399 				mult = scf[src_cnt];
2400 			ppc440spe_adma_pq_set_src_mult(sw_desc,
2401 				mult, src_cnt,  dst_cnt - 1);
2402 		}
2403 
2404 		/* Setup byte count foreach slot just allocated */
2405 		sw_desc->async_tx.flags = flags;
2406 		list_for_each_entry(iter, &sw_desc->group_list,
2407 				chain_node) {
2408 			ppc440spe_desc_set_byte_count(iter,
2409 				ppc440spe_chan, len);
2410 			iter->unmap_len = len;
2411 		}
2412 	}
2413 	spin_unlock_bh(&ppc440spe_chan->lock);
2414 
2415 	return sw_desc;
2416 }
2417 
ppc440spe_dma2_prep_pq(struct ppc440spe_adma_chan * ppc440spe_chan,dma_addr_t * dst,int dst_cnt,dma_addr_t * src,int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)2418 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2419 		struct ppc440spe_adma_chan *ppc440spe_chan,
2420 		dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2421 		const unsigned char *scf, size_t len, unsigned long flags)
2422 {
2423 	int slot_cnt, descs_per_op;
2424 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2425 	unsigned long op = 0;
2426 	unsigned char mult = 1;
2427 
2428 	BUG_ON(!dst_cnt);
2429 	/*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2430 		 __func__, dst_cnt, src_cnt, len);*/
2431 
2432 	spin_lock_bh(&ppc440spe_chan->lock);
2433 	descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2434 	if (descs_per_op < 0) {
2435 		spin_unlock_bh(&ppc440spe_chan->lock);
2436 		return NULL;
2437 	}
2438 
2439 	/* depending on number of sources we have 1 or 2 RXOR chains */
2440 	slot_cnt = descs_per_op * dst_cnt;
2441 
2442 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2443 	if (sw_desc) {
2444 		op = slot_cnt;
2445 		sw_desc->async_tx.flags = flags;
2446 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2447 			ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2448 				--op ? 0 : flags);
2449 			ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2450 				len);
2451 			iter->unmap_len = len;
2452 
2453 			ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2454 			iter->rxor_cursor.len = len;
2455 			iter->descs_per_op = descs_per_op;
2456 		}
2457 		op = 0;
2458 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2459 			op++;
2460 			if (op % descs_per_op == 0)
2461 				ppc440spe_adma_init_dma2rxor_slot(iter, src,
2462 								  src_cnt);
2463 			if (likely(!list_is_last(&iter->chain_node,
2464 						 &sw_desc->group_list))) {
2465 				/* set 'next' pointer */
2466 				iter->hw_next =
2467 					list_entry(iter->chain_node.next,
2468 						struct ppc440spe_adma_desc_slot,
2469 						chain_node);
2470 				ppc440spe_xor_set_link(iter, iter->hw_next);
2471 			} else {
2472 				/* this is the last descriptor. */
2473 				iter->hw_next = NULL;
2474 			}
2475 		}
2476 
2477 		/* fixup head descriptor */
2478 		sw_desc->dst_cnt = dst_cnt;
2479 		if (flags & DMA_PREP_ZERO_P)
2480 			set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2481 		if (flags & DMA_PREP_ZERO_Q)
2482 			set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2483 
2484 		/* setup dst/src/mult */
2485 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2486 
2487 		while (src_cnt--) {
2488 			/* handle descriptors (if dst_cnt == 2) inside
2489 			 * the ppc440spe_adma_pq_set_srcxxx() functions
2490 			 */
2491 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2492 						  src_cnt);
2493 			if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2494 				mult = scf[src_cnt];
2495 			ppc440spe_adma_pq_set_src_mult(sw_desc,
2496 					mult, src_cnt, dst_cnt - 1);
2497 		}
2498 	}
2499 	spin_unlock_bh(&ppc440spe_chan->lock);
2500 	ppc440spe_desc_set_rxor_block_size(len);
2501 	return sw_desc;
2502 }
2503 
2504 /**
2505  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2506  */
ppc440spe_adma_prep_dma_pq(struct dma_chan * chan,dma_addr_t * dst,dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf,size_t len,unsigned long flags)2507 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2508 		struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2509 		unsigned int src_cnt, const unsigned char *scf,
2510 		size_t len, unsigned long flags)
2511 {
2512 	struct ppc440spe_adma_chan *ppc440spe_chan;
2513 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2514 	int dst_cnt = 0;
2515 
2516 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2517 
2518 	ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2519 				    dst, src, src_cnt));
2520 	BUG_ON(!len);
2521 	BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2522 	BUG_ON(!src_cnt);
2523 
2524 	if (src_cnt == 1 && dst[1] == src[0]) {
2525 		dma_addr_t dest[2];
2526 
2527 		/* dst[1] is real destination (Q) */
2528 		dest[0] = dst[1];
2529 		/* this is the page to multicast source data to */
2530 		dest[1] = ppc440spe_chan->qdest;
2531 		sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2532 				dest, 2, src, src_cnt, scf, len, flags);
2533 		return sw_desc ? &sw_desc->async_tx : NULL;
2534 	}
2535 
2536 	if (src_cnt == 2 && dst[1] == src[1]) {
2537 		sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2538 					&dst[1], src, 2, scf, len, flags);
2539 		return sw_desc ? &sw_desc->async_tx : NULL;
2540 	}
2541 
2542 	if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2543 		BUG_ON(!dst[0]);
2544 		dst_cnt++;
2545 		flags |= DMA_PREP_ZERO_P;
2546 	}
2547 
2548 	if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2549 		BUG_ON(!dst[1]);
2550 		dst_cnt++;
2551 		flags |= DMA_PREP_ZERO_Q;
2552 	}
2553 
2554 	BUG_ON(!dst_cnt);
2555 
2556 	dev_dbg(ppc440spe_chan->device->common.dev,
2557 		"ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2558 		ppc440spe_chan->device->id, __func__, src_cnt, len,
2559 		flags & DMA_PREP_INTERRUPT ? 1 : 0);
2560 
2561 	switch (ppc440spe_chan->device->id) {
2562 	case PPC440SPE_DMA0_ID:
2563 	case PPC440SPE_DMA1_ID:
2564 		sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2565 				dst, dst_cnt, src, src_cnt, scf,
2566 				len, flags);
2567 		break;
2568 
2569 	case PPC440SPE_XOR_ID:
2570 		sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2571 				dst, dst_cnt, src, src_cnt, scf,
2572 				len, flags);
2573 		break;
2574 	}
2575 
2576 	return sw_desc ? &sw_desc->async_tx : NULL;
2577 }
2578 
2579 /**
2580  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2581  * a PQ_ZERO_SUM operation
2582  */
ppc440spe_adma_prep_dma_pqzero_sum(struct dma_chan * chan,dma_addr_t * pq,dma_addr_t * src,unsigned int src_cnt,const unsigned char * scf,size_t len,enum sum_check_flags * pqres,unsigned long flags)2583 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2584 		struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2585 		unsigned int src_cnt, const unsigned char *scf, size_t len,
2586 		enum sum_check_flags *pqres, unsigned long flags)
2587 {
2588 	struct ppc440spe_adma_chan *ppc440spe_chan;
2589 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2590 	dma_addr_t pdest, qdest;
2591 	int slot_cnt, slots_per_op, idst, dst_cnt;
2592 
2593 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2594 
2595 	if (flags & DMA_PREP_PQ_DISABLE_P)
2596 		pdest = 0;
2597 	else
2598 		pdest = pq[0];
2599 
2600 	if (flags & DMA_PREP_PQ_DISABLE_Q)
2601 		qdest = 0;
2602 	else
2603 		qdest = pq[1];
2604 
2605 	ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2606 					    src, src_cnt, scf));
2607 
2608 	/* Always use WXOR for P/Q calculations (two destinations).
2609 	 * Need 1 or 2 extra slots to verify results are zero.
2610 	 */
2611 	idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2612 
2613 	/* One additional slot per destination to clone P/Q
2614 	 * before calculation (we have to preserve destinations).
2615 	 */
2616 	slot_cnt = src_cnt + dst_cnt * 2;
2617 	slots_per_op = 1;
2618 
2619 	spin_lock_bh(&ppc440spe_chan->lock);
2620 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2621 					     slots_per_op);
2622 	if (sw_desc) {
2623 		ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2624 
2625 		/* Setup byte count for each slot just allocated */
2626 		sw_desc->async_tx.flags = flags;
2627 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2628 			ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2629 						      len);
2630 			iter->unmap_len = len;
2631 		}
2632 
2633 		if (pdest) {
2634 			struct dma_cdb *hw_desc;
2635 			struct ppc440spe_adma_chan *chan;
2636 
2637 			iter = sw_desc->group_head;
2638 			chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2639 			memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2640 			iter->hw_next = list_entry(iter->chain_node.next,
2641 						struct ppc440spe_adma_desc_slot,
2642 						chain_node);
2643 			hw_desc = iter->hw_desc;
2644 			hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2645 			iter->src_cnt = 0;
2646 			iter->dst_cnt = 0;
2647 			ppc440spe_desc_set_dest_addr(iter, chan, 0,
2648 						     ppc440spe_chan->pdest, 0);
2649 			ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2650 			ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2651 						      len);
2652 			iter->unmap_len = 0;
2653 			/* override pdest to preserve original P */
2654 			pdest = ppc440spe_chan->pdest;
2655 		}
2656 		if (qdest) {
2657 			struct dma_cdb *hw_desc;
2658 			struct ppc440spe_adma_chan *chan;
2659 
2660 			iter = list_first_entry(&sw_desc->group_list,
2661 						struct ppc440spe_adma_desc_slot,
2662 						chain_node);
2663 			chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2664 
2665 			if (pdest) {
2666 				iter = list_entry(iter->chain_node.next,
2667 						struct ppc440spe_adma_desc_slot,
2668 						chain_node);
2669 			}
2670 
2671 			memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2672 			iter->hw_next = list_entry(iter->chain_node.next,
2673 						struct ppc440spe_adma_desc_slot,
2674 						chain_node);
2675 			hw_desc = iter->hw_desc;
2676 			hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2677 			iter->src_cnt = 0;
2678 			iter->dst_cnt = 0;
2679 			ppc440spe_desc_set_dest_addr(iter, chan, 0,
2680 						     ppc440spe_chan->qdest, 0);
2681 			ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2682 			ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2683 						      len);
2684 			iter->unmap_len = 0;
2685 			/* override qdest to preserve original Q */
2686 			qdest = ppc440spe_chan->qdest;
2687 		}
2688 
2689 		/* Setup destinations for P/Q ops */
2690 		ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2691 
2692 		/* Setup zero QWORDs into DCHECK CDBs */
2693 		idst = dst_cnt;
2694 		list_for_each_entry_reverse(iter, &sw_desc->group_list,
2695 					    chain_node) {
2696 			/*
2697 			 * The last CDB corresponds to Q-parity check,
2698 			 * the one before last CDB corresponds
2699 			 * P-parity check
2700 			 */
2701 			if (idst == DMA_DEST_MAX_NUM) {
2702 				if (idst == dst_cnt) {
2703 					set_bit(PPC440SPE_DESC_QCHECK,
2704 						&iter->flags);
2705 				} else {
2706 					set_bit(PPC440SPE_DESC_PCHECK,
2707 						&iter->flags);
2708 				}
2709 			} else {
2710 				if (qdest) {
2711 					set_bit(PPC440SPE_DESC_QCHECK,
2712 						&iter->flags);
2713 				} else {
2714 					set_bit(PPC440SPE_DESC_PCHECK,
2715 						&iter->flags);
2716 				}
2717 			}
2718 			iter->xor_check_result = pqres;
2719 
2720 			/*
2721 			 * set it to zero, if check fail then result will
2722 			 * be updated
2723 			 */
2724 			*iter->xor_check_result = 0;
2725 			ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2726 				ppc440spe_qword);
2727 
2728 			if (!(--dst_cnt))
2729 				break;
2730 		}
2731 
2732 		/* Setup sources and mults for P/Q ops */
2733 		list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2734 						     chain_node) {
2735 			struct ppc440spe_adma_chan *chan;
2736 			u32 mult_dst;
2737 
2738 			chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2739 			ppc440spe_desc_set_src_addr(iter, chan, 0,
2740 						    DMA_CUED_XOR_HB,
2741 						    src[src_cnt - 1]);
2742 			if (qdest) {
2743 				mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2744 							   DMA_CDB_SG_DST1;
2745 				ppc440spe_desc_set_src_mult(iter, chan,
2746 							    DMA_CUED_MULT1_OFF,
2747 							    mult_dst,
2748 							    scf[src_cnt - 1]);
2749 			}
2750 			if (!(--src_cnt))
2751 				break;
2752 		}
2753 	}
2754 	spin_unlock_bh(&ppc440spe_chan->lock);
2755 	return sw_desc ? &sw_desc->async_tx : NULL;
2756 }
2757 
2758 /**
2759  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2760  * XOR ZERO_SUM operation
2761  */
ppc440spe_adma_prep_dma_xor_zero_sum(struct dma_chan * chan,dma_addr_t * src,unsigned int src_cnt,size_t len,enum sum_check_flags * result,unsigned long flags)2762 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2763 		struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2764 		size_t len, enum sum_check_flags *result, unsigned long flags)
2765 {
2766 	struct dma_async_tx_descriptor *tx;
2767 	dma_addr_t pq[2];
2768 
2769 	/* validate P, disable Q */
2770 	pq[0] = src[0];
2771 	pq[1] = 0;
2772 	flags |= DMA_PREP_PQ_DISABLE_Q;
2773 
2774 	tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2775 						src_cnt - 1, 0, len,
2776 						result, flags);
2777 	return tx;
2778 }
2779 
2780 /**
2781  * ppc440spe_adma_set_dest - set destination address into descriptor
2782  */
ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot * sw_desc,dma_addr_t addr,int index)2783 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2784 		dma_addr_t addr, int index)
2785 {
2786 	struct ppc440spe_adma_chan *chan;
2787 
2788 	BUG_ON(index >= sw_desc->dst_cnt);
2789 
2790 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2791 
2792 	switch (chan->device->id) {
2793 	case PPC440SPE_DMA0_ID:
2794 	case PPC440SPE_DMA1_ID:
2795 		/* to do: support transfers lengths >
2796 		 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2797 		 */
2798 		ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2799 			chan, 0, addr, index);
2800 		break;
2801 	case PPC440SPE_XOR_ID:
2802 		sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2803 		ppc440spe_desc_set_dest_addr(sw_desc,
2804 			chan, 0, addr, index);
2805 		break;
2806 	}
2807 }
2808 
ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot * iter,struct ppc440spe_adma_chan * chan,dma_addr_t addr)2809 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2810 		struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2811 {
2812 	/*  To clear destinations update the descriptor
2813 	 * (P or Q depending on index) as follows:
2814 	 * addr is destination (0 corresponds to SG2):
2815 	 */
2816 	ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2817 
2818 	/* ... and the addr is source: */
2819 	ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2820 
2821 	/* addr is always SG2 then the mult is always DST1 */
2822 	ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2823 				    DMA_CDB_SG_DST1, 1);
2824 }
2825 
2826 /**
2827  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2828  * for the PQXOR operation
2829  */
ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot * sw_desc,dma_addr_t * addrs,unsigned long flags)2830 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2831 		dma_addr_t *addrs, unsigned long flags)
2832 {
2833 	struct ppc440spe_adma_desc_slot *iter;
2834 	struct ppc440spe_adma_chan *chan;
2835 	dma_addr_t paddr, qaddr;
2836 	dma_addr_t addr = 0, ppath, qpath;
2837 	int index = 0, i;
2838 
2839 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2840 
2841 	if (flags & DMA_PREP_PQ_DISABLE_P)
2842 		paddr = 0;
2843 	else
2844 		paddr = addrs[0];
2845 
2846 	if (flags & DMA_PREP_PQ_DISABLE_Q)
2847 		qaddr = 0;
2848 	else
2849 		qaddr = addrs[1];
2850 
2851 	if (!paddr || !qaddr)
2852 		addr = paddr ? paddr : qaddr;
2853 
2854 	switch (chan->device->id) {
2855 	case PPC440SPE_DMA0_ID:
2856 	case PPC440SPE_DMA1_ID:
2857 		/* walk through the WXOR source list and set P/Q-destinations
2858 		 * for each slot:
2859 		 */
2860 		if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2861 			/* This is WXOR-only chain; may have 1/2 zero descs */
2862 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2863 				index++;
2864 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2865 				index++;
2866 
2867 			iter = ppc440spe_get_group_entry(sw_desc, index);
2868 			if (addr) {
2869 				/* one destination */
2870 				list_for_each_entry_from(iter,
2871 					&sw_desc->group_list, chain_node)
2872 					ppc440spe_desc_set_dest_addr(iter, chan,
2873 						DMA_CUED_XOR_BASE, addr, 0);
2874 			} else {
2875 				/* two destinations */
2876 				list_for_each_entry_from(iter,
2877 					&sw_desc->group_list, chain_node) {
2878 					ppc440spe_desc_set_dest_addr(iter, chan,
2879 						DMA_CUED_XOR_BASE, paddr, 0);
2880 					ppc440spe_desc_set_dest_addr(iter, chan,
2881 						DMA_CUED_XOR_BASE, qaddr, 1);
2882 				}
2883 			}
2884 
2885 			if (index) {
2886 				/*  To clear destinations update the descriptor
2887 				 * (1st,2nd, or both depending on flags)
2888 				 */
2889 				index = 0;
2890 				if (test_bit(PPC440SPE_ZERO_P,
2891 						&sw_desc->flags)) {
2892 					iter = ppc440spe_get_group_entry(
2893 							sw_desc, index++);
2894 					ppc440spe_adma_pq_zero_op(iter, chan,
2895 							paddr);
2896 				}
2897 
2898 				if (test_bit(PPC440SPE_ZERO_Q,
2899 						&sw_desc->flags)) {
2900 					iter = ppc440spe_get_group_entry(
2901 							sw_desc, index++);
2902 					ppc440spe_adma_pq_zero_op(iter, chan,
2903 							qaddr);
2904 				}
2905 
2906 				return;
2907 			}
2908 		} else {
2909 			/* This is RXOR-only or RXOR/WXOR mixed chain */
2910 
2911 			/* If we want to include destination into calculations,
2912 			 * then make dest addresses cued with mult=1 (XOR).
2913 			 */
2914 			ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2915 					DMA_CUED_XOR_HB :
2916 					DMA_CUED_XOR_BASE |
2917 						(1 << DMA_CUED_MULT1_OFF);
2918 			qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2919 					DMA_CUED_XOR_HB :
2920 					DMA_CUED_XOR_BASE |
2921 						(1 << DMA_CUED_MULT1_OFF);
2922 
2923 			/* Setup destination(s) in RXOR slot(s) */
2924 			iter = ppc440spe_get_group_entry(sw_desc, index++);
2925 			ppc440spe_desc_set_dest_addr(iter, chan,
2926 						paddr ? ppath : qpath,
2927 						paddr ? paddr : qaddr, 0);
2928 			if (!addr) {
2929 				/* two destinations */
2930 				iter = ppc440spe_get_group_entry(sw_desc,
2931 								 index++);
2932 				ppc440spe_desc_set_dest_addr(iter, chan,
2933 						qpath, qaddr, 0);
2934 			}
2935 
2936 			if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2937 				/* Setup destination(s) in remaining WXOR
2938 				 * slots
2939 				 */
2940 				iter = ppc440spe_get_group_entry(sw_desc,
2941 								 index);
2942 				if (addr) {
2943 					/* one destination */
2944 					list_for_each_entry_from(iter,
2945 					    &sw_desc->group_list,
2946 					    chain_node)
2947 						ppc440spe_desc_set_dest_addr(
2948 							iter, chan,
2949 							DMA_CUED_XOR_BASE,
2950 							addr, 0);
2951 
2952 				} else {
2953 					/* two destinations */
2954 					list_for_each_entry_from(iter,
2955 					    &sw_desc->group_list,
2956 					    chain_node) {
2957 						ppc440spe_desc_set_dest_addr(
2958 							iter, chan,
2959 							DMA_CUED_XOR_BASE,
2960 							paddr, 0);
2961 						ppc440spe_desc_set_dest_addr(
2962 							iter, chan,
2963 							DMA_CUED_XOR_BASE,
2964 							qaddr, 1);
2965 					}
2966 				}
2967 			}
2968 
2969 		}
2970 		break;
2971 
2972 	case PPC440SPE_XOR_ID:
2973 		/* DMA2 descriptors have only 1 destination, so there are
2974 		 * two chains - one for each dest.
2975 		 * If we want to include destination into calculations,
2976 		 * then make dest addresses cued with mult=1 (XOR).
2977 		 */
2978 		ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2979 				DMA_CUED_XOR_HB :
2980 				DMA_CUED_XOR_BASE |
2981 					(1 << DMA_CUED_MULT1_OFF);
2982 
2983 		qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2984 				DMA_CUED_XOR_HB :
2985 				DMA_CUED_XOR_BASE |
2986 					(1 << DMA_CUED_MULT1_OFF);
2987 
2988 		iter = ppc440spe_get_group_entry(sw_desc, 0);
2989 		for (i = 0; i < sw_desc->descs_per_op; i++) {
2990 			ppc440spe_desc_set_dest_addr(iter, chan,
2991 				paddr ? ppath : qpath,
2992 				paddr ? paddr : qaddr, 0);
2993 			iter = list_entry(iter->chain_node.next,
2994 					  struct ppc440spe_adma_desc_slot,
2995 					  chain_node);
2996 		}
2997 
2998 		if (!addr) {
2999 			/* Two destinations; setup Q here */
3000 			iter = ppc440spe_get_group_entry(sw_desc,
3001 				sw_desc->descs_per_op);
3002 			for (i = 0; i < sw_desc->descs_per_op; i++) {
3003 				ppc440spe_desc_set_dest_addr(iter,
3004 					chan, qpath, qaddr, 0);
3005 				iter = list_entry(iter->chain_node.next,
3006 						struct ppc440spe_adma_desc_slot,
3007 						chain_node);
3008 			}
3009 		}
3010 
3011 		break;
3012 	}
3013 }
3014 
3015 /**
3016  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3017  * for the PQ_ZERO_SUM operation
3018  */
ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot * sw_desc,dma_addr_t paddr,dma_addr_t qaddr)3019 static void ppc440spe_adma_pqzero_sum_set_dest(
3020 		struct ppc440spe_adma_desc_slot *sw_desc,
3021 		dma_addr_t paddr, dma_addr_t qaddr)
3022 {
3023 	struct ppc440spe_adma_desc_slot *iter, *end;
3024 	struct ppc440spe_adma_chan *chan;
3025 	dma_addr_t addr = 0;
3026 	int idx;
3027 
3028 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3029 
3030 	/* walk through the WXOR source list and set P/Q-destinations
3031 	 * for each slot
3032 	 */
3033 	idx = (paddr && qaddr) ? 2 : 1;
3034 	/* set end */
3035 	list_for_each_entry_reverse(end, &sw_desc->group_list,
3036 				    chain_node) {
3037 		if (!(--idx))
3038 			break;
3039 	}
3040 	/* set start */
3041 	idx = (paddr && qaddr) ? 2 : 1;
3042 	iter = ppc440spe_get_group_entry(sw_desc, idx);
3043 
3044 	if (paddr && qaddr) {
3045 		/* two destinations */
3046 		list_for_each_entry_from(iter, &sw_desc->group_list,
3047 					 chain_node) {
3048 			if (unlikely(iter == end))
3049 				break;
3050 			ppc440spe_desc_set_dest_addr(iter, chan,
3051 						DMA_CUED_XOR_BASE, paddr, 0);
3052 			ppc440spe_desc_set_dest_addr(iter, chan,
3053 						DMA_CUED_XOR_BASE, qaddr, 1);
3054 		}
3055 	} else {
3056 		/* one destination */
3057 		addr = paddr ? paddr : qaddr;
3058 		list_for_each_entry_from(iter, &sw_desc->group_list,
3059 					 chain_node) {
3060 			if (unlikely(iter == end))
3061 				break;
3062 			ppc440spe_desc_set_dest_addr(iter, chan,
3063 						DMA_CUED_XOR_BASE, addr, 0);
3064 		}
3065 	}
3066 
3067 	/*  The remaining descriptors are DATACHECK. These have no need in
3068 	 * destination. Actually, these destinations are used there
3069 	 * as sources for check operation. So, set addr as source.
3070 	 */
3071 	ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3072 
3073 	if (!addr) {
3074 		end = list_entry(end->chain_node.next,
3075 				 struct ppc440spe_adma_desc_slot, chain_node);
3076 		ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3077 	}
3078 }
3079 
3080 /**
3081  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3082  */
ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot * desc,int src_cnt)3083 static inline void ppc440spe_desc_set_xor_src_cnt(
3084 			struct ppc440spe_adma_desc_slot *desc,
3085 			int src_cnt)
3086 {
3087 	struct xor_cb *hw_desc = desc->hw_desc;
3088 
3089 	hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3090 	hw_desc->cbc |= src_cnt;
3091 }
3092 
3093 /**
3094  * ppc440spe_adma_pq_set_src - set source address into descriptor
3095  */
ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot * sw_desc,dma_addr_t addr,int index)3096 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3097 		dma_addr_t addr, int index)
3098 {
3099 	struct ppc440spe_adma_chan *chan;
3100 	dma_addr_t haddr = 0;
3101 	struct ppc440spe_adma_desc_slot *iter = NULL;
3102 
3103 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3104 
3105 	switch (chan->device->id) {
3106 	case PPC440SPE_DMA0_ID:
3107 	case PPC440SPE_DMA1_ID:
3108 		/* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3109 		 */
3110 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3111 			/* RXOR-only or RXOR/WXOR operation */
3112 			int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3113 				&sw_desc->flags) ?  2 : 3;
3114 
3115 			if (index == 0) {
3116 				/* 1st slot (RXOR) */
3117 				/* setup sources region (R1-2-3, R1-2-4,
3118 				 * or R1-2-5)
3119 				 */
3120 				if (test_bit(PPC440SPE_DESC_RXOR12,
3121 						&sw_desc->flags))
3122 					haddr = DMA_RXOR12 <<
3123 						DMA_CUED_REGION_OFF;
3124 				else if (test_bit(PPC440SPE_DESC_RXOR123,
3125 				    &sw_desc->flags))
3126 					haddr = DMA_RXOR123 <<
3127 						DMA_CUED_REGION_OFF;
3128 				else if (test_bit(PPC440SPE_DESC_RXOR124,
3129 				    &sw_desc->flags))
3130 					haddr = DMA_RXOR124 <<
3131 						DMA_CUED_REGION_OFF;
3132 				else if (test_bit(PPC440SPE_DESC_RXOR125,
3133 				    &sw_desc->flags))
3134 					haddr = DMA_RXOR125 <<
3135 						DMA_CUED_REGION_OFF;
3136 				else
3137 					BUG();
3138 				haddr |= DMA_CUED_XOR_BASE;
3139 				iter = ppc440spe_get_group_entry(sw_desc, 0);
3140 			} else if (index < iskip) {
3141 				/* 1st slot (RXOR)
3142 				 * shall actually set source address only once
3143 				 * instead of first <iskip>
3144 				 */
3145 				iter = NULL;
3146 			} else {
3147 				/* 2nd/3d and next slots (WXOR);
3148 				 * skip first slot with RXOR
3149 				 */
3150 				haddr = DMA_CUED_XOR_HB;
3151 				iter = ppc440spe_get_group_entry(sw_desc,
3152 				    index - iskip + sw_desc->dst_cnt);
3153 			}
3154 		} else {
3155 			int znum = 0;
3156 
3157 			/* WXOR-only operation; skip first slots with
3158 			 * zeroing destinations
3159 			 */
3160 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3161 				znum++;
3162 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3163 				znum++;
3164 
3165 			haddr = DMA_CUED_XOR_HB;
3166 			iter = ppc440spe_get_group_entry(sw_desc,
3167 					index + znum);
3168 		}
3169 
3170 		if (likely(iter)) {
3171 			ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3172 
3173 			if (!index &&
3174 			    test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3175 			    sw_desc->dst_cnt == 2) {
3176 				/* if we have two destinations for RXOR, then
3177 				 * setup source in the second descr too
3178 				 */
3179 				iter = ppc440spe_get_group_entry(sw_desc, 1);
3180 				ppc440spe_desc_set_src_addr(iter, chan, 0,
3181 					haddr, addr);
3182 			}
3183 		}
3184 		break;
3185 
3186 	case PPC440SPE_XOR_ID:
3187 		/* DMA2 may do Biskup */
3188 		iter = sw_desc->group_head;
3189 		if (iter->dst_cnt == 2) {
3190 			/* both P & Q calculations required; set P src here */
3191 			ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3192 
3193 			/* this is for Q */
3194 			iter = ppc440spe_get_group_entry(sw_desc,
3195 				sw_desc->descs_per_op);
3196 		}
3197 		ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3198 		break;
3199 	}
3200 }
3201 
3202 /**
3203  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3204  */
ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot * sw_desc,dma_addr_t addr,int index)3205 static void ppc440spe_adma_memcpy_xor_set_src(
3206 		struct ppc440spe_adma_desc_slot *sw_desc,
3207 		dma_addr_t addr, int index)
3208 {
3209 	struct ppc440spe_adma_chan *chan;
3210 
3211 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3212 	sw_desc = sw_desc->group_head;
3213 
3214 	if (likely(sw_desc))
3215 		ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3216 }
3217 
3218 /**
3219  * ppc440spe_adma_dma2rxor_inc_addr  -
3220  */
ppc440spe_adma_dma2rxor_inc_addr(struct ppc440spe_adma_desc_slot * desc,struct ppc440spe_rxor * cursor,int index,int src_cnt)3221 static void ppc440spe_adma_dma2rxor_inc_addr(
3222 		struct ppc440spe_adma_desc_slot *desc,
3223 		struct ppc440spe_rxor *cursor, int index, int src_cnt)
3224 {
3225 	cursor->addr_count++;
3226 	if (index == src_cnt - 1) {
3227 		ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3228 	} else if (cursor->addr_count == XOR_MAX_OPS) {
3229 		ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3230 		cursor->addr_count = 0;
3231 		cursor->desc_count++;
3232 	}
3233 }
3234 
3235 /**
3236  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3237  */
ppc440spe_adma_dma2rxor_prep_src(struct ppc440spe_adma_desc_slot * hdesc,struct ppc440spe_rxor * cursor,int index,int src_cnt,u32 addr)3238 static int ppc440spe_adma_dma2rxor_prep_src(
3239 		struct ppc440spe_adma_desc_slot *hdesc,
3240 		struct ppc440spe_rxor *cursor, int index,
3241 		int src_cnt, u32 addr)
3242 {
3243 	u32 sign;
3244 	struct ppc440spe_adma_desc_slot *desc = hdesc;
3245 	int i;
3246 
3247 	for (i = 0; i < cursor->desc_count; i++) {
3248 		desc = list_entry(hdesc->chain_node.next,
3249 				  struct ppc440spe_adma_desc_slot,
3250 				  chain_node);
3251 	}
3252 
3253 	switch (cursor->state) {
3254 	case 0:
3255 		if (addr == cursor->addrl + cursor->len) {
3256 			/* direct RXOR */
3257 			cursor->state = 1;
3258 			cursor->xor_count++;
3259 			if (index == src_cnt-1) {
3260 				ppc440spe_rxor_set_region(desc,
3261 					cursor->addr_count,
3262 					DMA_RXOR12 << DMA_CUED_REGION_OFF);
3263 				ppc440spe_adma_dma2rxor_inc_addr(
3264 					desc, cursor, index, src_cnt);
3265 			}
3266 		} else if (cursor->addrl == addr + cursor->len) {
3267 			/* reverse RXOR */
3268 			cursor->state = 1;
3269 			cursor->xor_count++;
3270 			set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3271 			if (index == src_cnt-1) {
3272 				ppc440spe_rxor_set_region(desc,
3273 					cursor->addr_count,
3274 					DMA_RXOR12 << DMA_CUED_REGION_OFF);
3275 				ppc440spe_adma_dma2rxor_inc_addr(
3276 					desc, cursor, index, src_cnt);
3277 			}
3278 		} else {
3279 			printk(KERN_ERR "Cannot build "
3280 				"DMA2 RXOR command block.\n");
3281 			BUG();
3282 		}
3283 		break;
3284 	case 1:
3285 		sign = test_bit(cursor->addr_count,
3286 				desc->reverse_flags)
3287 			? -1 : 1;
3288 		if (index == src_cnt-2 || (sign == -1
3289 			&& addr != cursor->addrl - 2*cursor->len)) {
3290 			cursor->state = 0;
3291 			cursor->xor_count = 1;
3292 			cursor->addrl = addr;
3293 			ppc440spe_rxor_set_region(desc,
3294 				cursor->addr_count,
3295 				DMA_RXOR12 << DMA_CUED_REGION_OFF);
3296 			ppc440spe_adma_dma2rxor_inc_addr(
3297 				desc, cursor, index, src_cnt);
3298 		} else if (addr == cursor->addrl + 2*sign*cursor->len) {
3299 			cursor->state = 2;
3300 			cursor->xor_count = 0;
3301 			ppc440spe_rxor_set_region(desc,
3302 				cursor->addr_count,
3303 				DMA_RXOR123 << DMA_CUED_REGION_OFF);
3304 			if (index == src_cnt-1) {
3305 				ppc440spe_adma_dma2rxor_inc_addr(
3306 					desc, cursor, index, src_cnt);
3307 			}
3308 		} else if (addr == cursor->addrl + 3*cursor->len) {
3309 			cursor->state = 2;
3310 			cursor->xor_count = 0;
3311 			ppc440spe_rxor_set_region(desc,
3312 				cursor->addr_count,
3313 				DMA_RXOR124 << DMA_CUED_REGION_OFF);
3314 			if (index == src_cnt-1) {
3315 				ppc440spe_adma_dma2rxor_inc_addr(
3316 					desc, cursor, index, src_cnt);
3317 			}
3318 		} else if (addr == cursor->addrl + 4*cursor->len) {
3319 			cursor->state = 2;
3320 			cursor->xor_count = 0;
3321 			ppc440spe_rxor_set_region(desc,
3322 				cursor->addr_count,
3323 				DMA_RXOR125 << DMA_CUED_REGION_OFF);
3324 			if (index == src_cnt-1) {
3325 				ppc440spe_adma_dma2rxor_inc_addr(
3326 					desc, cursor, index, src_cnt);
3327 			}
3328 		} else {
3329 			cursor->state = 0;
3330 			cursor->xor_count = 1;
3331 			cursor->addrl = addr;
3332 			ppc440spe_rxor_set_region(desc,
3333 				cursor->addr_count,
3334 				DMA_RXOR12 << DMA_CUED_REGION_OFF);
3335 			ppc440spe_adma_dma2rxor_inc_addr(
3336 				desc, cursor, index, src_cnt);
3337 		}
3338 		break;
3339 	case 2:
3340 		cursor->state = 0;
3341 		cursor->addrl = addr;
3342 		cursor->xor_count++;
3343 		if (index) {
3344 			ppc440spe_adma_dma2rxor_inc_addr(
3345 				desc, cursor, index, src_cnt);
3346 		}
3347 		break;
3348 	}
3349 
3350 	return 0;
3351 }
3352 
3353 /**
3354  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3355  *	ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3356  */
ppc440spe_adma_dma2rxor_set_src(struct ppc440spe_adma_desc_slot * desc,int index,dma_addr_t addr)3357 static void ppc440spe_adma_dma2rxor_set_src(
3358 		struct ppc440spe_adma_desc_slot *desc,
3359 		int index, dma_addr_t addr)
3360 {
3361 	struct xor_cb *xcb = desc->hw_desc;
3362 	int k = 0, op = 0, lop = 0;
3363 
3364 	/* get the RXOR operand which corresponds to index addr */
3365 	while (op <= index) {
3366 		lop = op;
3367 		if (k == XOR_MAX_OPS) {
3368 			k = 0;
3369 			desc = list_entry(desc->chain_node.next,
3370 				struct ppc440spe_adma_desc_slot, chain_node);
3371 			xcb = desc->hw_desc;
3372 
3373 		}
3374 		if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3375 		    (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3376 			op += 2;
3377 		else
3378 			op += 3;
3379 	}
3380 
3381 	BUG_ON(k < 1);
3382 
3383 	if (test_bit(k-1, desc->reverse_flags)) {
3384 		/* reverse operand order; put last op in RXOR group */
3385 		if (index == op - 1)
3386 			ppc440spe_rxor_set_src(desc, k - 1, addr);
3387 	} else {
3388 		/* direct operand order; put first op in RXOR group */
3389 		if (index == lop)
3390 			ppc440spe_rxor_set_src(desc, k - 1, addr);
3391 	}
3392 }
3393 
3394 /**
3395  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3396  *	ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3397  */
ppc440spe_adma_dma2rxor_set_mult(struct ppc440spe_adma_desc_slot * desc,int index,u8 mult)3398 static void ppc440spe_adma_dma2rxor_set_mult(
3399 		struct ppc440spe_adma_desc_slot *desc,
3400 		int index, u8 mult)
3401 {
3402 	struct xor_cb *xcb = desc->hw_desc;
3403 	int k = 0, op = 0, lop = 0;
3404 
3405 	/* get the RXOR operand which corresponds to index mult */
3406 	while (op <= index) {
3407 		lop = op;
3408 		if (k == XOR_MAX_OPS) {
3409 			k = 0;
3410 			desc = list_entry(desc->chain_node.next,
3411 					  struct ppc440spe_adma_desc_slot,
3412 					  chain_node);
3413 			xcb = desc->hw_desc;
3414 
3415 		}
3416 		if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3417 		    (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3418 			op += 2;
3419 		else
3420 			op += 3;
3421 	}
3422 
3423 	BUG_ON(k < 1);
3424 	if (test_bit(k-1, desc->reverse_flags)) {
3425 		/* reverse order */
3426 		ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3427 	} else {
3428 		/* direct order */
3429 		ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3430 	}
3431 }
3432 
3433 /**
3434  * ppc440spe_init_rxor_cursor -
3435  */
ppc440spe_init_rxor_cursor(struct ppc440spe_rxor * cursor)3436 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3437 {
3438 	memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3439 	cursor->state = 2;
3440 }
3441 
3442 /**
3443  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3444  * descriptor for the PQXOR operation
3445  */
ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot * sw_desc,unsigned char mult,int index,int dst_pos)3446 static void ppc440spe_adma_pq_set_src_mult(
3447 		struct ppc440spe_adma_desc_slot *sw_desc,
3448 		unsigned char mult, int index, int dst_pos)
3449 {
3450 	struct ppc440spe_adma_chan *chan;
3451 	u32 mult_idx, mult_dst;
3452 	struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3453 
3454 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3455 
3456 	switch (chan->device->id) {
3457 	case PPC440SPE_DMA0_ID:
3458 	case PPC440SPE_DMA1_ID:
3459 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3460 			int region = test_bit(PPC440SPE_DESC_RXOR12,
3461 					&sw_desc->flags) ? 2 : 3;
3462 
3463 			if (index < region) {
3464 				/* RXOR multipliers */
3465 				iter = ppc440spe_get_group_entry(sw_desc,
3466 					sw_desc->dst_cnt - 1);
3467 				if (sw_desc->dst_cnt == 2)
3468 					iter1 = ppc440spe_get_group_entry(
3469 							sw_desc, 0);
3470 
3471 				mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3472 				mult_dst = DMA_CDB_SG_SRC;
3473 			} else {
3474 				/* WXOR multiplier */
3475 				iter = ppc440spe_get_group_entry(sw_desc,
3476 							index - region +
3477 							sw_desc->dst_cnt);
3478 				mult_idx = DMA_CUED_MULT1_OFF;
3479 				mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3480 						     DMA_CDB_SG_DST1;
3481 			}
3482 		} else {
3483 			int znum = 0;
3484 
3485 			/* WXOR-only;
3486 			 * skip first slots with destinations (if ZERO_DST has
3487 			 * place)
3488 			 */
3489 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3490 				znum++;
3491 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3492 				znum++;
3493 
3494 			iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3495 			mult_idx = DMA_CUED_MULT1_OFF;
3496 			mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3497 		}
3498 
3499 		if (likely(iter)) {
3500 			ppc440spe_desc_set_src_mult(iter, chan,
3501 				mult_idx, mult_dst, mult);
3502 
3503 			if (unlikely(iter1)) {
3504 				/* if we have two destinations for RXOR, then
3505 				 * we've just set Q mult. Set-up P now.
3506 				 */
3507 				ppc440spe_desc_set_src_mult(iter1, chan,
3508 					mult_idx, mult_dst, 1);
3509 			}
3510 
3511 		}
3512 		break;
3513 
3514 	case PPC440SPE_XOR_ID:
3515 		iter = sw_desc->group_head;
3516 		if (sw_desc->dst_cnt == 2) {
3517 			/* both P & Q calculations required; set P mult here */
3518 			ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3519 
3520 			/* and then set Q mult */
3521 			iter = ppc440spe_get_group_entry(sw_desc,
3522 			       sw_desc->descs_per_op);
3523 		}
3524 		ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3525 		break;
3526 	}
3527 }
3528 
3529 /**
3530  * ppc440spe_adma_free_chan_resources - free the resources allocated
3531  */
ppc440spe_adma_free_chan_resources(struct dma_chan * chan)3532 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3533 {
3534 	struct ppc440spe_adma_chan *ppc440spe_chan;
3535 	struct ppc440spe_adma_desc_slot *iter, *_iter;
3536 	int in_use_descs = 0;
3537 
3538 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3539 	ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3540 
3541 	spin_lock_bh(&ppc440spe_chan->lock);
3542 	list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3543 					chain_node) {
3544 		in_use_descs++;
3545 		list_del(&iter->chain_node);
3546 	}
3547 	list_for_each_entry_safe_reverse(iter, _iter,
3548 			&ppc440spe_chan->all_slots, slot_node) {
3549 		list_del(&iter->slot_node);
3550 		kfree(iter);
3551 		ppc440spe_chan->slots_allocated--;
3552 	}
3553 	ppc440spe_chan->last_used = NULL;
3554 
3555 	dev_dbg(ppc440spe_chan->device->common.dev,
3556 		"ppc440spe adma%d %s slots_allocated %d\n",
3557 		ppc440spe_chan->device->id,
3558 		__func__, ppc440spe_chan->slots_allocated);
3559 	spin_unlock_bh(&ppc440spe_chan->lock);
3560 
3561 	/* one is ok since we left it on there on purpose */
3562 	if (in_use_descs > 1)
3563 		printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3564 			in_use_descs - 1);
3565 }
3566 
3567 /**
3568  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3569  * @chan: ADMA channel handle
3570  * @cookie: ADMA transaction identifier
3571  * @txstate: a holder for the current state of the channel
3572  */
ppc440spe_adma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)3573 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3574 			dma_cookie_t cookie, struct dma_tx_state *txstate)
3575 {
3576 	struct ppc440spe_adma_chan *ppc440spe_chan;
3577 	enum dma_status ret;
3578 
3579 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3580 	ret = dma_cookie_status(chan, cookie, txstate);
3581 	if (ret == DMA_COMPLETE)
3582 		return ret;
3583 
3584 	ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3585 
3586 	return dma_cookie_status(chan, cookie, txstate);
3587 }
3588 
3589 /**
3590  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3591  */
ppc440spe_adma_eot_handler(int irq,void * data)3592 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3593 {
3594 	struct ppc440spe_adma_chan *chan = data;
3595 
3596 	dev_dbg(chan->device->common.dev,
3597 		"ppc440spe adma%d: %s\n", chan->device->id, __func__);
3598 
3599 	tasklet_schedule(&chan->irq_tasklet);
3600 	ppc440spe_adma_device_clear_eot_status(chan);
3601 
3602 	return IRQ_HANDLED;
3603 }
3604 
3605 /**
3606  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3607  *	do the same things as a eot handler
3608  */
ppc440spe_adma_err_handler(int irq,void * data)3609 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3610 {
3611 	struct ppc440spe_adma_chan *chan = data;
3612 
3613 	dev_dbg(chan->device->common.dev,
3614 		"ppc440spe adma%d: %s\n", chan->device->id, __func__);
3615 
3616 	tasklet_schedule(&chan->irq_tasklet);
3617 	ppc440spe_adma_device_clear_eot_status(chan);
3618 
3619 	return IRQ_HANDLED;
3620 }
3621 
3622 /**
3623  * ppc440spe_test_callback - called when test operation has been done
3624  */
ppc440spe_test_callback(void * unused)3625 static void ppc440spe_test_callback(void *unused)
3626 {
3627 	complete(&ppc440spe_r6_test_comp);
3628 }
3629 
3630 /**
3631  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3632  */
ppc440spe_adma_issue_pending(struct dma_chan * chan)3633 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3634 {
3635 	struct ppc440spe_adma_chan *ppc440spe_chan;
3636 
3637 	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3638 	dev_dbg(ppc440spe_chan->device->common.dev,
3639 		"ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3640 		__func__, ppc440spe_chan->pending);
3641 
3642 	if (ppc440spe_chan->pending) {
3643 		ppc440spe_chan->pending = 0;
3644 		ppc440spe_chan_append(ppc440spe_chan);
3645 	}
3646 }
3647 
3648 /**
3649  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3650  *	use FIFOs (as opposite to chains used in XOR) so this is a XOR
3651  *	specific operation)
3652  */
ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan * chan)3653 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3654 {
3655 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3656 	dma_cookie_t cookie;
3657 	int slot_cnt, slots_per_op;
3658 
3659 	dev_dbg(chan->device->common.dev,
3660 		"ppc440spe adma%d: %s\n", chan->device->id, __func__);
3661 
3662 	spin_lock_bh(&chan->lock);
3663 	slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3664 	sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3665 	if (sw_desc) {
3666 		group_start = sw_desc->group_head;
3667 		list_splice_init(&sw_desc->group_list, &chan->chain);
3668 		async_tx_ack(&sw_desc->async_tx);
3669 		ppc440spe_desc_init_null_xor(group_start);
3670 
3671 		cookie = dma_cookie_assign(&sw_desc->async_tx);
3672 
3673 		/* initialize the completed cookie to be less than
3674 		 * the most recently used cookie
3675 		 */
3676 		chan->common.completed_cookie = cookie - 1;
3677 
3678 		/* channel should not be busy */
3679 		BUG_ON(ppc440spe_chan_is_busy(chan));
3680 
3681 		/* set the descriptor address */
3682 		ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3683 
3684 		/* run the descriptor */
3685 		ppc440spe_chan_run(chan);
3686 	} else
3687 		printk(KERN_ERR "ppc440spe adma%d"
3688 			" failed to allocate null descriptor\n",
3689 			chan->device->id);
3690 	spin_unlock_bh(&chan->lock);
3691 }
3692 
3693 /**
3694  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3695  *	For this we just perform one WXOR operation with the same source
3696  *	and destination addresses, the GF-multiplier is 1; so if RAID-6
3697  *	capabilities are enabled then we'll get src/dst filled with zero.
3698  */
ppc440spe_test_raid6(struct ppc440spe_adma_chan * chan)3699 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3700 {
3701 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3702 	struct page *pg;
3703 	char *a;
3704 	dma_addr_t dma_addr, addrs[2];
3705 	unsigned long op = 0;
3706 	int rval = 0;
3707 
3708 	set_bit(PPC440SPE_DESC_WXOR, &op);
3709 
3710 	pg = alloc_page(GFP_KERNEL);
3711 	if (!pg)
3712 		return -ENOMEM;
3713 
3714 	spin_lock_bh(&chan->lock);
3715 	sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3716 	if (sw_desc) {
3717 		/* 1 src, 1 dsr, int_ena, WXOR */
3718 		ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3719 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3720 			ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3721 			iter->unmap_len = PAGE_SIZE;
3722 		}
3723 	} else {
3724 		rval = -EFAULT;
3725 		spin_unlock_bh(&chan->lock);
3726 		goto exit;
3727 	}
3728 	spin_unlock_bh(&chan->lock);
3729 
3730 	/* Fill the test page with ones */
3731 	memset(page_address(pg), 0xFF, PAGE_SIZE);
3732 	dma_addr = dma_map_page(chan->device->dev, pg, 0,
3733 				PAGE_SIZE, DMA_BIDIRECTIONAL);
3734 
3735 	/* Setup addresses */
3736 	ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3737 	ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3738 	addrs[0] = dma_addr;
3739 	addrs[1] = 0;
3740 	ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3741 
3742 	async_tx_ack(&sw_desc->async_tx);
3743 	sw_desc->async_tx.callback = ppc440spe_test_callback;
3744 	sw_desc->async_tx.callback_param = NULL;
3745 
3746 	init_completion(&ppc440spe_r6_test_comp);
3747 
3748 	ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3749 	ppc440spe_adma_issue_pending(&chan->common);
3750 
3751 	wait_for_completion(&ppc440spe_r6_test_comp);
3752 
3753 	/* Now check if the test page is zeroed */
3754 	a = page_address(pg);
3755 	if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3756 		/* page is zero - RAID-6 enabled */
3757 		rval = 0;
3758 	} else {
3759 		/* RAID-6 was not enabled */
3760 		rval = -EINVAL;
3761 	}
3762 exit:
3763 	__free_page(pg);
3764 	return rval;
3765 }
3766 
ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device * adev)3767 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3768 {
3769 	switch (adev->id) {
3770 	case PPC440SPE_DMA0_ID:
3771 	case PPC440SPE_DMA1_ID:
3772 		dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3773 		dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3774 		dma_cap_set(DMA_PQ, adev->common.cap_mask);
3775 		dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3776 		dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3777 		break;
3778 	case PPC440SPE_XOR_ID:
3779 		dma_cap_set(DMA_XOR, adev->common.cap_mask);
3780 		dma_cap_set(DMA_PQ, adev->common.cap_mask);
3781 		dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3782 		adev->common.cap_mask = adev->common.cap_mask;
3783 		break;
3784 	}
3785 
3786 	/* Set base routines */
3787 	adev->common.device_alloc_chan_resources =
3788 				ppc440spe_adma_alloc_chan_resources;
3789 	adev->common.device_free_chan_resources =
3790 				ppc440spe_adma_free_chan_resources;
3791 	adev->common.device_tx_status = ppc440spe_adma_tx_status;
3792 	adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3793 
3794 	/* Set prep routines based on capability */
3795 	if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3796 		adev->common.device_prep_dma_memcpy =
3797 			ppc440spe_adma_prep_dma_memcpy;
3798 	}
3799 	if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3800 		adev->common.max_xor = XOR_MAX_OPS;
3801 		adev->common.device_prep_dma_xor =
3802 			ppc440spe_adma_prep_dma_xor;
3803 	}
3804 	if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3805 		switch (adev->id) {
3806 		case PPC440SPE_DMA0_ID:
3807 			dma_set_maxpq(&adev->common,
3808 				DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3809 			break;
3810 		case PPC440SPE_DMA1_ID:
3811 			dma_set_maxpq(&adev->common,
3812 				DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3813 			break;
3814 		case PPC440SPE_XOR_ID:
3815 			adev->common.max_pq = XOR_MAX_OPS * 3;
3816 			break;
3817 		}
3818 		adev->common.device_prep_dma_pq =
3819 			ppc440spe_adma_prep_dma_pq;
3820 	}
3821 	if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3822 		switch (adev->id) {
3823 		case PPC440SPE_DMA0_ID:
3824 			adev->common.max_pq = DMA0_FIFO_SIZE /
3825 						sizeof(struct dma_cdb);
3826 			break;
3827 		case PPC440SPE_DMA1_ID:
3828 			adev->common.max_pq = DMA1_FIFO_SIZE /
3829 						sizeof(struct dma_cdb);
3830 			break;
3831 		}
3832 		adev->common.device_prep_dma_pq_val =
3833 			ppc440spe_adma_prep_dma_pqzero_sum;
3834 	}
3835 	if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3836 		switch (adev->id) {
3837 		case PPC440SPE_DMA0_ID:
3838 			adev->common.max_xor = DMA0_FIFO_SIZE /
3839 						sizeof(struct dma_cdb);
3840 			break;
3841 		case PPC440SPE_DMA1_ID:
3842 			adev->common.max_xor = DMA1_FIFO_SIZE /
3843 						sizeof(struct dma_cdb);
3844 			break;
3845 		}
3846 		adev->common.device_prep_dma_xor_val =
3847 			ppc440spe_adma_prep_dma_xor_zero_sum;
3848 	}
3849 	if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3850 		adev->common.device_prep_dma_interrupt =
3851 			ppc440spe_adma_prep_dma_interrupt;
3852 	}
3853 	pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3854 	  "( %s%s%s%s%s%s)\n",
3855 	  dev_name(adev->dev),
3856 	  dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3857 	  dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3858 	  dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3859 	  dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3860 	  dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3861 	  dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3862 }
3863 
ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device * adev,struct ppc440spe_adma_chan * chan,int * initcode)3864 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3865 				     struct ppc440spe_adma_chan *chan,
3866 				     int *initcode)
3867 {
3868 	struct platform_device *ofdev;
3869 	struct device_node *np;
3870 	int ret;
3871 
3872 	ofdev = container_of(adev->dev, struct platform_device, dev);
3873 	np = ofdev->dev.of_node;
3874 	if (adev->id != PPC440SPE_XOR_ID) {
3875 		adev->err_irq = irq_of_parse_and_map(np, 1);
3876 		if (!adev->err_irq) {
3877 			dev_warn(adev->dev, "no err irq resource?\n");
3878 			*initcode = PPC_ADMA_INIT_IRQ2;
3879 			adev->err_irq = -ENXIO;
3880 		} else
3881 			atomic_inc(&ppc440spe_adma_err_irq_ref);
3882 	} else {
3883 		adev->err_irq = -ENXIO;
3884 	}
3885 
3886 	adev->irq = irq_of_parse_and_map(np, 0);
3887 	if (!adev->irq) {
3888 		dev_err(adev->dev, "no irq resource\n");
3889 		*initcode = PPC_ADMA_INIT_IRQ1;
3890 		ret = -ENXIO;
3891 		goto err_irq_map;
3892 	}
3893 	dev_dbg(adev->dev, "irq %d, err irq %d\n",
3894 		adev->irq, adev->err_irq);
3895 
3896 	ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3897 			  0, dev_driver_string(adev->dev), chan);
3898 	if (ret) {
3899 		dev_err(adev->dev, "can't request irq %d\n",
3900 			adev->irq);
3901 		*initcode = PPC_ADMA_INIT_IRQ1;
3902 		ret = -EIO;
3903 		goto err_req1;
3904 	}
3905 
3906 	/* only DMA engines have a separate error IRQ
3907 	 * so it's Ok if err_irq < 0 in XOR engine case.
3908 	 */
3909 	if (adev->err_irq > 0) {
3910 		/* both DMA engines share common error IRQ */
3911 		ret = request_irq(adev->err_irq,
3912 				  ppc440spe_adma_err_handler,
3913 				  IRQF_SHARED,
3914 				  dev_driver_string(adev->dev),
3915 				  chan);
3916 		if (ret) {
3917 			dev_err(adev->dev, "can't request irq %d\n",
3918 				adev->err_irq);
3919 			*initcode = PPC_ADMA_INIT_IRQ2;
3920 			ret = -EIO;
3921 			goto err_req2;
3922 		}
3923 	}
3924 
3925 	if (adev->id == PPC440SPE_XOR_ID) {
3926 		/* enable XOR engine interrupts */
3927 		iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3928 			    XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3929 			    &adev->xor_reg->ier);
3930 	} else {
3931 		u32 mask, enable;
3932 
3933 		np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3934 		if (!np) {
3935 			pr_err("%s: can't find I2O device tree node\n",
3936 				__func__);
3937 			ret = -ENODEV;
3938 			goto err_req2;
3939 		}
3940 		adev->i2o_reg = of_iomap(np, 0);
3941 		if (!adev->i2o_reg) {
3942 			pr_err("%s: failed to map I2O registers\n", __func__);
3943 			of_node_put(np);
3944 			ret = -EINVAL;
3945 			goto err_req2;
3946 		}
3947 		of_node_put(np);
3948 		/* Unmask 'CS FIFO Attention' interrupts and
3949 		 * enable generating interrupts on errors
3950 		 */
3951 		enable = (adev->id == PPC440SPE_DMA0_ID) ?
3952 			 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3953 			 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3954 		mask = ioread32(&adev->i2o_reg->iopim) & enable;
3955 		iowrite32(mask, &adev->i2o_reg->iopim);
3956 	}
3957 	return 0;
3958 
3959 err_req2:
3960 	free_irq(adev->irq, chan);
3961 err_req1:
3962 	irq_dispose_mapping(adev->irq);
3963 err_irq_map:
3964 	if (adev->err_irq > 0) {
3965 		if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3966 			irq_dispose_mapping(adev->err_irq);
3967 	}
3968 	return ret;
3969 }
3970 
ppc440spe_adma_release_irqs(struct ppc440spe_adma_device * adev,struct ppc440spe_adma_chan * chan)3971 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3972 					struct ppc440spe_adma_chan *chan)
3973 {
3974 	u32 mask, disable;
3975 
3976 	if (adev->id == PPC440SPE_XOR_ID) {
3977 		/* disable XOR engine interrupts */
3978 		mask = ioread32be(&adev->xor_reg->ier);
3979 		mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3980 			  XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3981 		iowrite32be(mask, &adev->xor_reg->ier);
3982 	} else {
3983 		/* disable DMAx engine interrupts */
3984 		disable = (adev->id == PPC440SPE_DMA0_ID) ?
3985 			  (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3986 			  (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3987 		mask = ioread32(&adev->i2o_reg->iopim) | disable;
3988 		iowrite32(mask, &adev->i2o_reg->iopim);
3989 	}
3990 	free_irq(adev->irq, chan);
3991 	irq_dispose_mapping(adev->irq);
3992 	if (adev->err_irq > 0) {
3993 		free_irq(adev->err_irq, chan);
3994 		if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
3995 			irq_dispose_mapping(adev->err_irq);
3996 			iounmap(adev->i2o_reg);
3997 		}
3998 	}
3999 }
4000 
4001 /**
4002  * ppc440spe_adma_probe - probe the asynch device
4003  */
ppc440spe_adma_probe(struct platform_device * ofdev)4004 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4005 {
4006 	struct device_node *np = ofdev->dev.of_node;
4007 	struct resource res;
4008 	struct ppc440spe_adma_device *adev;
4009 	struct ppc440spe_adma_chan *chan;
4010 	struct ppc_dma_chan_ref *ref, *_ref;
4011 	int ret = 0, initcode = PPC_ADMA_INIT_OK;
4012 	const u32 *idx;
4013 	int len;
4014 	void *regs;
4015 	u32 id, pool_size;
4016 
4017 	if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4018 		id = PPC440SPE_XOR_ID;
4019 		/* As far as the XOR engine is concerned, it does not
4020 		 * use FIFOs but uses linked list. So there is no dependency
4021 		 * between pool size to allocate and the engine configuration.
4022 		 */
4023 		pool_size = PAGE_SIZE << 1;
4024 	} else {
4025 		/* it is DMA0 or DMA1 */
4026 		idx = of_get_property(np, "cell-index", &len);
4027 		if (!idx || (len != sizeof(u32))) {
4028 			dev_err(&ofdev->dev, "Device node %pOF has missing "
4029 				"or invalid cell-index property\n",
4030 				np);
4031 			return -EINVAL;
4032 		}
4033 		id = *idx;
4034 		/* DMA0,1 engines use FIFO to maintain CDBs, so we
4035 		 * should allocate the pool accordingly to size of this
4036 		 * FIFO. Thus, the pool size depends on the FIFO depth:
4037 		 * how much CDBs pointers the FIFO may contain then so
4038 		 * much CDBs we should provide in the pool.
4039 		 * That is
4040 		 *   CDB size = 32B;
4041 		 *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4042 		 *   Pool size = CDBs number * CDB size =
4043 		 *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4044 		 */
4045 		pool_size = (id == PPC440SPE_DMA0_ID) ?
4046 			    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4047 		pool_size <<= 2;
4048 	}
4049 
4050 	if (of_address_to_resource(np, 0, &res)) {
4051 		dev_err(&ofdev->dev, "failed to get memory resource\n");
4052 		initcode = PPC_ADMA_INIT_MEMRES;
4053 		ret = -ENODEV;
4054 		goto out;
4055 	}
4056 
4057 	if (!request_mem_region(res.start, resource_size(&res),
4058 				dev_driver_string(&ofdev->dev))) {
4059 		dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4060 			&res);
4061 		initcode = PPC_ADMA_INIT_MEMREG;
4062 		ret = -EBUSY;
4063 		goto out;
4064 	}
4065 
4066 	/* create a device */
4067 	adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4068 	if (!adev) {
4069 		initcode = PPC_ADMA_INIT_ALLOC;
4070 		ret = -ENOMEM;
4071 		goto err_adev_alloc;
4072 	}
4073 
4074 	adev->id = id;
4075 	adev->pool_size = pool_size;
4076 	/* allocate coherent memory for hardware descriptors */
4077 	adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4078 					adev->pool_size, &adev->dma_desc_pool,
4079 					GFP_KERNEL);
4080 	if (adev->dma_desc_pool_virt == NULL) {
4081 		dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4082 			"memory for hardware descriptors\n",
4083 			adev->pool_size);
4084 		initcode = PPC_ADMA_INIT_COHERENT;
4085 		ret = -ENOMEM;
4086 		goto err_dma_alloc;
4087 	}
4088 	dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4089 		adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4090 
4091 	regs = ioremap(res.start, resource_size(&res));
4092 	if (!regs) {
4093 		dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4094 		ret = -ENOMEM;
4095 		goto err_regs_alloc;
4096 	}
4097 
4098 	if (adev->id == PPC440SPE_XOR_ID) {
4099 		adev->xor_reg = regs;
4100 		/* Reset XOR */
4101 		iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4102 		iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4103 	} else {
4104 		size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4105 				   DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4106 		adev->dma_reg = regs;
4107 		/* DMAx_FIFO_SIZE is defined in bytes,
4108 		 * <fsiz> - is defined in number of CDB pointers (8byte).
4109 		 * DMA FIFO Length = CSlength + CPlength, where
4110 		 * CSlength = CPlength = (fsiz + 1) * 8.
4111 		 */
4112 		iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4113 			  &adev->dma_reg->fsiz);
4114 		/* Configure DMA engine */
4115 		iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4116 			  &adev->dma_reg->cfg);
4117 		/* Clear Status */
4118 		iowrite32(~0, &adev->dma_reg->dsts);
4119 	}
4120 
4121 	adev->dev = &ofdev->dev;
4122 	adev->common.dev = &ofdev->dev;
4123 	INIT_LIST_HEAD(&adev->common.channels);
4124 	platform_set_drvdata(ofdev, adev);
4125 
4126 	/* create a channel */
4127 	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4128 	if (!chan) {
4129 		initcode = PPC_ADMA_INIT_CHANNEL;
4130 		ret = -ENOMEM;
4131 		goto err_chan_alloc;
4132 	}
4133 
4134 	spin_lock_init(&chan->lock);
4135 	INIT_LIST_HEAD(&chan->chain);
4136 	INIT_LIST_HEAD(&chan->all_slots);
4137 	chan->device = adev;
4138 	chan->common.device = &adev->common;
4139 	dma_cookie_init(&chan->common);
4140 	list_add_tail(&chan->common.device_node, &adev->common.channels);
4141 	tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet);
4142 
4143 	/* allocate and map helper pages for async validation or
4144 	 * async_mult/async_sum_product operations on DMA0/1.
4145 	 */
4146 	if (adev->id != PPC440SPE_XOR_ID) {
4147 		chan->pdest_page = alloc_page(GFP_KERNEL);
4148 		chan->qdest_page = alloc_page(GFP_KERNEL);
4149 		if (!chan->pdest_page ||
4150 		    !chan->qdest_page) {
4151 			if (chan->pdest_page)
4152 				__free_page(chan->pdest_page);
4153 			if (chan->qdest_page)
4154 				__free_page(chan->qdest_page);
4155 			ret = -ENOMEM;
4156 			goto err_page_alloc;
4157 		}
4158 		chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4159 					   PAGE_SIZE, DMA_BIDIRECTIONAL);
4160 		chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4161 					   PAGE_SIZE, DMA_BIDIRECTIONAL);
4162 	}
4163 
4164 	ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4165 	if (ref) {
4166 		ref->chan = &chan->common;
4167 		INIT_LIST_HEAD(&ref->node);
4168 		list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4169 	} else {
4170 		dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4171 		ret = -ENOMEM;
4172 		goto err_ref_alloc;
4173 	}
4174 
4175 	ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4176 	if (ret)
4177 		goto err_irq;
4178 
4179 	ppc440spe_adma_init_capabilities(adev);
4180 
4181 	ret = dma_async_device_register(&adev->common);
4182 	if (ret) {
4183 		initcode = PPC_ADMA_INIT_REGISTER;
4184 		dev_err(&ofdev->dev, "failed to register dma device\n");
4185 		goto err_dev_reg;
4186 	}
4187 
4188 	goto out;
4189 
4190 err_dev_reg:
4191 	ppc440spe_adma_release_irqs(adev, chan);
4192 err_irq:
4193 	list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4194 		if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4195 			list_del(&ref->node);
4196 			kfree(ref);
4197 		}
4198 	}
4199 err_ref_alloc:
4200 	if (adev->id != PPC440SPE_XOR_ID) {
4201 		dma_unmap_page(&ofdev->dev, chan->pdest,
4202 			       PAGE_SIZE, DMA_BIDIRECTIONAL);
4203 		dma_unmap_page(&ofdev->dev, chan->qdest,
4204 			       PAGE_SIZE, DMA_BIDIRECTIONAL);
4205 		__free_page(chan->pdest_page);
4206 		__free_page(chan->qdest_page);
4207 	}
4208 err_page_alloc:
4209 	kfree(chan);
4210 err_chan_alloc:
4211 	if (adev->id == PPC440SPE_XOR_ID)
4212 		iounmap(adev->xor_reg);
4213 	else
4214 		iounmap(adev->dma_reg);
4215 err_regs_alloc:
4216 	dma_free_coherent(adev->dev, adev->pool_size,
4217 			  adev->dma_desc_pool_virt,
4218 			  adev->dma_desc_pool);
4219 err_dma_alloc:
4220 	kfree(adev);
4221 err_adev_alloc:
4222 	release_mem_region(res.start, resource_size(&res));
4223 out:
4224 	if (id < PPC440SPE_ADMA_ENGINES_NUM)
4225 		ppc440spe_adma_devices[id] = initcode;
4226 
4227 	return ret;
4228 }
4229 
4230 /**
4231  * ppc440spe_adma_remove - remove the asynch device
4232  */
ppc440spe_adma_remove(struct platform_device * ofdev)4233 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4234 {
4235 	struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4236 	struct device_node *np = ofdev->dev.of_node;
4237 	struct resource res;
4238 	struct dma_chan *chan, *_chan;
4239 	struct ppc_dma_chan_ref *ref, *_ref;
4240 	struct ppc440spe_adma_chan *ppc440spe_chan;
4241 
4242 	if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4243 		ppc440spe_adma_devices[adev->id] = -1;
4244 
4245 	dma_async_device_unregister(&adev->common);
4246 
4247 	list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4248 				 device_node) {
4249 		ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4250 		ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4251 		tasklet_kill(&ppc440spe_chan->irq_tasklet);
4252 		if (adev->id != PPC440SPE_XOR_ID) {
4253 			dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4254 					PAGE_SIZE, DMA_BIDIRECTIONAL);
4255 			dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4256 					PAGE_SIZE, DMA_BIDIRECTIONAL);
4257 			__free_page(ppc440spe_chan->pdest_page);
4258 			__free_page(ppc440spe_chan->qdest_page);
4259 		}
4260 		list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4261 					 node) {
4262 			if (ppc440spe_chan ==
4263 			    to_ppc440spe_adma_chan(ref->chan)) {
4264 				list_del(&ref->node);
4265 				kfree(ref);
4266 			}
4267 		}
4268 		list_del(&chan->device_node);
4269 		kfree(ppc440spe_chan);
4270 	}
4271 
4272 	dma_free_coherent(adev->dev, adev->pool_size,
4273 			  adev->dma_desc_pool_virt, adev->dma_desc_pool);
4274 	if (adev->id == PPC440SPE_XOR_ID)
4275 		iounmap(adev->xor_reg);
4276 	else
4277 		iounmap(adev->dma_reg);
4278 	of_address_to_resource(np, 0, &res);
4279 	release_mem_region(res.start, resource_size(&res));
4280 	kfree(adev);
4281 	return 0;
4282 }
4283 
4284 /*
4285  * /sys driver interface to enable h/w RAID-6 capabilities
4286  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4287  * directory are "devices", "enable" and "poly".
4288  * "devices" shows available engines.
4289  * "enable" is used to enable RAID-6 capabilities or to check
4290  * whether these has been activated.
4291  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4292  */
4293 
devices_show(struct device_driver * dev,char * buf)4294 static ssize_t devices_show(struct device_driver *dev, char *buf)
4295 {
4296 	ssize_t size = 0;
4297 	int i;
4298 
4299 	for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4300 		if (ppc440spe_adma_devices[i] == -1)
4301 			continue;
4302 		size += sysfs_emit_at(buf, size, "PPC440SP(E)-ADMA.%d: %s\n",
4303 				     i, ppc_adma_errors[ppc440spe_adma_devices[i]]);
4304 	}
4305 	return size;
4306 }
4307 static DRIVER_ATTR_RO(devices);
4308 
enable_show(struct device_driver * dev,char * buf)4309 static ssize_t enable_show(struct device_driver *dev, char *buf)
4310 {
4311 	return sysfs_emit(buf, "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4312 			  ppc440spe_r6_enabled ? "EN" : "DIS");
4313 }
4314 
enable_store(struct device_driver * dev,const char * buf,size_t count)4315 static ssize_t enable_store(struct device_driver *dev, const char *buf,
4316 			    size_t count)
4317 {
4318 	unsigned long val;
4319 	int err;
4320 
4321 	if (!count || count > 11)
4322 		return -EINVAL;
4323 
4324 	if (!ppc440spe_r6_tchan)
4325 		return -EFAULT;
4326 
4327 	/* Write a key */
4328 	err = kstrtoul(buf, 16, &val);
4329 	if (err)
4330 		return err;
4331 
4332 	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4333 	isync();
4334 
4335 	/* Verify whether it really works now */
4336 	if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4337 		pr_info("PPC440SP(e) RAID-6 has been activated "
4338 			"successfully\n");
4339 		ppc440spe_r6_enabled = 1;
4340 	} else {
4341 		pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4342 			" Error key ?\n");
4343 		ppc440spe_r6_enabled = 0;
4344 	}
4345 	return count;
4346 }
4347 static DRIVER_ATTR_RW(enable);
4348 
poly_show(struct device_driver * dev,char * buf)4349 static ssize_t poly_show(struct device_driver *dev, char *buf)
4350 {
4351 	ssize_t size = 0;
4352 	u32 reg;
4353 
4354 #ifdef CONFIG_440SP
4355 	/* 440SP has fixed polynomial */
4356 	reg = 0x4d;
4357 #else
4358 	reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4359 	reg >>= MQ0_CFBHL_POLY;
4360 	reg &= 0xFF;
4361 #endif
4362 
4363 	size = sysfs_emit(buf, "PPC440SP(e) RAID-6 driver "
4364 			"uses 0x1%02x polynomial.\n", reg);
4365 	return size;
4366 }
4367 
poly_store(struct device_driver * dev,const char * buf,size_t count)4368 static ssize_t poly_store(struct device_driver *dev, const char *buf,
4369 			  size_t count)
4370 {
4371 	unsigned long reg, val;
4372 	int err;
4373 #ifdef CONFIG_440SP
4374 	/* 440SP uses default 0x14D polynomial only */
4375 	return -EINVAL;
4376 #endif
4377 
4378 	if (!count || count > 6)
4379 		return -EINVAL;
4380 
4381 	/* e.g., 0x14D or 0x11D */
4382 	err = kstrtoul(buf, 16, &val);
4383 	if (err)
4384 		return err;
4385 
4386 	if (val & ~0x1FF)
4387 		return -EINVAL;
4388 
4389 	val &= 0xFF;
4390 	reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4391 	reg &= ~(0xFF << MQ0_CFBHL_POLY);
4392 	reg |= val << MQ0_CFBHL_POLY;
4393 	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4394 
4395 	return count;
4396 }
4397 static DRIVER_ATTR_RW(poly);
4398 
4399 /*
4400  * Common initialisation for RAID engines; allocate memory for
4401  * DMAx FIFOs, perform configuration common for all DMA engines.
4402  * Further DMA engine specific configuration is done at probe time.
4403  */
ppc440spe_configure_raid_devices(void)4404 static int ppc440spe_configure_raid_devices(void)
4405 {
4406 	struct device_node *np;
4407 	struct resource i2o_res;
4408 	struct i2o_regs __iomem *i2o_reg;
4409 	dcr_host_t i2o_dcr_host;
4410 	unsigned int dcr_base, dcr_len;
4411 	int i, ret;
4412 
4413 	np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4414 	if (!np) {
4415 		pr_err("%s: can't find I2O device tree node\n",
4416 			__func__);
4417 		return -ENODEV;
4418 	}
4419 
4420 	if (of_address_to_resource(np, 0, &i2o_res)) {
4421 		of_node_put(np);
4422 		return -EINVAL;
4423 	}
4424 
4425 	i2o_reg = of_iomap(np, 0);
4426 	if (!i2o_reg) {
4427 		pr_err("%s: failed to map I2O registers\n", __func__);
4428 		of_node_put(np);
4429 		return -EINVAL;
4430 	}
4431 
4432 	/* Get I2O DCRs base */
4433 	dcr_base = dcr_resource_start(np, 0);
4434 	dcr_len = dcr_resource_len(np, 0);
4435 	if (!dcr_base && !dcr_len) {
4436 		pr_err("%pOF: can't get DCR registers base/len!\n", np);
4437 		of_node_put(np);
4438 		iounmap(i2o_reg);
4439 		return -ENODEV;
4440 	}
4441 
4442 	i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4443 	if (!DCR_MAP_OK(i2o_dcr_host)) {
4444 		pr_err("%pOF: failed to map DCRs!\n", np);
4445 		of_node_put(np);
4446 		iounmap(i2o_reg);
4447 		return -ENODEV;
4448 	}
4449 	of_node_put(np);
4450 
4451 	/* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4452 	 * the base address of FIFO memory space.
4453 	 * Actually we need twice more physical memory than programmed in the
4454 	 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4455 	 */
4456 	ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4457 					 GFP_KERNEL);
4458 	if (!ppc440spe_dma_fifo_buf) {
4459 		pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4460 		iounmap(i2o_reg);
4461 		dcr_unmap(i2o_dcr_host, dcr_len);
4462 		return -ENOMEM;
4463 	}
4464 
4465 	/*
4466 	 * Configure h/w
4467 	 */
4468 	/* Reset I2O/DMA */
4469 	mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4470 	mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4471 
4472 	/* Setup the base address of mmaped registers */
4473 	dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4474 	dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4475 						I2O_REG_ENABLE);
4476 	dcr_unmap(i2o_dcr_host, dcr_len);
4477 
4478 	/* Setup FIFO memory space base address */
4479 	iowrite32(0, &i2o_reg->ifbah);
4480 	iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4481 
4482 	/* set zero FIFO size for I2O, so the whole
4483 	 * ppc440spe_dma_fifo_buf is used by DMAs.
4484 	 * DMAx_FIFOs will be configured while probe.
4485 	 */
4486 	iowrite32(0, &i2o_reg->ifsiz);
4487 	iounmap(i2o_reg);
4488 
4489 	/* To prepare WXOR/RXOR functionality we need access to
4490 	 * Memory Queue Module DCRs (finally it will be enabled
4491 	 * via /sys interface of the ppc440spe ADMA driver).
4492 	 */
4493 	np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4494 	if (!np) {
4495 		pr_err("%s: can't find MQ device tree node\n",
4496 			__func__);
4497 		ret = -ENODEV;
4498 		goto out_free;
4499 	}
4500 
4501 	/* Get MQ DCRs base */
4502 	dcr_base = dcr_resource_start(np, 0);
4503 	dcr_len = dcr_resource_len(np, 0);
4504 	if (!dcr_base && !dcr_len) {
4505 		pr_err("%pOF: can't get DCR registers base/len!\n", np);
4506 		ret = -ENODEV;
4507 		goto out_mq;
4508 	}
4509 
4510 	ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4511 	if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4512 		pr_err("%pOF: failed to map DCRs!\n", np);
4513 		ret = -ENODEV;
4514 		goto out_mq;
4515 	}
4516 	of_node_put(np);
4517 	ppc440spe_mq_dcr_len = dcr_len;
4518 
4519 	/* Set HB alias */
4520 	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4521 
4522 	/* Set:
4523 	 * - LL transaction passing limit to 1;
4524 	 * - Memory controller cycle limit to 1;
4525 	 * - Galois Polynomial to 0x14d (default)
4526 	 */
4527 	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4528 		  (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4529 		  (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4530 
4531 	atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4532 	for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4533 		ppc440spe_adma_devices[i] = -1;
4534 
4535 	return 0;
4536 
4537 out_mq:
4538 	of_node_put(np);
4539 out_free:
4540 	kfree(ppc440spe_dma_fifo_buf);
4541 	return ret;
4542 }
4543 
4544 static const struct of_device_id ppc440spe_adma_of_match[] = {
4545 	{ .compatible	= "ibm,dma-440spe", },
4546 	{ .compatible	= "amcc,xor-accelerator", },
4547 	{},
4548 };
4549 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4550 
4551 static struct platform_driver ppc440spe_adma_driver = {
4552 	.probe = ppc440spe_adma_probe,
4553 	.remove = ppc440spe_adma_remove,
4554 	.driver = {
4555 		.name = "PPC440SP(E)-ADMA",
4556 		.of_match_table = ppc440spe_adma_of_match,
4557 	},
4558 };
4559 
ppc440spe_adma_init(void)4560 static __init int ppc440spe_adma_init(void)
4561 {
4562 	int ret;
4563 
4564 	ret = ppc440spe_configure_raid_devices();
4565 	if (ret)
4566 		return ret;
4567 
4568 	ret = platform_driver_register(&ppc440spe_adma_driver);
4569 	if (ret) {
4570 		pr_err("%s: failed to register platform driver\n",
4571 			__func__);
4572 		goto out_reg;
4573 	}
4574 
4575 	/* Initialization status */
4576 	ret = driver_create_file(&ppc440spe_adma_driver.driver,
4577 				 &driver_attr_devices);
4578 	if (ret)
4579 		goto out_dev;
4580 
4581 	/* RAID-6 h/w enable entry */
4582 	ret = driver_create_file(&ppc440spe_adma_driver.driver,
4583 				 &driver_attr_enable);
4584 	if (ret)
4585 		goto out_en;
4586 
4587 	/* GF polynomial to use */
4588 	ret = driver_create_file(&ppc440spe_adma_driver.driver,
4589 				 &driver_attr_poly);
4590 	if (!ret)
4591 		return ret;
4592 
4593 	driver_remove_file(&ppc440spe_adma_driver.driver,
4594 			   &driver_attr_enable);
4595 out_en:
4596 	driver_remove_file(&ppc440spe_adma_driver.driver,
4597 			   &driver_attr_devices);
4598 out_dev:
4599 	/* User will not be able to enable h/w RAID-6 */
4600 	pr_err("%s: failed to create RAID-6 driver interface\n",
4601 		__func__);
4602 	platform_driver_unregister(&ppc440spe_adma_driver);
4603 out_reg:
4604 	dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4605 	kfree(ppc440spe_dma_fifo_buf);
4606 	return ret;
4607 }
4608 
ppc440spe_adma_exit(void)4609 static void __exit ppc440spe_adma_exit(void)
4610 {
4611 	driver_remove_file(&ppc440spe_adma_driver.driver,
4612 			   &driver_attr_poly);
4613 	driver_remove_file(&ppc440spe_adma_driver.driver,
4614 			   &driver_attr_enable);
4615 	driver_remove_file(&ppc440spe_adma_driver.driver,
4616 			   &driver_attr_devices);
4617 	platform_driver_unregister(&ppc440spe_adma_driver);
4618 	dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4619 	kfree(ppc440spe_dma_fifo_buf);
4620 }
4621 
4622 arch_initcall(ppc440spe_adma_init);
4623 module_exit(ppc440spe_adma_exit);
4624 
4625 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4626 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4627 MODULE_LICENSE("GPL");
4628