xref: /openbmc/linux/drivers/crypto/talitos.c (revision 1c2dd16a)
1 /*
2  * talitos - Freescale Integrated Security Engine (SEC) device driver
3  *
4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
5  *
6  * Scatterlist Crypto API glue code copied from files with the following:
7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8  *
9  * Crypto algorithm registration code copied from hifn driver:
10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11  * All rights reserved.
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with this program; if not, write to the Free Software
25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26  */
27 
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_platform.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/io.h>
40 #include <linux/spinlock.h>
41 #include <linux/rtnetlink.h>
42 #include <linux/slab.h>
43 
44 #include <crypto/algapi.h>
45 #include <crypto/aes.h>
46 #include <crypto/des.h>
47 #include <crypto/sha.h>
48 #include <crypto/md5.h>
49 #include <crypto/internal/aead.h>
50 #include <crypto/authenc.h>
51 #include <crypto/skcipher.h>
52 #include <crypto/hash.h>
53 #include <crypto/internal/hash.h>
54 #include <crypto/scatterwalk.h>
55 
56 #include "talitos.h"
57 
58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
59 			   bool is_sec1)
60 {
61 	ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
62 	if (!is_sec1)
63 		ptr->eptr = upper_32_bits(dma_addr);
64 }
65 
66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
67 			     struct talitos_ptr *src_ptr, bool is_sec1)
68 {
69 	dst_ptr->ptr = src_ptr->ptr;
70 	if (!is_sec1)
71 		dst_ptr->eptr = src_ptr->eptr;
72 }
73 
74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
75 			       bool is_sec1)
76 {
77 	if (is_sec1) {
78 		ptr->res = 0;
79 		ptr->len1 = cpu_to_be16(len);
80 	} else {
81 		ptr->len = cpu_to_be16(len);
82 	}
83 }
84 
85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
86 					   bool is_sec1)
87 {
88 	if (is_sec1)
89 		return be16_to_cpu(ptr->len1);
90 	else
91 		return be16_to_cpu(ptr->len);
92 }
93 
94 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
95 				   bool is_sec1)
96 {
97 	if (!is_sec1)
98 		ptr->j_extent = val;
99 }
100 
101 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
102 {
103 	if (!is_sec1)
104 		ptr->j_extent |= val;
105 }
106 
107 /*
108  * map virtual single (contiguous) pointer to h/w descriptor pointer
109  */
110 static void map_single_talitos_ptr(struct device *dev,
111 				   struct talitos_ptr *ptr,
112 				   unsigned int len, void *data,
113 				   enum dma_data_direction dir)
114 {
115 	dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
116 	struct talitos_private *priv = dev_get_drvdata(dev);
117 	bool is_sec1 = has_ftr_sec1(priv);
118 
119 	to_talitos_ptr_len(ptr, len, is_sec1);
120 	to_talitos_ptr(ptr, dma_addr, is_sec1);
121 	to_talitos_ptr_ext_set(ptr, 0, is_sec1);
122 }
123 
124 /*
125  * unmap bus single (contiguous) h/w descriptor pointer
126  */
127 static void unmap_single_talitos_ptr(struct device *dev,
128 				     struct talitos_ptr *ptr,
129 				     enum dma_data_direction dir)
130 {
131 	struct talitos_private *priv = dev_get_drvdata(dev);
132 	bool is_sec1 = has_ftr_sec1(priv);
133 
134 	dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
135 			 from_talitos_ptr_len(ptr, is_sec1), dir);
136 }
137 
138 static int reset_channel(struct device *dev, int ch)
139 {
140 	struct talitos_private *priv = dev_get_drvdata(dev);
141 	unsigned int timeout = TALITOS_TIMEOUT;
142 	bool is_sec1 = has_ftr_sec1(priv);
143 
144 	if (is_sec1) {
145 		setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
146 			  TALITOS1_CCCR_LO_RESET);
147 
148 		while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
149 			TALITOS1_CCCR_LO_RESET) && --timeout)
150 			cpu_relax();
151 	} else {
152 		setbits32(priv->chan[ch].reg + TALITOS_CCCR,
153 			  TALITOS2_CCCR_RESET);
154 
155 		while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
156 			TALITOS2_CCCR_RESET) && --timeout)
157 			cpu_relax();
158 	}
159 
160 	if (timeout == 0) {
161 		dev_err(dev, "failed to reset channel %d\n", ch);
162 		return -EIO;
163 	}
164 
165 	/* set 36-bit addressing, done writeback enable and done IRQ enable */
166 	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
167 		  TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
168 
169 	/* and ICCR writeback, if available */
170 	if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
171 		setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
172 		          TALITOS_CCCR_LO_IWSE);
173 
174 	return 0;
175 }
176 
177 static int reset_device(struct device *dev)
178 {
179 	struct talitos_private *priv = dev_get_drvdata(dev);
180 	unsigned int timeout = TALITOS_TIMEOUT;
181 	bool is_sec1 = has_ftr_sec1(priv);
182 	u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
183 
184 	setbits32(priv->reg + TALITOS_MCR, mcr);
185 
186 	while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
187 	       && --timeout)
188 		cpu_relax();
189 
190 	if (priv->irq[1]) {
191 		mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
192 		setbits32(priv->reg + TALITOS_MCR, mcr);
193 	}
194 
195 	if (timeout == 0) {
196 		dev_err(dev, "failed to reset device\n");
197 		return -EIO;
198 	}
199 
200 	return 0;
201 }
202 
203 /*
204  * Reset and initialize the device
205  */
206 static int init_device(struct device *dev)
207 {
208 	struct talitos_private *priv = dev_get_drvdata(dev);
209 	int ch, err;
210 	bool is_sec1 = has_ftr_sec1(priv);
211 
212 	/*
213 	 * Master reset
214 	 * errata documentation: warning: certain SEC interrupts
215 	 * are not fully cleared by writing the MCR:SWR bit,
216 	 * set bit twice to completely reset
217 	 */
218 	err = reset_device(dev);
219 	if (err)
220 		return err;
221 
222 	err = reset_device(dev);
223 	if (err)
224 		return err;
225 
226 	/* reset channels */
227 	for (ch = 0; ch < priv->num_channels; ch++) {
228 		err = reset_channel(dev, ch);
229 		if (err)
230 			return err;
231 	}
232 
233 	/* enable channel done and error interrupts */
234 	if (is_sec1) {
235 		clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
236 		clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
237 		/* disable parity error check in DEU (erroneous? test vect.) */
238 		setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
239 	} else {
240 		setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
241 		setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
242 	}
243 
244 	/* disable integrity check error interrupts (use writeback instead) */
245 	if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
246 		setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
247 		          TALITOS_MDEUICR_LO_ICE);
248 
249 	return 0;
250 }
251 
252 /**
253  * talitos_submit - submits a descriptor to the device for processing
254  * @dev:	the SEC device to be used
255  * @ch:		the SEC device channel to be used
256  * @desc:	the descriptor to be processed by the device
257  * @callback:	whom to call when processing is complete
258  * @context:	a handle for use by caller (optional)
259  *
260  * desc must contain valid dma-mapped (bus physical) address pointers.
261  * callback must check err and feedback in descriptor header
262  * for device processing status.
263  */
264 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
265 		   void (*callback)(struct device *dev,
266 				    struct talitos_desc *desc,
267 				    void *context, int error),
268 		   void *context)
269 {
270 	struct talitos_private *priv = dev_get_drvdata(dev);
271 	struct talitos_request *request;
272 	unsigned long flags;
273 	int head;
274 	bool is_sec1 = has_ftr_sec1(priv);
275 
276 	spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
277 
278 	if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
279 		/* h/w fifo is full */
280 		spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
281 		return -EAGAIN;
282 	}
283 
284 	head = priv->chan[ch].head;
285 	request = &priv->chan[ch].fifo[head];
286 
287 	/* map descriptor and save caller data */
288 	if (is_sec1) {
289 		desc->hdr1 = desc->hdr;
290 		desc->next_desc = 0;
291 		request->dma_desc = dma_map_single(dev, &desc->hdr1,
292 						   TALITOS_DESC_SIZE,
293 						   DMA_BIDIRECTIONAL);
294 	} else {
295 		request->dma_desc = dma_map_single(dev, desc,
296 						   TALITOS_DESC_SIZE,
297 						   DMA_BIDIRECTIONAL);
298 	}
299 	request->callback = callback;
300 	request->context = context;
301 
302 	/* increment fifo head */
303 	priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
304 
305 	smp_wmb();
306 	request->desc = desc;
307 
308 	/* GO! */
309 	wmb();
310 	out_be32(priv->chan[ch].reg + TALITOS_FF,
311 		 upper_32_bits(request->dma_desc));
312 	out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
313 		 lower_32_bits(request->dma_desc));
314 
315 	spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
316 
317 	return -EINPROGRESS;
318 }
319 EXPORT_SYMBOL(talitos_submit);
320 
321 /*
322  * process what was done, notify callback of error if not
323  */
324 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
325 {
326 	struct talitos_private *priv = dev_get_drvdata(dev);
327 	struct talitos_request *request, saved_req;
328 	unsigned long flags;
329 	int tail, status;
330 	bool is_sec1 = has_ftr_sec1(priv);
331 
332 	spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
333 
334 	tail = priv->chan[ch].tail;
335 	while (priv->chan[ch].fifo[tail].desc) {
336 		__be32 hdr;
337 
338 		request = &priv->chan[ch].fifo[tail];
339 
340 		/* descriptors with their done bits set don't get the error */
341 		rmb();
342 		hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
343 
344 		if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
345 			status = 0;
346 		else
347 			if (!error)
348 				break;
349 			else
350 				status = error;
351 
352 		dma_unmap_single(dev, request->dma_desc,
353 				 TALITOS_DESC_SIZE,
354 				 DMA_BIDIRECTIONAL);
355 
356 		/* copy entries so we can call callback outside lock */
357 		saved_req.desc = request->desc;
358 		saved_req.callback = request->callback;
359 		saved_req.context = request->context;
360 
361 		/* release request entry in fifo */
362 		smp_wmb();
363 		request->desc = NULL;
364 
365 		/* increment fifo tail */
366 		priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
367 
368 		spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
369 
370 		atomic_dec(&priv->chan[ch].submit_count);
371 
372 		saved_req.callback(dev, saved_req.desc, saved_req.context,
373 				   status);
374 		/* channel may resume processing in single desc error case */
375 		if (error && !reset_ch && status == error)
376 			return;
377 		spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
378 		tail = priv->chan[ch].tail;
379 	}
380 
381 	spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
382 }
383 
384 /*
385  * process completed requests for channels that have done status
386  */
387 #define DEF_TALITOS1_DONE(name, ch_done_mask)				\
388 static void talitos1_done_##name(unsigned long data)			\
389 {									\
390 	struct device *dev = (struct device *)data;			\
391 	struct talitos_private *priv = dev_get_drvdata(dev);		\
392 	unsigned long flags;						\
393 									\
394 	if (ch_done_mask & 0x10000000)					\
395 		flush_channel(dev, 0, 0, 0);			\
396 	if (priv->num_channels == 1)					\
397 		goto out;						\
398 	if (ch_done_mask & 0x40000000)					\
399 		flush_channel(dev, 1, 0, 0);			\
400 	if (ch_done_mask & 0x00010000)					\
401 		flush_channel(dev, 2, 0, 0);			\
402 	if (ch_done_mask & 0x00040000)					\
403 		flush_channel(dev, 3, 0, 0);			\
404 									\
405 out:									\
406 	/* At this point, all completed channels have been processed */	\
407 	/* Unmask done interrupts for channels completed later on. */	\
408 	spin_lock_irqsave(&priv->reg_lock, flags);			\
409 	clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);		\
410 	clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);	\
411 	spin_unlock_irqrestore(&priv->reg_lock, flags);			\
412 }
413 
414 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
415 
416 #define DEF_TALITOS2_DONE(name, ch_done_mask)				\
417 static void talitos2_done_##name(unsigned long data)			\
418 {									\
419 	struct device *dev = (struct device *)data;			\
420 	struct talitos_private *priv = dev_get_drvdata(dev);		\
421 	unsigned long flags;						\
422 									\
423 	if (ch_done_mask & 1)						\
424 		flush_channel(dev, 0, 0, 0);				\
425 	if (priv->num_channels == 1)					\
426 		goto out;						\
427 	if (ch_done_mask & (1 << 2))					\
428 		flush_channel(dev, 1, 0, 0);				\
429 	if (ch_done_mask & (1 << 4))					\
430 		flush_channel(dev, 2, 0, 0);				\
431 	if (ch_done_mask & (1 << 6))					\
432 		flush_channel(dev, 3, 0, 0);				\
433 									\
434 out:									\
435 	/* At this point, all completed channels have been processed */	\
436 	/* Unmask done interrupts for channels completed later on. */	\
437 	spin_lock_irqsave(&priv->reg_lock, flags);			\
438 	setbits32(priv->reg + TALITOS_IMR, ch_done_mask);		\
439 	setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);	\
440 	spin_unlock_irqrestore(&priv->reg_lock, flags);			\
441 }
442 
443 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
444 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
445 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
446 
447 /*
448  * locate current (offending) descriptor
449  */
450 static u32 current_desc_hdr(struct device *dev, int ch)
451 {
452 	struct talitos_private *priv = dev_get_drvdata(dev);
453 	int tail, iter;
454 	dma_addr_t cur_desc;
455 
456 	cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
457 	cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
458 
459 	if (!cur_desc) {
460 		dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
461 		return 0;
462 	}
463 
464 	tail = priv->chan[ch].tail;
465 
466 	iter = tail;
467 	while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
468 		iter = (iter + 1) & (priv->fifo_len - 1);
469 		if (iter == tail) {
470 			dev_err(dev, "couldn't locate current descriptor\n");
471 			return 0;
472 		}
473 	}
474 
475 	return priv->chan[ch].fifo[iter].desc->hdr;
476 }
477 
478 /*
479  * user diagnostics; report root cause of error based on execution unit status
480  */
481 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
482 {
483 	struct talitos_private *priv = dev_get_drvdata(dev);
484 	int i;
485 
486 	if (!desc_hdr)
487 		desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
488 
489 	switch (desc_hdr & DESC_HDR_SEL0_MASK) {
490 	case DESC_HDR_SEL0_AFEU:
491 		dev_err(dev, "AFEUISR 0x%08x_%08x\n",
492 			in_be32(priv->reg_afeu + TALITOS_EUISR),
493 			in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
494 		break;
495 	case DESC_HDR_SEL0_DEU:
496 		dev_err(dev, "DEUISR 0x%08x_%08x\n",
497 			in_be32(priv->reg_deu + TALITOS_EUISR),
498 			in_be32(priv->reg_deu + TALITOS_EUISR_LO));
499 		break;
500 	case DESC_HDR_SEL0_MDEUA:
501 	case DESC_HDR_SEL0_MDEUB:
502 		dev_err(dev, "MDEUISR 0x%08x_%08x\n",
503 			in_be32(priv->reg_mdeu + TALITOS_EUISR),
504 			in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
505 		break;
506 	case DESC_HDR_SEL0_RNG:
507 		dev_err(dev, "RNGUISR 0x%08x_%08x\n",
508 			in_be32(priv->reg_rngu + TALITOS_ISR),
509 			in_be32(priv->reg_rngu + TALITOS_ISR_LO));
510 		break;
511 	case DESC_HDR_SEL0_PKEU:
512 		dev_err(dev, "PKEUISR 0x%08x_%08x\n",
513 			in_be32(priv->reg_pkeu + TALITOS_EUISR),
514 			in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
515 		break;
516 	case DESC_HDR_SEL0_AESU:
517 		dev_err(dev, "AESUISR 0x%08x_%08x\n",
518 			in_be32(priv->reg_aesu + TALITOS_EUISR),
519 			in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
520 		break;
521 	case DESC_HDR_SEL0_CRCU:
522 		dev_err(dev, "CRCUISR 0x%08x_%08x\n",
523 			in_be32(priv->reg_crcu + TALITOS_EUISR),
524 			in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
525 		break;
526 	case DESC_HDR_SEL0_KEU:
527 		dev_err(dev, "KEUISR 0x%08x_%08x\n",
528 			in_be32(priv->reg_pkeu + TALITOS_EUISR),
529 			in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
530 		break;
531 	}
532 
533 	switch (desc_hdr & DESC_HDR_SEL1_MASK) {
534 	case DESC_HDR_SEL1_MDEUA:
535 	case DESC_HDR_SEL1_MDEUB:
536 		dev_err(dev, "MDEUISR 0x%08x_%08x\n",
537 			in_be32(priv->reg_mdeu + TALITOS_EUISR),
538 			in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
539 		break;
540 	case DESC_HDR_SEL1_CRCU:
541 		dev_err(dev, "CRCUISR 0x%08x_%08x\n",
542 			in_be32(priv->reg_crcu + TALITOS_EUISR),
543 			in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
544 		break;
545 	}
546 
547 	for (i = 0; i < 8; i++)
548 		dev_err(dev, "DESCBUF 0x%08x_%08x\n",
549 			in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
550 			in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
551 }
552 
553 /*
554  * recover from error interrupts
555  */
556 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
557 {
558 	struct talitos_private *priv = dev_get_drvdata(dev);
559 	unsigned int timeout = TALITOS_TIMEOUT;
560 	int ch, error, reset_dev = 0;
561 	u32 v_lo;
562 	bool is_sec1 = has_ftr_sec1(priv);
563 	int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
564 
565 	for (ch = 0; ch < priv->num_channels; ch++) {
566 		/* skip channels without errors */
567 		if (is_sec1) {
568 			/* bits 29, 31, 17, 19 */
569 			if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
570 				continue;
571 		} else {
572 			if (!(isr & (1 << (ch * 2 + 1))))
573 				continue;
574 		}
575 
576 		error = -EINVAL;
577 
578 		v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
579 
580 		if (v_lo & TALITOS_CCPSR_LO_DOF) {
581 			dev_err(dev, "double fetch fifo overflow error\n");
582 			error = -EAGAIN;
583 			reset_ch = 1;
584 		}
585 		if (v_lo & TALITOS_CCPSR_LO_SOF) {
586 			/* h/w dropped descriptor */
587 			dev_err(dev, "single fetch fifo overflow error\n");
588 			error = -EAGAIN;
589 		}
590 		if (v_lo & TALITOS_CCPSR_LO_MDTE)
591 			dev_err(dev, "master data transfer error\n");
592 		if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
593 			dev_err(dev, is_sec1 ? "pointer not complete error\n"
594 					     : "s/g data length zero error\n");
595 		if (v_lo & TALITOS_CCPSR_LO_FPZ)
596 			dev_err(dev, is_sec1 ? "parity error\n"
597 					     : "fetch pointer zero error\n");
598 		if (v_lo & TALITOS_CCPSR_LO_IDH)
599 			dev_err(dev, "illegal descriptor header error\n");
600 		if (v_lo & TALITOS_CCPSR_LO_IEU)
601 			dev_err(dev, is_sec1 ? "static assignment error\n"
602 					     : "invalid exec unit error\n");
603 		if (v_lo & TALITOS_CCPSR_LO_EU)
604 			report_eu_error(dev, ch, current_desc_hdr(dev, ch));
605 		if (!is_sec1) {
606 			if (v_lo & TALITOS_CCPSR_LO_GB)
607 				dev_err(dev, "gather boundary error\n");
608 			if (v_lo & TALITOS_CCPSR_LO_GRL)
609 				dev_err(dev, "gather return/length error\n");
610 			if (v_lo & TALITOS_CCPSR_LO_SB)
611 				dev_err(dev, "scatter boundary error\n");
612 			if (v_lo & TALITOS_CCPSR_LO_SRL)
613 				dev_err(dev, "scatter return/length error\n");
614 		}
615 
616 		flush_channel(dev, ch, error, reset_ch);
617 
618 		if (reset_ch) {
619 			reset_channel(dev, ch);
620 		} else {
621 			setbits32(priv->chan[ch].reg + TALITOS_CCCR,
622 				  TALITOS2_CCCR_CONT);
623 			setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
624 			while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
625 			       TALITOS2_CCCR_CONT) && --timeout)
626 				cpu_relax();
627 			if (timeout == 0) {
628 				dev_err(dev, "failed to restart channel %d\n",
629 					ch);
630 				reset_dev = 1;
631 			}
632 		}
633 	}
634 	if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
635 	    (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
636 		if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
637 			dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
638 				isr, isr_lo);
639 		else
640 			dev_err(dev, "done overflow, internal time out, or "
641 				"rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
642 
643 		/* purge request queues */
644 		for (ch = 0; ch < priv->num_channels; ch++)
645 			flush_channel(dev, ch, -EIO, 1);
646 
647 		/* reset and reinitialize the device */
648 		init_device(dev);
649 	}
650 }
651 
652 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)	       \
653 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)	       \
654 {									       \
655 	struct device *dev = data;					       \
656 	struct talitos_private *priv = dev_get_drvdata(dev);		       \
657 	u32 isr, isr_lo;						       \
658 	unsigned long flags;						       \
659 									       \
660 	spin_lock_irqsave(&priv->reg_lock, flags);			       \
661 	isr = in_be32(priv->reg + TALITOS_ISR);				       \
662 	isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);			       \
663 	/* Acknowledge interrupt */					       \
664 	out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
665 	out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);			       \
666 									       \
667 	if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
668 		spin_unlock_irqrestore(&priv->reg_lock, flags);		       \
669 		talitos_error(dev, isr & ch_err_mask, isr_lo);		       \
670 	}								       \
671 	else {								       \
672 		if (likely(isr & ch_done_mask)) {			       \
673 			/* mask further done interrupts. */		       \
674 			setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
675 			/* done_task will unmask done interrupts at exit */    \
676 			tasklet_schedule(&priv->done_task[tlet]);	       \
677 		}							       \
678 		spin_unlock_irqrestore(&priv->reg_lock, flags);		       \
679 	}								       \
680 									       \
681 	return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
682 								IRQ_NONE;      \
683 }
684 
685 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
686 
687 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)	       \
688 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)	       \
689 {									       \
690 	struct device *dev = data;					       \
691 	struct talitos_private *priv = dev_get_drvdata(dev);		       \
692 	u32 isr, isr_lo;						       \
693 	unsigned long flags;						       \
694 									       \
695 	spin_lock_irqsave(&priv->reg_lock, flags);			       \
696 	isr = in_be32(priv->reg + TALITOS_ISR);				       \
697 	isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);			       \
698 	/* Acknowledge interrupt */					       \
699 	out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
700 	out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);			       \
701 									       \
702 	if (unlikely(isr & ch_err_mask || isr_lo)) {			       \
703 		spin_unlock_irqrestore(&priv->reg_lock, flags);		       \
704 		talitos_error(dev, isr & ch_err_mask, isr_lo);		       \
705 	}								       \
706 	else {								       \
707 		if (likely(isr & ch_done_mask)) {			       \
708 			/* mask further done interrupts. */		       \
709 			clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
710 			/* done_task will unmask done interrupts at exit */    \
711 			tasklet_schedule(&priv->done_task[tlet]);	       \
712 		}							       \
713 		spin_unlock_irqrestore(&priv->reg_lock, flags);		       \
714 	}								       \
715 									       \
716 	return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
717 								IRQ_NONE;      \
718 }
719 
720 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
721 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
722 		       0)
723 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
724 		       1)
725 
726 /*
727  * hwrng
728  */
729 static int talitos_rng_data_present(struct hwrng *rng, int wait)
730 {
731 	struct device *dev = (struct device *)rng->priv;
732 	struct talitos_private *priv = dev_get_drvdata(dev);
733 	u32 ofl;
734 	int i;
735 
736 	for (i = 0; i < 20; i++) {
737 		ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
738 		      TALITOS_RNGUSR_LO_OFL;
739 		if (ofl || !wait)
740 			break;
741 		udelay(10);
742 	}
743 
744 	return !!ofl;
745 }
746 
747 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
748 {
749 	struct device *dev = (struct device *)rng->priv;
750 	struct talitos_private *priv = dev_get_drvdata(dev);
751 
752 	/* rng fifo requires 64-bit accesses */
753 	*data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
754 	*data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
755 
756 	return sizeof(u32);
757 }
758 
759 static int talitos_rng_init(struct hwrng *rng)
760 {
761 	struct device *dev = (struct device *)rng->priv;
762 	struct talitos_private *priv = dev_get_drvdata(dev);
763 	unsigned int timeout = TALITOS_TIMEOUT;
764 
765 	setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
766 	while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
767 		 & TALITOS_RNGUSR_LO_RD)
768 	       && --timeout)
769 		cpu_relax();
770 	if (timeout == 0) {
771 		dev_err(dev, "failed to reset rng hw\n");
772 		return -ENODEV;
773 	}
774 
775 	/* start generating */
776 	setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
777 
778 	return 0;
779 }
780 
781 static int talitos_register_rng(struct device *dev)
782 {
783 	struct talitos_private *priv = dev_get_drvdata(dev);
784 	int err;
785 
786 	priv->rng.name		= dev_driver_string(dev),
787 	priv->rng.init		= talitos_rng_init,
788 	priv->rng.data_present	= talitos_rng_data_present,
789 	priv->rng.data_read	= talitos_rng_data_read,
790 	priv->rng.priv		= (unsigned long)dev;
791 
792 	err = hwrng_register(&priv->rng);
793 	if (!err)
794 		priv->rng_registered = true;
795 
796 	return err;
797 }
798 
799 static void talitos_unregister_rng(struct device *dev)
800 {
801 	struct talitos_private *priv = dev_get_drvdata(dev);
802 
803 	if (!priv->rng_registered)
804 		return;
805 
806 	hwrng_unregister(&priv->rng);
807 	priv->rng_registered = false;
808 }
809 
810 /*
811  * crypto alg
812  */
813 #define TALITOS_CRA_PRIORITY		3000
814 /*
815  * Defines a priority for doing AEAD with descriptors type
816  * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
817  */
818 #define TALITOS_CRA_PRIORITY_AEAD_HSNA	(TALITOS_CRA_PRIORITY - 1)
819 #define TALITOS_MAX_KEY_SIZE		96
820 #define TALITOS_MAX_IV_LENGTH		16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
821 
822 struct talitos_ctx {
823 	struct device *dev;
824 	int ch;
825 	__be32 desc_hdr_template;
826 	u8 key[TALITOS_MAX_KEY_SIZE];
827 	u8 iv[TALITOS_MAX_IV_LENGTH];
828 	unsigned int keylen;
829 	unsigned int enckeylen;
830 	unsigned int authkeylen;
831 };
832 
833 #define HASH_MAX_BLOCK_SIZE		SHA512_BLOCK_SIZE
834 #define TALITOS_MDEU_MAX_CONTEXT_SIZE	TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
835 
836 struct talitos_ahash_req_ctx {
837 	u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
838 	unsigned int hw_context_size;
839 	u8 buf[HASH_MAX_BLOCK_SIZE];
840 	u8 bufnext[HASH_MAX_BLOCK_SIZE];
841 	unsigned int swinit;
842 	unsigned int first;
843 	unsigned int last;
844 	unsigned int to_hash_later;
845 	unsigned int nbuf;
846 	struct scatterlist bufsl[2];
847 	struct scatterlist *psrc;
848 };
849 
850 struct talitos_export_state {
851 	u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
852 	u8 buf[HASH_MAX_BLOCK_SIZE];
853 	unsigned int swinit;
854 	unsigned int first;
855 	unsigned int last;
856 	unsigned int to_hash_later;
857 	unsigned int nbuf;
858 };
859 
860 static int aead_setkey(struct crypto_aead *authenc,
861 		       const u8 *key, unsigned int keylen)
862 {
863 	struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
864 	struct crypto_authenc_keys keys;
865 
866 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
867 		goto badkey;
868 
869 	if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
870 		goto badkey;
871 
872 	memcpy(ctx->key, keys.authkey, keys.authkeylen);
873 	memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
874 
875 	ctx->keylen = keys.authkeylen + keys.enckeylen;
876 	ctx->enckeylen = keys.enckeylen;
877 	ctx->authkeylen = keys.authkeylen;
878 
879 	return 0;
880 
881 badkey:
882 	crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
883 	return -EINVAL;
884 }
885 
886 /*
887  * talitos_edesc - s/w-extended descriptor
888  * @src_nents: number of segments in input scatterlist
889  * @dst_nents: number of segments in output scatterlist
890  * @icv_ool: whether ICV is out-of-line
891  * @iv_dma: dma address of iv for checking continuity and link table
892  * @dma_len: length of dma mapped link_tbl space
893  * @dma_link_tbl: bus physical address of link_tbl/buf
894  * @desc: h/w descriptor
895  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
896  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
897  *
898  * if decrypting (with authcheck), or either one of src_nents or dst_nents
899  * is greater than 1, an integrity check value is concatenated to the end
900  * of link_tbl data
901  */
902 struct talitos_edesc {
903 	int src_nents;
904 	int dst_nents;
905 	bool icv_ool;
906 	dma_addr_t iv_dma;
907 	int dma_len;
908 	dma_addr_t dma_link_tbl;
909 	struct talitos_desc desc;
910 	union {
911 		struct talitos_ptr link_tbl[0];
912 		u8 buf[0];
913 	};
914 };
915 
916 static void talitos_sg_unmap(struct device *dev,
917 			     struct talitos_edesc *edesc,
918 			     struct scatterlist *src,
919 			     struct scatterlist *dst,
920 			     unsigned int len, unsigned int offset)
921 {
922 	struct talitos_private *priv = dev_get_drvdata(dev);
923 	bool is_sec1 = has_ftr_sec1(priv);
924 	unsigned int src_nents = edesc->src_nents ? : 1;
925 	unsigned int dst_nents = edesc->dst_nents ? : 1;
926 
927 	if (is_sec1 && dst && dst_nents > 1) {
928 		dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
929 					   len, DMA_FROM_DEVICE);
930 		sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
931 				     offset);
932 	}
933 	if (src != dst) {
934 		if (src_nents == 1 || !is_sec1)
935 			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
936 
937 		if (dst && (dst_nents == 1 || !is_sec1))
938 			dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
939 	} else if (src_nents == 1 || !is_sec1) {
940 		dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
941 	}
942 }
943 
944 static void ipsec_esp_unmap(struct device *dev,
945 			    struct talitos_edesc *edesc,
946 			    struct aead_request *areq)
947 {
948 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
949 	struct talitos_ctx *ctx = crypto_aead_ctx(aead);
950 	unsigned int ivsize = crypto_aead_ivsize(aead);
951 
952 	if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)
953 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
954 					 DMA_FROM_DEVICE);
955 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
956 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
957 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
958 
959 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
960 			 areq->assoclen);
961 
962 	if (edesc->dma_len)
963 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
964 				 DMA_BIDIRECTIONAL);
965 
966 	if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
967 		unsigned int dst_nents = edesc->dst_nents ? : 1;
968 
969 		sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
970 				   areq->assoclen + areq->cryptlen - ivsize);
971 	}
972 }
973 
974 /*
975  * ipsec_esp descriptor callbacks
976  */
977 static void ipsec_esp_encrypt_done(struct device *dev,
978 				   struct talitos_desc *desc, void *context,
979 				   int err)
980 {
981 	struct talitos_private *priv = dev_get_drvdata(dev);
982 	bool is_sec1 = has_ftr_sec1(priv);
983 	struct aead_request *areq = context;
984 	struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
985 	unsigned int authsize = crypto_aead_authsize(authenc);
986 	struct talitos_edesc *edesc;
987 	struct scatterlist *sg;
988 	void *icvdata;
989 
990 	edesc = container_of(desc, struct talitos_edesc, desc);
991 
992 	ipsec_esp_unmap(dev, edesc, areq);
993 
994 	/* copy the generated ICV to dst */
995 	if (edesc->icv_ool) {
996 		if (is_sec1)
997 			icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
998 		else
999 			icvdata = &edesc->link_tbl[edesc->src_nents +
1000 						   edesc->dst_nents + 2];
1001 		sg = sg_last(areq->dst, edesc->dst_nents);
1002 		memcpy((char *)sg_virt(sg) + sg->length - authsize,
1003 		       icvdata, authsize);
1004 	}
1005 
1006 	kfree(edesc);
1007 
1008 	aead_request_complete(areq, err);
1009 }
1010 
1011 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1012 					  struct talitos_desc *desc,
1013 					  void *context, int err)
1014 {
1015 	struct aead_request *req = context;
1016 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1017 	unsigned int authsize = crypto_aead_authsize(authenc);
1018 	struct talitos_edesc *edesc;
1019 	struct scatterlist *sg;
1020 	char *oicv, *icv;
1021 	struct talitos_private *priv = dev_get_drvdata(dev);
1022 	bool is_sec1 = has_ftr_sec1(priv);
1023 
1024 	edesc = container_of(desc, struct talitos_edesc, desc);
1025 
1026 	ipsec_esp_unmap(dev, edesc, req);
1027 
1028 	if (!err) {
1029 		/* auth check */
1030 		sg = sg_last(req->dst, edesc->dst_nents ? : 1);
1031 		icv = (char *)sg_virt(sg) + sg->length - authsize;
1032 
1033 		if (edesc->dma_len) {
1034 			if (is_sec1)
1035 				oicv = (char *)&edesc->dma_link_tbl +
1036 					       req->assoclen + req->cryptlen;
1037 			else
1038 				oicv = (char *)
1039 				       &edesc->link_tbl[edesc->src_nents +
1040 							edesc->dst_nents + 2];
1041 			if (edesc->icv_ool)
1042 				icv = oicv + authsize;
1043 		} else
1044 			oicv = (char *)&edesc->link_tbl[0];
1045 
1046 		err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1047 	}
1048 
1049 	kfree(edesc);
1050 
1051 	aead_request_complete(req, err);
1052 }
1053 
1054 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1055 					  struct talitos_desc *desc,
1056 					  void *context, int err)
1057 {
1058 	struct aead_request *req = context;
1059 	struct talitos_edesc *edesc;
1060 
1061 	edesc = container_of(desc, struct talitos_edesc, desc);
1062 
1063 	ipsec_esp_unmap(dev, edesc, req);
1064 
1065 	/* check ICV auth status */
1066 	if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1067 		     DESC_HDR_LO_ICCR1_PASS))
1068 		err = -EBADMSG;
1069 
1070 	kfree(edesc);
1071 
1072 	aead_request_complete(req, err);
1073 }
1074 
1075 /*
1076  * convert scatterlist to SEC h/w link table format
1077  * stop at cryptlen bytes
1078  */
1079 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1080 				 unsigned int offset, int cryptlen,
1081 				 struct talitos_ptr *link_tbl_ptr)
1082 {
1083 	int n_sg = sg_count;
1084 	int count = 0;
1085 
1086 	while (cryptlen && sg && n_sg--) {
1087 		unsigned int len = sg_dma_len(sg);
1088 
1089 		if (offset >= len) {
1090 			offset -= len;
1091 			goto next;
1092 		}
1093 
1094 		len -= offset;
1095 
1096 		if (len > cryptlen)
1097 			len = cryptlen;
1098 
1099 		to_talitos_ptr(link_tbl_ptr + count,
1100 			       sg_dma_address(sg) + offset, 0);
1101 		to_talitos_ptr_len(link_tbl_ptr + count, len, 0);
1102 		to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1103 		count++;
1104 		cryptlen -= len;
1105 		offset = 0;
1106 
1107 next:
1108 		sg = sg_next(sg);
1109 	}
1110 
1111 	/* tag end of link table */
1112 	if (count > 0)
1113 		to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1114 				       DESC_PTR_LNKTBL_RETURN, 0);
1115 
1116 	return count;
1117 }
1118 
1119 int talitos_sg_map(struct device *dev, struct scatterlist *src,
1120 		   unsigned int len, struct talitos_edesc *edesc,
1121 		   struct talitos_ptr *ptr,
1122 		   int sg_count, unsigned int offset, int tbl_off)
1123 {
1124 	struct talitos_private *priv = dev_get_drvdata(dev);
1125 	bool is_sec1 = has_ftr_sec1(priv);
1126 
1127 	to_talitos_ptr_len(ptr, len, is_sec1);
1128 	to_talitos_ptr_ext_set(ptr, 0, is_sec1);
1129 
1130 	if (sg_count == 1) {
1131 		to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1);
1132 		return sg_count;
1133 	}
1134 	if (is_sec1) {
1135 		to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1);
1136 		return sg_count;
1137 	}
1138 	sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1139 					 &edesc->link_tbl[tbl_off]);
1140 	if (sg_count == 1) {
1141 		/* Only one segment now, so no link tbl needed*/
1142 		copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1143 		return sg_count;
1144 	}
1145 	to_talitos_ptr(ptr, edesc->dma_link_tbl +
1146 			    tbl_off * sizeof(struct talitos_ptr), is_sec1);
1147 	to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1148 
1149 	return sg_count;
1150 }
1151 
1152 /*
1153  * fill in and submit ipsec_esp descriptor
1154  */
1155 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1156 		     void (*callback)(struct device *dev,
1157 				      struct talitos_desc *desc,
1158 				      void *context, int error))
1159 {
1160 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1161 	unsigned int authsize = crypto_aead_authsize(aead);
1162 	struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1163 	struct device *dev = ctx->dev;
1164 	struct talitos_desc *desc = &edesc->desc;
1165 	unsigned int cryptlen = areq->cryptlen;
1166 	unsigned int ivsize = crypto_aead_ivsize(aead);
1167 	int tbl_off = 0;
1168 	int sg_count, ret;
1169 	int sg_link_tbl_len;
1170 	bool sync_needed = false;
1171 	struct talitos_private *priv = dev_get_drvdata(dev);
1172 	bool is_sec1 = has_ftr_sec1(priv);
1173 
1174 	/* hmac key */
1175 	map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1176 			       DMA_TO_DEVICE);
1177 
1178 	sg_count = edesc->src_nents ?: 1;
1179 	if (is_sec1 && sg_count > 1)
1180 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1181 				  areq->assoclen + cryptlen);
1182 	else
1183 		sg_count = dma_map_sg(dev, areq->src, sg_count,
1184 				      (areq->src == areq->dst) ?
1185 				      DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1186 
1187 	/* hmac data */
1188 	ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1189 			     &desc->ptr[1], sg_count, 0, tbl_off);
1190 
1191 	if (ret > 1) {
1192 		tbl_off += ret;
1193 		sync_needed = true;
1194 	}
1195 
1196 	/* cipher iv */
1197 	if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1198 		to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1);
1199 		to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1);
1200 		to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1);
1201 	} else {
1202 		to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1);
1203 		to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1);
1204 		to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1);
1205 	}
1206 
1207 	/* cipher key */
1208 	if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1209 		map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1210 				       (char *)&ctx->key + ctx->authkeylen,
1211 				       DMA_TO_DEVICE);
1212 	else
1213 		map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen,
1214 				       (char *)&ctx->key + ctx->authkeylen,
1215 				       DMA_TO_DEVICE);
1216 
1217 	/*
1218 	 * cipher in
1219 	 * map and adjust cipher len to aead request cryptlen.
1220 	 * extent is bytes of HMAC postpended to ciphertext,
1221 	 * typically 12 for ipsec
1222 	 */
1223 	to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1);
1224 	to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1);
1225 
1226 	sg_link_tbl_len = cryptlen;
1227 
1228 	if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1229 		to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1230 
1231 		if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1232 			sg_link_tbl_len += authsize;
1233 	}
1234 
1235 	sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1236 				  &desc->ptr[4], sg_count, areq->assoclen,
1237 				  tbl_off);
1238 
1239 	if (sg_count > 1) {
1240 		tbl_off += sg_count;
1241 		sync_needed = true;
1242 	}
1243 
1244 	/* cipher out */
1245 	if (areq->src != areq->dst) {
1246 		sg_count = edesc->dst_nents ? : 1;
1247 		if (!is_sec1 || sg_count == 1)
1248 			dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1249 	}
1250 
1251 	sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc,
1252 				  &desc->ptr[5], sg_count, areq->assoclen,
1253 				  tbl_off);
1254 
1255 	if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1256 		to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1257 
1258 	if (sg_count > 1) {
1259 		edesc->icv_ool = true;
1260 		sync_needed = true;
1261 
1262 		if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1263 			struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1264 			int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1265 				     sizeof(struct talitos_ptr) + authsize;
1266 
1267 			/* Add an entry to the link table for ICV data */
1268 			tbl_ptr += sg_count - 1;
1269 			to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1);
1270 			tbl_ptr++;
1271 			to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1272 					       is_sec1);
1273 			to_talitos_ptr_len(tbl_ptr, authsize, is_sec1);
1274 
1275 			/* icv data follows link tables */
1276 			to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
1277 				       is_sec1);
1278 		}
1279 	} else {
1280 		edesc->icv_ool = false;
1281 	}
1282 
1283 	/* ICV data */
1284 	if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
1285 		to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1);
1286 		to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl +
1287 			       areq->assoclen + cryptlen, is_sec1);
1288 	}
1289 
1290 	/* iv out */
1291 	if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1292 		map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1293 				       DMA_FROM_DEVICE);
1294 
1295 	if (sync_needed)
1296 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1297 					   edesc->dma_len,
1298 					   DMA_BIDIRECTIONAL);
1299 
1300 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1301 	if (ret != -EINPROGRESS) {
1302 		ipsec_esp_unmap(dev, edesc, areq);
1303 		kfree(edesc);
1304 	}
1305 	return ret;
1306 }
1307 
1308 /*
1309  * allocate and map the extended descriptor
1310  */
1311 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1312 						 struct scatterlist *src,
1313 						 struct scatterlist *dst,
1314 						 u8 *iv,
1315 						 unsigned int assoclen,
1316 						 unsigned int cryptlen,
1317 						 unsigned int authsize,
1318 						 unsigned int ivsize,
1319 						 int icv_stashing,
1320 						 u32 cryptoflags,
1321 						 bool encrypt)
1322 {
1323 	struct talitos_edesc *edesc;
1324 	int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1325 	dma_addr_t iv_dma = 0;
1326 	gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1327 		      GFP_ATOMIC;
1328 	struct talitos_private *priv = dev_get_drvdata(dev);
1329 	bool is_sec1 = has_ftr_sec1(priv);
1330 	int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1331 	void *err;
1332 
1333 	if (cryptlen + authsize > max_len) {
1334 		dev_err(dev, "length exceeds h/w max limit\n");
1335 		return ERR_PTR(-EINVAL);
1336 	}
1337 
1338 	if (ivsize)
1339 		iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1340 
1341 	if (!dst || dst == src) {
1342 		src_len = assoclen + cryptlen + authsize;
1343 		src_nents = sg_nents_for_len(src, src_len);
1344 		if (src_nents < 0) {
1345 			dev_err(dev, "Invalid number of src SG.\n");
1346 			err = ERR_PTR(-EINVAL);
1347 			goto error_sg;
1348 		}
1349 		src_nents = (src_nents == 1) ? 0 : src_nents;
1350 		dst_nents = dst ? src_nents : 0;
1351 		dst_len = 0;
1352 	} else { /* dst && dst != src*/
1353 		src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1354 		src_nents = sg_nents_for_len(src, src_len);
1355 		if (src_nents < 0) {
1356 			dev_err(dev, "Invalid number of src SG.\n");
1357 			err = ERR_PTR(-EINVAL);
1358 			goto error_sg;
1359 		}
1360 		src_nents = (src_nents == 1) ? 0 : src_nents;
1361 		dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1362 		dst_nents = sg_nents_for_len(dst, dst_len);
1363 		if (dst_nents < 0) {
1364 			dev_err(dev, "Invalid number of dst SG.\n");
1365 			err = ERR_PTR(-EINVAL);
1366 			goto error_sg;
1367 		}
1368 		dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1369 	}
1370 
1371 	/*
1372 	 * allocate space for base edesc plus the link tables,
1373 	 * allowing for two separate entries for AD and generated ICV (+ 2),
1374 	 * and space for two sets of ICVs (stashed and generated)
1375 	 */
1376 	alloc_len = sizeof(struct talitos_edesc);
1377 	if (src_nents || dst_nents) {
1378 		if (is_sec1)
1379 			dma_len = (src_nents ? src_len : 0) +
1380 				  (dst_nents ? dst_len : 0);
1381 		else
1382 			dma_len = (src_nents + dst_nents + 2) *
1383 				  sizeof(struct talitos_ptr) + authsize * 2;
1384 		alloc_len += dma_len;
1385 	} else {
1386 		dma_len = 0;
1387 		alloc_len += icv_stashing ? authsize : 0;
1388 	}
1389 
1390 	edesc = kmalloc(alloc_len, GFP_DMA | flags);
1391 	if (!edesc) {
1392 		dev_err(dev, "could not allocate edescriptor\n");
1393 		err = ERR_PTR(-ENOMEM);
1394 		goto error_sg;
1395 	}
1396 
1397 	edesc->src_nents = src_nents;
1398 	edesc->dst_nents = dst_nents;
1399 	edesc->iv_dma = iv_dma;
1400 	edesc->dma_len = dma_len;
1401 	if (dma_len)
1402 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1403 						     edesc->dma_len,
1404 						     DMA_BIDIRECTIONAL);
1405 
1406 	return edesc;
1407 error_sg:
1408 	if (iv_dma)
1409 		dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1410 	return err;
1411 }
1412 
1413 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1414 					      int icv_stashing, bool encrypt)
1415 {
1416 	struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1417 	unsigned int authsize = crypto_aead_authsize(authenc);
1418 	struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1419 	unsigned int ivsize = crypto_aead_ivsize(authenc);
1420 
1421 	return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1422 				   iv, areq->assoclen, areq->cryptlen,
1423 				   authsize, ivsize, icv_stashing,
1424 				   areq->base.flags, encrypt);
1425 }
1426 
1427 static int aead_encrypt(struct aead_request *req)
1428 {
1429 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1430 	struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1431 	struct talitos_edesc *edesc;
1432 
1433 	/* allocate extended descriptor */
1434 	edesc = aead_edesc_alloc(req, req->iv, 0, true);
1435 	if (IS_ERR(edesc))
1436 		return PTR_ERR(edesc);
1437 
1438 	/* set encrypt */
1439 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1440 
1441 	return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1442 }
1443 
1444 static int aead_decrypt(struct aead_request *req)
1445 {
1446 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1447 	unsigned int authsize = crypto_aead_authsize(authenc);
1448 	struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1449 	struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1450 	struct talitos_edesc *edesc;
1451 	struct scatterlist *sg;
1452 	void *icvdata;
1453 
1454 	req->cryptlen -= authsize;
1455 
1456 	/* allocate extended descriptor */
1457 	edesc = aead_edesc_alloc(req, req->iv, 1, false);
1458 	if (IS_ERR(edesc))
1459 		return PTR_ERR(edesc);
1460 
1461 	if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1462 	    ((!edesc->src_nents && !edesc->dst_nents) ||
1463 	     priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1464 
1465 		/* decrypt and check the ICV */
1466 		edesc->desc.hdr = ctx->desc_hdr_template |
1467 				  DESC_HDR_DIR_INBOUND |
1468 				  DESC_HDR_MODE1_MDEU_CICV;
1469 
1470 		/* reset integrity check result bits */
1471 		edesc->desc.hdr_lo = 0;
1472 
1473 		return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1474 	}
1475 
1476 	/* Have to check the ICV with software */
1477 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1478 
1479 	/* stash incoming ICV for later cmp with ICV generated by the h/w */
1480 	if (edesc->dma_len)
1481 		icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1482 						   edesc->dst_nents + 2];
1483 	else
1484 		icvdata = &edesc->link_tbl[0];
1485 
1486 	sg = sg_last(req->src, edesc->src_nents ? : 1);
1487 
1488 	memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1489 
1490 	return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1491 }
1492 
1493 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1494 			     const u8 *key, unsigned int keylen)
1495 {
1496 	struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1497 
1498 	memcpy(&ctx->key, key, keylen);
1499 	ctx->keylen = keylen;
1500 
1501 	return 0;
1502 }
1503 
1504 static void common_nonsnoop_unmap(struct device *dev,
1505 				  struct talitos_edesc *edesc,
1506 				  struct ablkcipher_request *areq)
1507 {
1508 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1509 
1510 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
1511 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1512 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1513 
1514 	if (edesc->dma_len)
1515 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1516 				 DMA_BIDIRECTIONAL);
1517 }
1518 
1519 static void ablkcipher_done(struct device *dev,
1520 			    struct talitos_desc *desc, void *context,
1521 			    int err)
1522 {
1523 	struct ablkcipher_request *areq = context;
1524 	struct talitos_edesc *edesc;
1525 
1526 	edesc = container_of(desc, struct talitos_edesc, desc);
1527 
1528 	common_nonsnoop_unmap(dev, edesc, areq);
1529 
1530 	kfree(edesc);
1531 
1532 	areq->base.complete(&areq->base, err);
1533 }
1534 
1535 static int common_nonsnoop(struct talitos_edesc *edesc,
1536 			   struct ablkcipher_request *areq,
1537 			   void (*callback) (struct device *dev,
1538 					     struct talitos_desc *desc,
1539 					     void *context, int error))
1540 {
1541 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1542 	struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1543 	struct device *dev = ctx->dev;
1544 	struct talitos_desc *desc = &edesc->desc;
1545 	unsigned int cryptlen = areq->nbytes;
1546 	unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1547 	int sg_count, ret;
1548 	bool sync_needed = false;
1549 	struct talitos_private *priv = dev_get_drvdata(dev);
1550 	bool is_sec1 = has_ftr_sec1(priv);
1551 
1552 	/* first DWORD empty */
1553 	desc->ptr[0] = zero_entry;
1554 
1555 	/* cipher iv */
1556 	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1557 	to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1558 	to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1);
1559 
1560 	/* cipher key */
1561 	map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1562 			       (char *)&ctx->key, DMA_TO_DEVICE);
1563 
1564 	sg_count = edesc->src_nents ?: 1;
1565 	if (is_sec1 && sg_count > 1)
1566 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1567 				  cryptlen);
1568 	else
1569 		sg_count = dma_map_sg(dev, areq->src, sg_count,
1570 				      (areq->src == areq->dst) ?
1571 				      DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1572 	/*
1573 	 * cipher in
1574 	 */
1575 	sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1576 				  &desc->ptr[3], sg_count, 0, 0);
1577 	if (sg_count > 1)
1578 		sync_needed = true;
1579 
1580 	/* cipher out */
1581 	if (areq->src != areq->dst) {
1582 		sg_count = edesc->dst_nents ? : 1;
1583 		if (!is_sec1 || sg_count == 1)
1584 			dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1585 	}
1586 
1587 	ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1588 			     sg_count, 0, (edesc->src_nents + 1));
1589 	if (ret > 1)
1590 		sync_needed = true;
1591 
1592 	/* iv out */
1593 	map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1594 			       DMA_FROM_DEVICE);
1595 
1596 	/* last DWORD empty */
1597 	desc->ptr[6] = zero_entry;
1598 
1599 	if (sync_needed)
1600 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1601 					   edesc->dma_len, DMA_BIDIRECTIONAL);
1602 
1603 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1604 	if (ret != -EINPROGRESS) {
1605 		common_nonsnoop_unmap(dev, edesc, areq);
1606 		kfree(edesc);
1607 	}
1608 	return ret;
1609 }
1610 
1611 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1612 						    areq, bool encrypt)
1613 {
1614 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1615 	struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1616 	unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1617 
1618 	return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1619 				   areq->info, 0, areq->nbytes, 0, ivsize, 0,
1620 				   areq->base.flags, encrypt);
1621 }
1622 
1623 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1624 {
1625 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1626 	struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1627 	struct talitos_edesc *edesc;
1628 
1629 	/* allocate extended descriptor */
1630 	edesc = ablkcipher_edesc_alloc(areq, true);
1631 	if (IS_ERR(edesc))
1632 		return PTR_ERR(edesc);
1633 
1634 	/* set encrypt */
1635 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1636 
1637 	return common_nonsnoop(edesc, areq, ablkcipher_done);
1638 }
1639 
1640 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1641 {
1642 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1643 	struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1644 	struct talitos_edesc *edesc;
1645 
1646 	/* allocate extended descriptor */
1647 	edesc = ablkcipher_edesc_alloc(areq, false);
1648 	if (IS_ERR(edesc))
1649 		return PTR_ERR(edesc);
1650 
1651 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1652 
1653 	return common_nonsnoop(edesc, areq, ablkcipher_done);
1654 }
1655 
1656 static void common_nonsnoop_hash_unmap(struct device *dev,
1657 				       struct talitos_edesc *edesc,
1658 				       struct ahash_request *areq)
1659 {
1660 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1661 	struct talitos_private *priv = dev_get_drvdata(dev);
1662 	bool is_sec1 = has_ftr_sec1(priv);
1663 
1664 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1665 
1666 	talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1667 
1668 	/* When using hashctx-in, must unmap it. */
1669 	if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1670 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1671 					 DMA_TO_DEVICE);
1672 
1673 	if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1674 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1675 					 DMA_TO_DEVICE);
1676 
1677 	if (edesc->dma_len)
1678 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1679 				 DMA_BIDIRECTIONAL);
1680 
1681 }
1682 
1683 static void ahash_done(struct device *dev,
1684 		       struct talitos_desc *desc, void *context,
1685 		       int err)
1686 {
1687 	struct ahash_request *areq = context;
1688 	struct talitos_edesc *edesc =
1689 		 container_of(desc, struct talitos_edesc, desc);
1690 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1691 
1692 	if (!req_ctx->last && req_ctx->to_hash_later) {
1693 		/* Position any partial block for next update/final/finup */
1694 		memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1695 		req_ctx->nbuf = req_ctx->to_hash_later;
1696 	}
1697 	common_nonsnoop_hash_unmap(dev, edesc, areq);
1698 
1699 	kfree(edesc);
1700 
1701 	areq->base.complete(&areq->base, err);
1702 }
1703 
1704 /*
1705  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1706  * ourself and submit a padded block
1707  */
1708 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1709 			       struct talitos_edesc *edesc,
1710 			       struct talitos_ptr *ptr)
1711 {
1712 	static u8 padded_hash[64] = {
1713 		0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1714 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1715 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1716 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1717 	};
1718 
1719 	pr_err_once("Bug in SEC1, padding ourself\n");
1720 	edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1721 	map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1722 			       (char *)padded_hash, DMA_TO_DEVICE);
1723 }
1724 
1725 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1726 				struct ahash_request *areq, unsigned int length,
1727 				void (*callback) (struct device *dev,
1728 						  struct talitos_desc *desc,
1729 						  void *context, int error))
1730 {
1731 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1732 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1733 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1734 	struct device *dev = ctx->dev;
1735 	struct talitos_desc *desc = &edesc->desc;
1736 	int ret;
1737 	bool sync_needed = false;
1738 	struct talitos_private *priv = dev_get_drvdata(dev);
1739 	bool is_sec1 = has_ftr_sec1(priv);
1740 	int sg_count;
1741 
1742 	/* first DWORD empty */
1743 	desc->ptr[0] = zero_entry;
1744 
1745 	/* hash context in */
1746 	if (!req_ctx->first || req_ctx->swinit) {
1747 		map_single_talitos_ptr(dev, &desc->ptr[1],
1748 				       req_ctx->hw_context_size,
1749 				       (char *)req_ctx->hw_context,
1750 				       DMA_TO_DEVICE);
1751 		req_ctx->swinit = 0;
1752 	} else {
1753 		desc->ptr[1] = zero_entry;
1754 		/* Indicate next op is not the first. */
1755 		req_ctx->first = 0;
1756 	}
1757 
1758 	/* HMAC key */
1759 	if (ctx->keylen)
1760 		map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1761 				       (char *)&ctx->key, DMA_TO_DEVICE);
1762 	else
1763 		desc->ptr[2] = zero_entry;
1764 
1765 	sg_count = edesc->src_nents ?: 1;
1766 	if (is_sec1 && sg_count > 1)
1767 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf, length);
1768 	else
1769 		sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1770 				      DMA_TO_DEVICE);
1771 	/*
1772 	 * data in
1773 	 */
1774 	sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1775 				  &desc->ptr[3], sg_count, 0, 0);
1776 	if (sg_count > 1)
1777 		sync_needed = true;
1778 
1779 	/* fifth DWORD empty */
1780 	desc->ptr[4] = zero_entry;
1781 
1782 	/* hash/HMAC out -or- hash context out */
1783 	if (req_ctx->last)
1784 		map_single_talitos_ptr(dev, &desc->ptr[5],
1785 				       crypto_ahash_digestsize(tfm),
1786 				       areq->result, DMA_FROM_DEVICE);
1787 	else
1788 		map_single_talitos_ptr(dev, &desc->ptr[5],
1789 				       req_ctx->hw_context_size,
1790 				       req_ctx->hw_context, DMA_FROM_DEVICE);
1791 
1792 	/* last DWORD empty */
1793 	desc->ptr[6] = zero_entry;
1794 
1795 	if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1796 		talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1797 
1798 	if (sync_needed)
1799 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1800 					   edesc->dma_len, DMA_BIDIRECTIONAL);
1801 
1802 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1803 	if (ret != -EINPROGRESS) {
1804 		common_nonsnoop_hash_unmap(dev, edesc, areq);
1805 		kfree(edesc);
1806 	}
1807 	return ret;
1808 }
1809 
1810 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1811 					       unsigned int nbytes)
1812 {
1813 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1814 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1815 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1816 
1817 	return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1818 				   nbytes, 0, 0, 0, areq->base.flags, false);
1819 }
1820 
1821 static int ahash_init(struct ahash_request *areq)
1822 {
1823 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1824 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1825 
1826 	/* Initialize the context */
1827 	req_ctx->nbuf = 0;
1828 	req_ctx->first = 1; /* first indicates h/w must init its context */
1829 	req_ctx->swinit = 0; /* assume h/w init of context */
1830 	req_ctx->hw_context_size =
1831 		(crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1832 			? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1833 			: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1834 
1835 	return 0;
1836 }
1837 
1838 /*
1839  * on h/w without explicit sha224 support, we initialize h/w context
1840  * manually with sha224 constants, and tell it to run sha256.
1841  */
1842 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1843 {
1844 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1845 
1846 	ahash_init(areq);
1847 	req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1848 
1849 	req_ctx->hw_context[0] = SHA224_H0;
1850 	req_ctx->hw_context[1] = SHA224_H1;
1851 	req_ctx->hw_context[2] = SHA224_H2;
1852 	req_ctx->hw_context[3] = SHA224_H3;
1853 	req_ctx->hw_context[4] = SHA224_H4;
1854 	req_ctx->hw_context[5] = SHA224_H5;
1855 	req_ctx->hw_context[6] = SHA224_H6;
1856 	req_ctx->hw_context[7] = SHA224_H7;
1857 
1858 	/* init 64-bit count */
1859 	req_ctx->hw_context[8] = 0;
1860 	req_ctx->hw_context[9] = 0;
1861 
1862 	return 0;
1863 }
1864 
1865 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1866 {
1867 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1868 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1869 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1870 	struct talitos_edesc *edesc;
1871 	unsigned int blocksize =
1872 			crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1873 	unsigned int nbytes_to_hash;
1874 	unsigned int to_hash_later;
1875 	unsigned int nsg;
1876 	int nents;
1877 
1878 	if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1879 		/* Buffer up to one whole block */
1880 		nents = sg_nents_for_len(areq->src, nbytes);
1881 		if (nents < 0) {
1882 			dev_err(ctx->dev, "Invalid number of src SG.\n");
1883 			return nents;
1884 		}
1885 		sg_copy_to_buffer(areq->src, nents,
1886 				  req_ctx->buf + req_ctx->nbuf, nbytes);
1887 		req_ctx->nbuf += nbytes;
1888 		return 0;
1889 	}
1890 
1891 	/* At least (blocksize + 1) bytes are available to hash */
1892 	nbytes_to_hash = nbytes + req_ctx->nbuf;
1893 	to_hash_later = nbytes_to_hash & (blocksize - 1);
1894 
1895 	if (req_ctx->last)
1896 		to_hash_later = 0;
1897 	else if (to_hash_later)
1898 		/* There is a partial block. Hash the full block(s) now */
1899 		nbytes_to_hash -= to_hash_later;
1900 	else {
1901 		/* Keep one block buffered */
1902 		nbytes_to_hash -= blocksize;
1903 		to_hash_later = blocksize;
1904 	}
1905 
1906 	/* Chain in any previously buffered data */
1907 	if (req_ctx->nbuf) {
1908 		nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1909 		sg_init_table(req_ctx->bufsl, nsg);
1910 		sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1911 		if (nsg > 1)
1912 			sg_chain(req_ctx->bufsl, 2, areq->src);
1913 		req_ctx->psrc = req_ctx->bufsl;
1914 	} else
1915 		req_ctx->psrc = areq->src;
1916 
1917 	if (to_hash_later) {
1918 		nents = sg_nents_for_len(areq->src, nbytes);
1919 		if (nents < 0) {
1920 			dev_err(ctx->dev, "Invalid number of src SG.\n");
1921 			return nents;
1922 		}
1923 		sg_pcopy_to_buffer(areq->src, nents,
1924 				      req_ctx->bufnext,
1925 				      to_hash_later,
1926 				      nbytes - to_hash_later);
1927 	}
1928 	req_ctx->to_hash_later = to_hash_later;
1929 
1930 	/* Allocate extended descriptor */
1931 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1932 	if (IS_ERR(edesc))
1933 		return PTR_ERR(edesc);
1934 
1935 	edesc->desc.hdr = ctx->desc_hdr_template;
1936 
1937 	/* On last one, request SEC to pad; otherwise continue */
1938 	if (req_ctx->last)
1939 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1940 	else
1941 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1942 
1943 	/* request SEC to INIT hash. */
1944 	if (req_ctx->first && !req_ctx->swinit)
1945 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1946 
1947 	/* When the tfm context has a keylen, it's an HMAC.
1948 	 * A first or last (ie. not middle) descriptor must request HMAC.
1949 	 */
1950 	if (ctx->keylen && (req_ctx->first || req_ctx->last))
1951 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1952 
1953 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1954 				    ahash_done);
1955 }
1956 
1957 static int ahash_update(struct ahash_request *areq)
1958 {
1959 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1960 
1961 	req_ctx->last = 0;
1962 
1963 	return ahash_process_req(areq, areq->nbytes);
1964 }
1965 
1966 static int ahash_final(struct ahash_request *areq)
1967 {
1968 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1969 
1970 	req_ctx->last = 1;
1971 
1972 	return ahash_process_req(areq, 0);
1973 }
1974 
1975 static int ahash_finup(struct ahash_request *areq)
1976 {
1977 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1978 
1979 	req_ctx->last = 1;
1980 
1981 	return ahash_process_req(areq, areq->nbytes);
1982 }
1983 
1984 static int ahash_digest(struct ahash_request *areq)
1985 {
1986 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1987 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1988 
1989 	ahash->init(areq);
1990 	req_ctx->last = 1;
1991 
1992 	return ahash_process_req(areq, areq->nbytes);
1993 }
1994 
1995 static int ahash_export(struct ahash_request *areq, void *out)
1996 {
1997 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1998 	struct talitos_export_state *export = out;
1999 
2000 	memcpy(export->hw_context, req_ctx->hw_context,
2001 	       req_ctx->hw_context_size);
2002 	memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2003 	export->swinit = req_ctx->swinit;
2004 	export->first = req_ctx->first;
2005 	export->last = req_ctx->last;
2006 	export->to_hash_later = req_ctx->to_hash_later;
2007 	export->nbuf = req_ctx->nbuf;
2008 
2009 	return 0;
2010 }
2011 
2012 static int ahash_import(struct ahash_request *areq, const void *in)
2013 {
2014 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2015 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2016 	const struct talitos_export_state *export = in;
2017 
2018 	memset(req_ctx, 0, sizeof(*req_ctx));
2019 	req_ctx->hw_context_size =
2020 		(crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2021 			? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2022 			: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2023 	memcpy(req_ctx->hw_context, export->hw_context,
2024 	       req_ctx->hw_context_size);
2025 	memcpy(req_ctx->buf, export->buf, export->nbuf);
2026 	req_ctx->swinit = export->swinit;
2027 	req_ctx->first = export->first;
2028 	req_ctx->last = export->last;
2029 	req_ctx->to_hash_later = export->to_hash_later;
2030 	req_ctx->nbuf = export->nbuf;
2031 
2032 	return 0;
2033 }
2034 
2035 struct keyhash_result {
2036 	struct completion completion;
2037 	int err;
2038 };
2039 
2040 static void keyhash_complete(struct crypto_async_request *req, int err)
2041 {
2042 	struct keyhash_result *res = req->data;
2043 
2044 	if (err == -EINPROGRESS)
2045 		return;
2046 
2047 	res->err = err;
2048 	complete(&res->completion);
2049 }
2050 
2051 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2052 		   u8 *hash)
2053 {
2054 	struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2055 
2056 	struct scatterlist sg[1];
2057 	struct ahash_request *req;
2058 	struct keyhash_result hresult;
2059 	int ret;
2060 
2061 	init_completion(&hresult.completion);
2062 
2063 	req = ahash_request_alloc(tfm, GFP_KERNEL);
2064 	if (!req)
2065 		return -ENOMEM;
2066 
2067 	/* Keep tfm keylen == 0 during hash of the long key */
2068 	ctx->keylen = 0;
2069 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2070 				   keyhash_complete, &hresult);
2071 
2072 	sg_init_one(&sg[0], key, keylen);
2073 
2074 	ahash_request_set_crypt(req, sg, hash, keylen);
2075 	ret = crypto_ahash_digest(req);
2076 	switch (ret) {
2077 	case 0:
2078 		break;
2079 	case -EINPROGRESS:
2080 	case -EBUSY:
2081 		ret = wait_for_completion_interruptible(
2082 			&hresult.completion);
2083 		if (!ret)
2084 			ret = hresult.err;
2085 		break;
2086 	default:
2087 		break;
2088 	}
2089 	ahash_request_free(req);
2090 
2091 	return ret;
2092 }
2093 
2094 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2095 			unsigned int keylen)
2096 {
2097 	struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2098 	unsigned int blocksize =
2099 			crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2100 	unsigned int digestsize = crypto_ahash_digestsize(tfm);
2101 	unsigned int keysize = keylen;
2102 	u8 hash[SHA512_DIGEST_SIZE];
2103 	int ret;
2104 
2105 	if (keylen <= blocksize)
2106 		memcpy(ctx->key, key, keysize);
2107 	else {
2108 		/* Must get the hash of the long key */
2109 		ret = keyhash(tfm, key, keylen, hash);
2110 
2111 		if (ret) {
2112 			crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2113 			return -EINVAL;
2114 		}
2115 
2116 		keysize = digestsize;
2117 		memcpy(ctx->key, hash, digestsize);
2118 	}
2119 
2120 	ctx->keylen = keysize;
2121 
2122 	return 0;
2123 }
2124 
2125 
2126 struct talitos_alg_template {
2127 	u32 type;
2128 	u32 priority;
2129 	union {
2130 		struct crypto_alg crypto;
2131 		struct ahash_alg hash;
2132 		struct aead_alg aead;
2133 	} alg;
2134 	__be32 desc_hdr_template;
2135 };
2136 
2137 static struct talitos_alg_template driver_algs[] = {
2138 	/* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2139 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2140 		.alg.aead = {
2141 			.base = {
2142 				.cra_name = "authenc(hmac(sha1),cbc(aes))",
2143 				.cra_driver_name = "authenc-hmac-sha1-"
2144 						   "cbc-aes-talitos",
2145 				.cra_blocksize = AES_BLOCK_SIZE,
2146 				.cra_flags = CRYPTO_ALG_ASYNC,
2147 			},
2148 			.ivsize = AES_BLOCK_SIZE,
2149 			.maxauthsize = SHA1_DIGEST_SIZE,
2150 		},
2151 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2152 			             DESC_HDR_SEL0_AESU |
2153 		                     DESC_HDR_MODE0_AESU_CBC |
2154 		                     DESC_HDR_SEL1_MDEUA |
2155 		                     DESC_HDR_MODE1_MDEU_INIT |
2156 		                     DESC_HDR_MODE1_MDEU_PAD |
2157 		                     DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2158 	},
2159 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2160 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2161 		.alg.aead = {
2162 			.base = {
2163 				.cra_name = "authenc(hmac(sha1),cbc(aes))",
2164 				.cra_driver_name = "authenc-hmac-sha1-"
2165 						   "cbc-aes-talitos",
2166 				.cra_blocksize = AES_BLOCK_SIZE,
2167 				.cra_flags = CRYPTO_ALG_ASYNC,
2168 			},
2169 			.ivsize = AES_BLOCK_SIZE,
2170 			.maxauthsize = SHA1_DIGEST_SIZE,
2171 		},
2172 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2173 				     DESC_HDR_SEL0_AESU |
2174 				     DESC_HDR_MODE0_AESU_CBC |
2175 				     DESC_HDR_SEL1_MDEUA |
2176 				     DESC_HDR_MODE1_MDEU_INIT |
2177 				     DESC_HDR_MODE1_MDEU_PAD |
2178 				     DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2179 	},
2180 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2181 		.alg.aead = {
2182 			.base = {
2183 				.cra_name = "authenc(hmac(sha1),"
2184 					    "cbc(des3_ede))",
2185 				.cra_driver_name = "authenc-hmac-sha1-"
2186 						   "cbc-3des-talitos",
2187 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2188 				.cra_flags = CRYPTO_ALG_ASYNC,
2189 			},
2190 			.ivsize = DES3_EDE_BLOCK_SIZE,
2191 			.maxauthsize = SHA1_DIGEST_SIZE,
2192 		},
2193 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2194 			             DESC_HDR_SEL0_DEU |
2195 		                     DESC_HDR_MODE0_DEU_CBC |
2196 		                     DESC_HDR_MODE0_DEU_3DES |
2197 		                     DESC_HDR_SEL1_MDEUA |
2198 		                     DESC_HDR_MODE1_MDEU_INIT |
2199 		                     DESC_HDR_MODE1_MDEU_PAD |
2200 		                     DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2201 	},
2202 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2203 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2204 		.alg.aead = {
2205 			.base = {
2206 				.cra_name = "authenc(hmac(sha1),"
2207 					    "cbc(des3_ede))",
2208 				.cra_driver_name = "authenc-hmac-sha1-"
2209 						   "cbc-3des-talitos",
2210 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2211 				.cra_flags = CRYPTO_ALG_ASYNC,
2212 			},
2213 			.ivsize = DES3_EDE_BLOCK_SIZE,
2214 			.maxauthsize = SHA1_DIGEST_SIZE,
2215 		},
2216 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2217 				     DESC_HDR_SEL0_DEU |
2218 				     DESC_HDR_MODE0_DEU_CBC |
2219 				     DESC_HDR_MODE0_DEU_3DES |
2220 				     DESC_HDR_SEL1_MDEUA |
2221 				     DESC_HDR_MODE1_MDEU_INIT |
2222 				     DESC_HDR_MODE1_MDEU_PAD |
2223 				     DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2224 	},
2225 	{       .type = CRYPTO_ALG_TYPE_AEAD,
2226 		.alg.aead = {
2227 			.base = {
2228 				.cra_name = "authenc(hmac(sha224),cbc(aes))",
2229 				.cra_driver_name = "authenc-hmac-sha224-"
2230 						   "cbc-aes-talitos",
2231 				.cra_blocksize = AES_BLOCK_SIZE,
2232 				.cra_flags = CRYPTO_ALG_ASYNC,
2233 			},
2234 			.ivsize = AES_BLOCK_SIZE,
2235 			.maxauthsize = SHA224_DIGEST_SIZE,
2236 		},
2237 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2238 				     DESC_HDR_SEL0_AESU |
2239 				     DESC_HDR_MODE0_AESU_CBC |
2240 				     DESC_HDR_SEL1_MDEUA |
2241 				     DESC_HDR_MODE1_MDEU_INIT |
2242 				     DESC_HDR_MODE1_MDEU_PAD |
2243 				     DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2244 	},
2245 	{       .type = CRYPTO_ALG_TYPE_AEAD,
2246 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2247 		.alg.aead = {
2248 			.base = {
2249 				.cra_name = "authenc(hmac(sha224),cbc(aes))",
2250 				.cra_driver_name = "authenc-hmac-sha224-"
2251 						   "cbc-aes-talitos",
2252 				.cra_blocksize = AES_BLOCK_SIZE,
2253 				.cra_flags = CRYPTO_ALG_ASYNC,
2254 			},
2255 			.ivsize = AES_BLOCK_SIZE,
2256 			.maxauthsize = SHA224_DIGEST_SIZE,
2257 		},
2258 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2259 				     DESC_HDR_SEL0_AESU |
2260 				     DESC_HDR_MODE0_AESU_CBC |
2261 				     DESC_HDR_SEL1_MDEUA |
2262 				     DESC_HDR_MODE1_MDEU_INIT |
2263 				     DESC_HDR_MODE1_MDEU_PAD |
2264 				     DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2265 	},
2266 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2267 		.alg.aead = {
2268 			.base = {
2269 				.cra_name = "authenc(hmac(sha224),"
2270 					    "cbc(des3_ede))",
2271 				.cra_driver_name = "authenc-hmac-sha224-"
2272 						   "cbc-3des-talitos",
2273 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2274 				.cra_flags = CRYPTO_ALG_ASYNC,
2275 			},
2276 			.ivsize = DES3_EDE_BLOCK_SIZE,
2277 			.maxauthsize = SHA224_DIGEST_SIZE,
2278 		},
2279 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2280 			             DESC_HDR_SEL0_DEU |
2281 		                     DESC_HDR_MODE0_DEU_CBC |
2282 		                     DESC_HDR_MODE0_DEU_3DES |
2283 		                     DESC_HDR_SEL1_MDEUA |
2284 		                     DESC_HDR_MODE1_MDEU_INIT |
2285 		                     DESC_HDR_MODE1_MDEU_PAD |
2286 		                     DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2287 	},
2288 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2289 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2290 		.alg.aead = {
2291 			.base = {
2292 				.cra_name = "authenc(hmac(sha224),"
2293 					    "cbc(des3_ede))",
2294 				.cra_driver_name = "authenc-hmac-sha224-"
2295 						   "cbc-3des-talitos",
2296 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2297 				.cra_flags = CRYPTO_ALG_ASYNC,
2298 			},
2299 			.ivsize = DES3_EDE_BLOCK_SIZE,
2300 			.maxauthsize = SHA224_DIGEST_SIZE,
2301 		},
2302 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2303 				     DESC_HDR_SEL0_DEU |
2304 				     DESC_HDR_MODE0_DEU_CBC |
2305 				     DESC_HDR_MODE0_DEU_3DES |
2306 				     DESC_HDR_SEL1_MDEUA |
2307 				     DESC_HDR_MODE1_MDEU_INIT |
2308 				     DESC_HDR_MODE1_MDEU_PAD |
2309 				     DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2310 	},
2311 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2312 		.alg.aead = {
2313 			.base = {
2314 				.cra_name = "authenc(hmac(sha256),cbc(aes))",
2315 				.cra_driver_name = "authenc-hmac-sha256-"
2316 						   "cbc-aes-talitos",
2317 				.cra_blocksize = AES_BLOCK_SIZE,
2318 				.cra_flags = CRYPTO_ALG_ASYNC,
2319 			},
2320 			.ivsize = AES_BLOCK_SIZE,
2321 			.maxauthsize = SHA256_DIGEST_SIZE,
2322 		},
2323 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2324 			             DESC_HDR_SEL0_AESU |
2325 		                     DESC_HDR_MODE0_AESU_CBC |
2326 		                     DESC_HDR_SEL1_MDEUA |
2327 		                     DESC_HDR_MODE1_MDEU_INIT |
2328 		                     DESC_HDR_MODE1_MDEU_PAD |
2329 		                     DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2330 	},
2331 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2332 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2333 		.alg.aead = {
2334 			.base = {
2335 				.cra_name = "authenc(hmac(sha256),cbc(aes))",
2336 				.cra_driver_name = "authenc-hmac-sha256-"
2337 						   "cbc-aes-talitos",
2338 				.cra_blocksize = AES_BLOCK_SIZE,
2339 				.cra_flags = CRYPTO_ALG_ASYNC,
2340 			},
2341 			.ivsize = AES_BLOCK_SIZE,
2342 			.maxauthsize = SHA256_DIGEST_SIZE,
2343 		},
2344 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2345 				     DESC_HDR_SEL0_AESU |
2346 				     DESC_HDR_MODE0_AESU_CBC |
2347 				     DESC_HDR_SEL1_MDEUA |
2348 				     DESC_HDR_MODE1_MDEU_INIT |
2349 				     DESC_HDR_MODE1_MDEU_PAD |
2350 				     DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2351 	},
2352 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2353 		.alg.aead = {
2354 			.base = {
2355 				.cra_name = "authenc(hmac(sha256),"
2356 					    "cbc(des3_ede))",
2357 				.cra_driver_name = "authenc-hmac-sha256-"
2358 						   "cbc-3des-talitos",
2359 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2360 				.cra_flags = CRYPTO_ALG_ASYNC,
2361 			},
2362 			.ivsize = DES3_EDE_BLOCK_SIZE,
2363 			.maxauthsize = SHA256_DIGEST_SIZE,
2364 		},
2365 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2366 			             DESC_HDR_SEL0_DEU |
2367 		                     DESC_HDR_MODE0_DEU_CBC |
2368 		                     DESC_HDR_MODE0_DEU_3DES |
2369 		                     DESC_HDR_SEL1_MDEUA |
2370 		                     DESC_HDR_MODE1_MDEU_INIT |
2371 		                     DESC_HDR_MODE1_MDEU_PAD |
2372 		                     DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2373 	},
2374 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2375 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2376 		.alg.aead = {
2377 			.base = {
2378 				.cra_name = "authenc(hmac(sha256),"
2379 					    "cbc(des3_ede))",
2380 				.cra_driver_name = "authenc-hmac-sha256-"
2381 						   "cbc-3des-talitos",
2382 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2383 				.cra_flags = CRYPTO_ALG_ASYNC,
2384 			},
2385 			.ivsize = DES3_EDE_BLOCK_SIZE,
2386 			.maxauthsize = SHA256_DIGEST_SIZE,
2387 		},
2388 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2389 				     DESC_HDR_SEL0_DEU |
2390 				     DESC_HDR_MODE0_DEU_CBC |
2391 				     DESC_HDR_MODE0_DEU_3DES |
2392 				     DESC_HDR_SEL1_MDEUA |
2393 				     DESC_HDR_MODE1_MDEU_INIT |
2394 				     DESC_HDR_MODE1_MDEU_PAD |
2395 				     DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2396 	},
2397 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2398 		.alg.aead = {
2399 			.base = {
2400 				.cra_name = "authenc(hmac(sha384),cbc(aes))",
2401 				.cra_driver_name = "authenc-hmac-sha384-"
2402 						   "cbc-aes-talitos",
2403 				.cra_blocksize = AES_BLOCK_SIZE,
2404 				.cra_flags = CRYPTO_ALG_ASYNC,
2405 			},
2406 			.ivsize = AES_BLOCK_SIZE,
2407 			.maxauthsize = SHA384_DIGEST_SIZE,
2408 		},
2409 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2410 			             DESC_HDR_SEL0_AESU |
2411 		                     DESC_HDR_MODE0_AESU_CBC |
2412 		                     DESC_HDR_SEL1_MDEUB |
2413 		                     DESC_HDR_MODE1_MDEU_INIT |
2414 		                     DESC_HDR_MODE1_MDEU_PAD |
2415 		                     DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2416 	},
2417 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2418 		.alg.aead = {
2419 			.base = {
2420 				.cra_name = "authenc(hmac(sha384),"
2421 					    "cbc(des3_ede))",
2422 				.cra_driver_name = "authenc-hmac-sha384-"
2423 						   "cbc-3des-talitos",
2424 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2425 				.cra_flags = CRYPTO_ALG_ASYNC,
2426 			},
2427 			.ivsize = DES3_EDE_BLOCK_SIZE,
2428 			.maxauthsize = SHA384_DIGEST_SIZE,
2429 		},
2430 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2431 			             DESC_HDR_SEL0_DEU |
2432 		                     DESC_HDR_MODE0_DEU_CBC |
2433 		                     DESC_HDR_MODE0_DEU_3DES |
2434 		                     DESC_HDR_SEL1_MDEUB |
2435 		                     DESC_HDR_MODE1_MDEU_INIT |
2436 		                     DESC_HDR_MODE1_MDEU_PAD |
2437 		                     DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2438 	},
2439 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2440 		.alg.aead = {
2441 			.base = {
2442 				.cra_name = "authenc(hmac(sha512),cbc(aes))",
2443 				.cra_driver_name = "authenc-hmac-sha512-"
2444 						   "cbc-aes-talitos",
2445 				.cra_blocksize = AES_BLOCK_SIZE,
2446 				.cra_flags = CRYPTO_ALG_ASYNC,
2447 			},
2448 			.ivsize = AES_BLOCK_SIZE,
2449 			.maxauthsize = SHA512_DIGEST_SIZE,
2450 		},
2451 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2452 			             DESC_HDR_SEL0_AESU |
2453 		                     DESC_HDR_MODE0_AESU_CBC |
2454 		                     DESC_HDR_SEL1_MDEUB |
2455 		                     DESC_HDR_MODE1_MDEU_INIT |
2456 		                     DESC_HDR_MODE1_MDEU_PAD |
2457 		                     DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2458 	},
2459 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2460 		.alg.aead = {
2461 			.base = {
2462 				.cra_name = "authenc(hmac(sha512),"
2463 					    "cbc(des3_ede))",
2464 				.cra_driver_name = "authenc-hmac-sha512-"
2465 						   "cbc-3des-talitos",
2466 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2467 				.cra_flags = CRYPTO_ALG_ASYNC,
2468 			},
2469 			.ivsize = DES3_EDE_BLOCK_SIZE,
2470 			.maxauthsize = SHA512_DIGEST_SIZE,
2471 		},
2472 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2473 			             DESC_HDR_SEL0_DEU |
2474 		                     DESC_HDR_MODE0_DEU_CBC |
2475 		                     DESC_HDR_MODE0_DEU_3DES |
2476 		                     DESC_HDR_SEL1_MDEUB |
2477 		                     DESC_HDR_MODE1_MDEU_INIT |
2478 		                     DESC_HDR_MODE1_MDEU_PAD |
2479 		                     DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2480 	},
2481 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2482 		.alg.aead = {
2483 			.base = {
2484 				.cra_name = "authenc(hmac(md5),cbc(aes))",
2485 				.cra_driver_name = "authenc-hmac-md5-"
2486 						   "cbc-aes-talitos",
2487 				.cra_blocksize = AES_BLOCK_SIZE,
2488 				.cra_flags = CRYPTO_ALG_ASYNC,
2489 			},
2490 			.ivsize = AES_BLOCK_SIZE,
2491 			.maxauthsize = MD5_DIGEST_SIZE,
2492 		},
2493 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2494 			             DESC_HDR_SEL0_AESU |
2495 		                     DESC_HDR_MODE0_AESU_CBC |
2496 		                     DESC_HDR_SEL1_MDEUA |
2497 		                     DESC_HDR_MODE1_MDEU_INIT |
2498 		                     DESC_HDR_MODE1_MDEU_PAD |
2499 		                     DESC_HDR_MODE1_MDEU_MD5_HMAC,
2500 	},
2501 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2502 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2503 		.alg.aead = {
2504 			.base = {
2505 				.cra_name = "authenc(hmac(md5),cbc(aes))",
2506 				.cra_driver_name = "authenc-hmac-md5-"
2507 						   "cbc-aes-talitos",
2508 				.cra_blocksize = AES_BLOCK_SIZE,
2509 				.cra_flags = CRYPTO_ALG_ASYNC,
2510 			},
2511 			.ivsize = AES_BLOCK_SIZE,
2512 			.maxauthsize = MD5_DIGEST_SIZE,
2513 		},
2514 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2515 				     DESC_HDR_SEL0_AESU |
2516 				     DESC_HDR_MODE0_AESU_CBC |
2517 				     DESC_HDR_SEL1_MDEUA |
2518 				     DESC_HDR_MODE1_MDEU_INIT |
2519 				     DESC_HDR_MODE1_MDEU_PAD |
2520 				     DESC_HDR_MODE1_MDEU_MD5_HMAC,
2521 	},
2522 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2523 		.alg.aead = {
2524 			.base = {
2525 				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2526 				.cra_driver_name = "authenc-hmac-md5-"
2527 						   "cbc-3des-talitos",
2528 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2529 				.cra_flags = CRYPTO_ALG_ASYNC,
2530 			},
2531 			.ivsize = DES3_EDE_BLOCK_SIZE,
2532 			.maxauthsize = MD5_DIGEST_SIZE,
2533 		},
2534 		.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2535 			             DESC_HDR_SEL0_DEU |
2536 		                     DESC_HDR_MODE0_DEU_CBC |
2537 		                     DESC_HDR_MODE0_DEU_3DES |
2538 		                     DESC_HDR_SEL1_MDEUA |
2539 		                     DESC_HDR_MODE1_MDEU_INIT |
2540 		                     DESC_HDR_MODE1_MDEU_PAD |
2541 		                     DESC_HDR_MODE1_MDEU_MD5_HMAC,
2542 	},
2543 	{	.type = CRYPTO_ALG_TYPE_AEAD,
2544 		.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2545 		.alg.aead = {
2546 			.base = {
2547 				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2548 				.cra_driver_name = "authenc-hmac-md5-"
2549 						   "cbc-3des-talitos",
2550 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2551 				.cra_flags = CRYPTO_ALG_ASYNC,
2552 			},
2553 			.ivsize = DES3_EDE_BLOCK_SIZE,
2554 			.maxauthsize = MD5_DIGEST_SIZE,
2555 		},
2556 		.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2557 				     DESC_HDR_SEL0_DEU |
2558 				     DESC_HDR_MODE0_DEU_CBC |
2559 				     DESC_HDR_MODE0_DEU_3DES |
2560 				     DESC_HDR_SEL1_MDEUA |
2561 				     DESC_HDR_MODE1_MDEU_INIT |
2562 				     DESC_HDR_MODE1_MDEU_PAD |
2563 				     DESC_HDR_MODE1_MDEU_MD5_HMAC,
2564 	},
2565 	/* ABLKCIPHER algorithms. */
2566 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2567 		.alg.crypto = {
2568 			.cra_name = "ecb(aes)",
2569 			.cra_driver_name = "ecb-aes-talitos",
2570 			.cra_blocksize = AES_BLOCK_SIZE,
2571 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2572 				     CRYPTO_ALG_ASYNC,
2573 			.cra_ablkcipher = {
2574 				.min_keysize = AES_MIN_KEY_SIZE,
2575 				.max_keysize = AES_MAX_KEY_SIZE,
2576 				.ivsize = AES_BLOCK_SIZE,
2577 			}
2578 		},
2579 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2580 				     DESC_HDR_SEL0_AESU,
2581 	},
2582 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2583 		.alg.crypto = {
2584 			.cra_name = "cbc(aes)",
2585 			.cra_driver_name = "cbc-aes-talitos",
2586 			.cra_blocksize = AES_BLOCK_SIZE,
2587 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2588                                      CRYPTO_ALG_ASYNC,
2589 			.cra_ablkcipher = {
2590 				.min_keysize = AES_MIN_KEY_SIZE,
2591 				.max_keysize = AES_MAX_KEY_SIZE,
2592 				.ivsize = AES_BLOCK_SIZE,
2593 			}
2594 		},
2595 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2596 				     DESC_HDR_SEL0_AESU |
2597 				     DESC_HDR_MODE0_AESU_CBC,
2598 	},
2599 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2600 		.alg.crypto = {
2601 			.cra_name = "ctr(aes)",
2602 			.cra_driver_name = "ctr-aes-talitos",
2603 			.cra_blocksize = AES_BLOCK_SIZE,
2604 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2605 				     CRYPTO_ALG_ASYNC,
2606 			.cra_ablkcipher = {
2607 				.min_keysize = AES_MIN_KEY_SIZE,
2608 				.max_keysize = AES_MAX_KEY_SIZE,
2609 				.ivsize = AES_BLOCK_SIZE,
2610 			}
2611 		},
2612 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2613 				     DESC_HDR_SEL0_AESU |
2614 				     DESC_HDR_MODE0_AESU_CTR,
2615 	},
2616 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2617 		.alg.crypto = {
2618 			.cra_name = "ecb(des)",
2619 			.cra_driver_name = "ecb-des-talitos",
2620 			.cra_blocksize = DES_BLOCK_SIZE,
2621 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2622 				     CRYPTO_ALG_ASYNC,
2623 			.cra_ablkcipher = {
2624 				.min_keysize = DES_KEY_SIZE,
2625 				.max_keysize = DES_KEY_SIZE,
2626 				.ivsize = DES_BLOCK_SIZE,
2627 			}
2628 		},
2629 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2630 				     DESC_HDR_SEL0_DEU,
2631 	},
2632 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2633 		.alg.crypto = {
2634 			.cra_name = "cbc(des)",
2635 			.cra_driver_name = "cbc-des-talitos",
2636 			.cra_blocksize = DES_BLOCK_SIZE,
2637 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2638 				     CRYPTO_ALG_ASYNC,
2639 			.cra_ablkcipher = {
2640 				.min_keysize = DES_KEY_SIZE,
2641 				.max_keysize = DES_KEY_SIZE,
2642 				.ivsize = DES_BLOCK_SIZE,
2643 			}
2644 		},
2645 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2646 				     DESC_HDR_SEL0_DEU |
2647 				     DESC_HDR_MODE0_DEU_CBC,
2648 	},
2649 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2650 		.alg.crypto = {
2651 			.cra_name = "ecb(des3_ede)",
2652 			.cra_driver_name = "ecb-3des-talitos",
2653 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2654 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2655 				     CRYPTO_ALG_ASYNC,
2656 			.cra_ablkcipher = {
2657 				.min_keysize = DES3_EDE_KEY_SIZE,
2658 				.max_keysize = DES3_EDE_KEY_SIZE,
2659 				.ivsize = DES3_EDE_BLOCK_SIZE,
2660 			}
2661 		},
2662 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2663 				     DESC_HDR_SEL0_DEU |
2664 				     DESC_HDR_MODE0_DEU_3DES,
2665 	},
2666 	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2667 		.alg.crypto = {
2668 			.cra_name = "cbc(des3_ede)",
2669 			.cra_driver_name = "cbc-3des-talitos",
2670 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2671 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2672                                      CRYPTO_ALG_ASYNC,
2673 			.cra_ablkcipher = {
2674 				.min_keysize = DES3_EDE_KEY_SIZE,
2675 				.max_keysize = DES3_EDE_KEY_SIZE,
2676 				.ivsize = DES3_EDE_BLOCK_SIZE,
2677 			}
2678 		},
2679 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2680 			             DESC_HDR_SEL0_DEU |
2681 		                     DESC_HDR_MODE0_DEU_CBC |
2682 		                     DESC_HDR_MODE0_DEU_3DES,
2683 	},
2684 	/* AHASH algorithms. */
2685 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2686 		.alg.hash = {
2687 			.halg.digestsize = MD5_DIGEST_SIZE,
2688 			.halg.statesize = sizeof(struct talitos_export_state),
2689 			.halg.base = {
2690 				.cra_name = "md5",
2691 				.cra_driver_name = "md5-talitos",
2692 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2693 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2694 					     CRYPTO_ALG_ASYNC,
2695 			}
2696 		},
2697 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2698 				     DESC_HDR_SEL0_MDEUA |
2699 				     DESC_HDR_MODE0_MDEU_MD5,
2700 	},
2701 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2702 		.alg.hash = {
2703 			.halg.digestsize = SHA1_DIGEST_SIZE,
2704 			.halg.statesize = sizeof(struct talitos_export_state),
2705 			.halg.base = {
2706 				.cra_name = "sha1",
2707 				.cra_driver_name = "sha1-talitos",
2708 				.cra_blocksize = SHA1_BLOCK_SIZE,
2709 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2710 					     CRYPTO_ALG_ASYNC,
2711 			}
2712 		},
2713 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2714 				     DESC_HDR_SEL0_MDEUA |
2715 				     DESC_HDR_MODE0_MDEU_SHA1,
2716 	},
2717 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2718 		.alg.hash = {
2719 			.halg.digestsize = SHA224_DIGEST_SIZE,
2720 			.halg.statesize = sizeof(struct talitos_export_state),
2721 			.halg.base = {
2722 				.cra_name = "sha224",
2723 				.cra_driver_name = "sha224-talitos",
2724 				.cra_blocksize = SHA224_BLOCK_SIZE,
2725 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2726 					     CRYPTO_ALG_ASYNC,
2727 			}
2728 		},
2729 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2730 				     DESC_HDR_SEL0_MDEUA |
2731 				     DESC_HDR_MODE0_MDEU_SHA224,
2732 	},
2733 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2734 		.alg.hash = {
2735 			.halg.digestsize = SHA256_DIGEST_SIZE,
2736 			.halg.statesize = sizeof(struct talitos_export_state),
2737 			.halg.base = {
2738 				.cra_name = "sha256",
2739 				.cra_driver_name = "sha256-talitos",
2740 				.cra_blocksize = SHA256_BLOCK_SIZE,
2741 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2742 					     CRYPTO_ALG_ASYNC,
2743 			}
2744 		},
2745 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2746 				     DESC_HDR_SEL0_MDEUA |
2747 				     DESC_HDR_MODE0_MDEU_SHA256,
2748 	},
2749 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2750 		.alg.hash = {
2751 			.halg.digestsize = SHA384_DIGEST_SIZE,
2752 			.halg.statesize = sizeof(struct talitos_export_state),
2753 			.halg.base = {
2754 				.cra_name = "sha384",
2755 				.cra_driver_name = "sha384-talitos",
2756 				.cra_blocksize = SHA384_BLOCK_SIZE,
2757 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2758 					     CRYPTO_ALG_ASYNC,
2759 			}
2760 		},
2761 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2762 				     DESC_HDR_SEL0_MDEUB |
2763 				     DESC_HDR_MODE0_MDEUB_SHA384,
2764 	},
2765 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2766 		.alg.hash = {
2767 			.halg.digestsize = SHA512_DIGEST_SIZE,
2768 			.halg.statesize = sizeof(struct talitos_export_state),
2769 			.halg.base = {
2770 				.cra_name = "sha512",
2771 				.cra_driver_name = "sha512-talitos",
2772 				.cra_blocksize = SHA512_BLOCK_SIZE,
2773 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2774 					     CRYPTO_ALG_ASYNC,
2775 			}
2776 		},
2777 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2778 				     DESC_HDR_SEL0_MDEUB |
2779 				     DESC_HDR_MODE0_MDEUB_SHA512,
2780 	},
2781 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2782 		.alg.hash = {
2783 			.halg.digestsize = MD5_DIGEST_SIZE,
2784 			.halg.statesize = sizeof(struct talitos_export_state),
2785 			.halg.base = {
2786 				.cra_name = "hmac(md5)",
2787 				.cra_driver_name = "hmac-md5-talitos",
2788 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2789 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2790 					     CRYPTO_ALG_ASYNC,
2791 			}
2792 		},
2793 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2794 				     DESC_HDR_SEL0_MDEUA |
2795 				     DESC_HDR_MODE0_MDEU_MD5,
2796 	},
2797 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2798 		.alg.hash = {
2799 			.halg.digestsize = SHA1_DIGEST_SIZE,
2800 			.halg.statesize = sizeof(struct talitos_export_state),
2801 			.halg.base = {
2802 				.cra_name = "hmac(sha1)",
2803 				.cra_driver_name = "hmac-sha1-talitos",
2804 				.cra_blocksize = SHA1_BLOCK_SIZE,
2805 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2806 					     CRYPTO_ALG_ASYNC,
2807 			}
2808 		},
2809 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2810 				     DESC_HDR_SEL0_MDEUA |
2811 				     DESC_HDR_MODE0_MDEU_SHA1,
2812 	},
2813 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2814 		.alg.hash = {
2815 			.halg.digestsize = SHA224_DIGEST_SIZE,
2816 			.halg.statesize = sizeof(struct talitos_export_state),
2817 			.halg.base = {
2818 				.cra_name = "hmac(sha224)",
2819 				.cra_driver_name = "hmac-sha224-talitos",
2820 				.cra_blocksize = SHA224_BLOCK_SIZE,
2821 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2822 					     CRYPTO_ALG_ASYNC,
2823 			}
2824 		},
2825 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2826 				     DESC_HDR_SEL0_MDEUA |
2827 				     DESC_HDR_MODE0_MDEU_SHA224,
2828 	},
2829 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2830 		.alg.hash = {
2831 			.halg.digestsize = SHA256_DIGEST_SIZE,
2832 			.halg.statesize = sizeof(struct talitos_export_state),
2833 			.halg.base = {
2834 				.cra_name = "hmac(sha256)",
2835 				.cra_driver_name = "hmac-sha256-talitos",
2836 				.cra_blocksize = SHA256_BLOCK_SIZE,
2837 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2838 					     CRYPTO_ALG_ASYNC,
2839 			}
2840 		},
2841 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2842 				     DESC_HDR_SEL0_MDEUA |
2843 				     DESC_HDR_MODE0_MDEU_SHA256,
2844 	},
2845 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2846 		.alg.hash = {
2847 			.halg.digestsize = SHA384_DIGEST_SIZE,
2848 			.halg.statesize = sizeof(struct talitos_export_state),
2849 			.halg.base = {
2850 				.cra_name = "hmac(sha384)",
2851 				.cra_driver_name = "hmac-sha384-talitos",
2852 				.cra_blocksize = SHA384_BLOCK_SIZE,
2853 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2854 					     CRYPTO_ALG_ASYNC,
2855 			}
2856 		},
2857 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2858 				     DESC_HDR_SEL0_MDEUB |
2859 				     DESC_HDR_MODE0_MDEUB_SHA384,
2860 	},
2861 	{	.type = CRYPTO_ALG_TYPE_AHASH,
2862 		.alg.hash = {
2863 			.halg.digestsize = SHA512_DIGEST_SIZE,
2864 			.halg.statesize = sizeof(struct talitos_export_state),
2865 			.halg.base = {
2866 				.cra_name = "hmac(sha512)",
2867 				.cra_driver_name = "hmac-sha512-talitos",
2868 				.cra_blocksize = SHA512_BLOCK_SIZE,
2869 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
2870 					     CRYPTO_ALG_ASYNC,
2871 			}
2872 		},
2873 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2874 				     DESC_HDR_SEL0_MDEUB |
2875 				     DESC_HDR_MODE0_MDEUB_SHA512,
2876 	}
2877 };
2878 
2879 struct talitos_crypto_alg {
2880 	struct list_head entry;
2881 	struct device *dev;
2882 	struct talitos_alg_template algt;
2883 };
2884 
2885 static int talitos_init_common(struct talitos_ctx *ctx,
2886 			       struct talitos_crypto_alg *talitos_alg)
2887 {
2888 	struct talitos_private *priv;
2889 
2890 	/* update context with ptr to dev */
2891 	ctx->dev = talitos_alg->dev;
2892 
2893 	/* assign SEC channel to tfm in round-robin fashion */
2894 	priv = dev_get_drvdata(ctx->dev);
2895 	ctx->ch = atomic_inc_return(&priv->last_chan) &
2896 		  (priv->num_channels - 1);
2897 
2898 	/* copy descriptor header template value */
2899 	ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2900 
2901 	/* select done notification */
2902 	ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2903 
2904 	return 0;
2905 }
2906 
2907 static int talitos_cra_init(struct crypto_tfm *tfm)
2908 {
2909 	struct crypto_alg *alg = tfm->__crt_alg;
2910 	struct talitos_crypto_alg *talitos_alg;
2911 	struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2912 
2913 	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2914 		talitos_alg = container_of(__crypto_ahash_alg(alg),
2915 					   struct talitos_crypto_alg,
2916 					   algt.alg.hash);
2917 	else
2918 		talitos_alg = container_of(alg, struct talitos_crypto_alg,
2919 					   algt.alg.crypto);
2920 
2921 	return talitos_init_common(ctx, talitos_alg);
2922 }
2923 
2924 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2925 {
2926 	struct aead_alg *alg = crypto_aead_alg(tfm);
2927 	struct talitos_crypto_alg *talitos_alg;
2928 	struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2929 
2930 	talitos_alg = container_of(alg, struct talitos_crypto_alg,
2931 				   algt.alg.aead);
2932 
2933 	return talitos_init_common(ctx, talitos_alg);
2934 }
2935 
2936 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2937 {
2938 	struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2939 
2940 	talitos_cra_init(tfm);
2941 
2942 	ctx->keylen = 0;
2943 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2944 				 sizeof(struct talitos_ahash_req_ctx));
2945 
2946 	return 0;
2947 }
2948 
2949 /*
2950  * given the alg's descriptor header template, determine whether descriptor
2951  * type and primary/secondary execution units required match the hw
2952  * capabilities description provided in the device tree node.
2953  */
2954 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2955 {
2956 	struct talitos_private *priv = dev_get_drvdata(dev);
2957 	int ret;
2958 
2959 	ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2960 	      (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2961 
2962 	if (SECONDARY_EU(desc_hdr_template))
2963 		ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2964 		              & priv->exec_units);
2965 
2966 	return ret;
2967 }
2968 
2969 static int talitos_remove(struct platform_device *ofdev)
2970 {
2971 	struct device *dev = &ofdev->dev;
2972 	struct talitos_private *priv = dev_get_drvdata(dev);
2973 	struct talitos_crypto_alg *t_alg, *n;
2974 	int i;
2975 
2976 	list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2977 		switch (t_alg->algt.type) {
2978 		case CRYPTO_ALG_TYPE_ABLKCIPHER:
2979 			break;
2980 		case CRYPTO_ALG_TYPE_AEAD:
2981 			crypto_unregister_aead(&t_alg->algt.alg.aead);
2982 		case CRYPTO_ALG_TYPE_AHASH:
2983 			crypto_unregister_ahash(&t_alg->algt.alg.hash);
2984 			break;
2985 		}
2986 		list_del(&t_alg->entry);
2987 		kfree(t_alg);
2988 	}
2989 
2990 	if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2991 		talitos_unregister_rng(dev);
2992 
2993 	for (i = 0; priv->chan && i < priv->num_channels; i++)
2994 		kfree(priv->chan[i].fifo);
2995 
2996 	kfree(priv->chan);
2997 
2998 	for (i = 0; i < 2; i++)
2999 		if (priv->irq[i]) {
3000 			free_irq(priv->irq[i], dev);
3001 			irq_dispose_mapping(priv->irq[i]);
3002 		}
3003 
3004 	tasklet_kill(&priv->done_task[0]);
3005 	if (priv->irq[1])
3006 		tasklet_kill(&priv->done_task[1]);
3007 
3008 	iounmap(priv->reg);
3009 
3010 	kfree(priv);
3011 
3012 	return 0;
3013 }
3014 
3015 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3016 						    struct talitos_alg_template
3017 						           *template)
3018 {
3019 	struct talitos_private *priv = dev_get_drvdata(dev);
3020 	struct talitos_crypto_alg *t_alg;
3021 	struct crypto_alg *alg;
3022 
3023 	t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
3024 	if (!t_alg)
3025 		return ERR_PTR(-ENOMEM);
3026 
3027 	t_alg->algt = *template;
3028 
3029 	switch (t_alg->algt.type) {
3030 	case CRYPTO_ALG_TYPE_ABLKCIPHER:
3031 		alg = &t_alg->algt.alg.crypto;
3032 		alg->cra_init = talitos_cra_init;
3033 		alg->cra_type = &crypto_ablkcipher_type;
3034 		alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3035 		alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3036 		alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3037 		alg->cra_ablkcipher.geniv = "eseqiv";
3038 		break;
3039 	case CRYPTO_ALG_TYPE_AEAD:
3040 		alg = &t_alg->algt.alg.aead.base;
3041 		t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3042 		t_alg->algt.alg.aead.setkey = aead_setkey;
3043 		t_alg->algt.alg.aead.encrypt = aead_encrypt;
3044 		t_alg->algt.alg.aead.decrypt = aead_decrypt;
3045 		break;
3046 	case CRYPTO_ALG_TYPE_AHASH:
3047 		alg = &t_alg->algt.alg.hash.halg.base;
3048 		alg->cra_init = talitos_cra_init_ahash;
3049 		alg->cra_type = &crypto_ahash_type;
3050 		t_alg->algt.alg.hash.init = ahash_init;
3051 		t_alg->algt.alg.hash.update = ahash_update;
3052 		t_alg->algt.alg.hash.final = ahash_final;
3053 		t_alg->algt.alg.hash.finup = ahash_finup;
3054 		t_alg->algt.alg.hash.digest = ahash_digest;
3055 		t_alg->algt.alg.hash.setkey = ahash_setkey;
3056 		t_alg->algt.alg.hash.import = ahash_import;
3057 		t_alg->algt.alg.hash.export = ahash_export;
3058 
3059 		if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3060 		    !strncmp(alg->cra_name, "hmac", 4)) {
3061 			kfree(t_alg);
3062 			return ERR_PTR(-ENOTSUPP);
3063 		}
3064 		if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3065 		    (!strcmp(alg->cra_name, "sha224") ||
3066 		     !strcmp(alg->cra_name, "hmac(sha224)"))) {
3067 			t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3068 			t_alg->algt.desc_hdr_template =
3069 					DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3070 					DESC_HDR_SEL0_MDEUA |
3071 					DESC_HDR_MODE0_MDEU_SHA256;
3072 		}
3073 		break;
3074 	default:
3075 		dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3076 		kfree(t_alg);
3077 		return ERR_PTR(-EINVAL);
3078 	}
3079 
3080 	alg->cra_module = THIS_MODULE;
3081 	if (t_alg->algt.priority)
3082 		alg->cra_priority = t_alg->algt.priority;
3083 	else
3084 		alg->cra_priority = TALITOS_CRA_PRIORITY;
3085 	alg->cra_alignmask = 0;
3086 	alg->cra_ctxsize = sizeof(struct talitos_ctx);
3087 	alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3088 
3089 	t_alg->dev = dev;
3090 
3091 	return t_alg;
3092 }
3093 
3094 static int talitos_probe_irq(struct platform_device *ofdev)
3095 {
3096 	struct device *dev = &ofdev->dev;
3097 	struct device_node *np = ofdev->dev.of_node;
3098 	struct talitos_private *priv = dev_get_drvdata(dev);
3099 	int err;
3100 	bool is_sec1 = has_ftr_sec1(priv);
3101 
3102 	priv->irq[0] = irq_of_parse_and_map(np, 0);
3103 	if (!priv->irq[0]) {
3104 		dev_err(dev, "failed to map irq\n");
3105 		return -EINVAL;
3106 	}
3107 	if (is_sec1) {
3108 		err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3109 				  dev_driver_string(dev), dev);
3110 		goto primary_out;
3111 	}
3112 
3113 	priv->irq[1] = irq_of_parse_and_map(np, 1);
3114 
3115 	/* get the primary irq line */
3116 	if (!priv->irq[1]) {
3117 		err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3118 				  dev_driver_string(dev), dev);
3119 		goto primary_out;
3120 	}
3121 
3122 	err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3123 			  dev_driver_string(dev), dev);
3124 	if (err)
3125 		goto primary_out;
3126 
3127 	/* get the secondary irq line */
3128 	err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3129 			  dev_driver_string(dev), dev);
3130 	if (err) {
3131 		dev_err(dev, "failed to request secondary irq\n");
3132 		irq_dispose_mapping(priv->irq[1]);
3133 		priv->irq[1] = 0;
3134 	}
3135 
3136 	return err;
3137 
3138 primary_out:
3139 	if (err) {
3140 		dev_err(dev, "failed to request primary irq\n");
3141 		irq_dispose_mapping(priv->irq[0]);
3142 		priv->irq[0] = 0;
3143 	}
3144 
3145 	return err;
3146 }
3147 
3148 static int talitos_probe(struct platform_device *ofdev)
3149 {
3150 	struct device *dev = &ofdev->dev;
3151 	struct device_node *np = ofdev->dev.of_node;
3152 	struct talitos_private *priv;
3153 	const unsigned int *prop;
3154 	int i, err;
3155 	int stride;
3156 
3157 	priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
3158 	if (!priv)
3159 		return -ENOMEM;
3160 
3161 	INIT_LIST_HEAD(&priv->alg_list);
3162 
3163 	dev_set_drvdata(dev, priv);
3164 
3165 	priv->ofdev = ofdev;
3166 
3167 	spin_lock_init(&priv->reg_lock);
3168 
3169 	priv->reg = of_iomap(np, 0);
3170 	if (!priv->reg) {
3171 		dev_err(dev, "failed to of_iomap\n");
3172 		err = -ENOMEM;
3173 		goto err_out;
3174 	}
3175 
3176 	/* get SEC version capabilities from device tree */
3177 	prop = of_get_property(np, "fsl,num-channels", NULL);
3178 	if (prop)
3179 		priv->num_channels = *prop;
3180 
3181 	prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
3182 	if (prop)
3183 		priv->chfifo_len = *prop;
3184 
3185 	prop = of_get_property(np, "fsl,exec-units-mask", NULL);
3186 	if (prop)
3187 		priv->exec_units = *prop;
3188 
3189 	prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
3190 	if (prop)
3191 		priv->desc_types = *prop;
3192 
3193 	if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3194 	    !priv->exec_units || !priv->desc_types) {
3195 		dev_err(dev, "invalid property data in device tree node\n");
3196 		err = -EINVAL;
3197 		goto err_out;
3198 	}
3199 
3200 	if (of_device_is_compatible(np, "fsl,sec3.0"))
3201 		priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3202 
3203 	if (of_device_is_compatible(np, "fsl,sec2.1"))
3204 		priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3205 				  TALITOS_FTR_SHA224_HWINIT |
3206 				  TALITOS_FTR_HMAC_OK;
3207 
3208 	if (of_device_is_compatible(np, "fsl,sec1.0"))
3209 		priv->features |= TALITOS_FTR_SEC1;
3210 
3211 	if (of_device_is_compatible(np, "fsl,sec1.2")) {
3212 		priv->reg_deu = priv->reg + TALITOS12_DEU;
3213 		priv->reg_aesu = priv->reg + TALITOS12_AESU;
3214 		priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3215 		stride = TALITOS1_CH_STRIDE;
3216 	} else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3217 		priv->reg_deu = priv->reg + TALITOS10_DEU;
3218 		priv->reg_aesu = priv->reg + TALITOS10_AESU;
3219 		priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3220 		priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3221 		priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3222 		priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3223 		stride = TALITOS1_CH_STRIDE;
3224 	} else {
3225 		priv->reg_deu = priv->reg + TALITOS2_DEU;
3226 		priv->reg_aesu = priv->reg + TALITOS2_AESU;
3227 		priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3228 		priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3229 		priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3230 		priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3231 		priv->reg_keu = priv->reg + TALITOS2_KEU;
3232 		priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3233 		stride = TALITOS2_CH_STRIDE;
3234 	}
3235 
3236 	err = talitos_probe_irq(ofdev);
3237 	if (err)
3238 		goto err_out;
3239 
3240 	if (of_device_is_compatible(np, "fsl,sec1.0")) {
3241 		tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3242 			     (unsigned long)dev);
3243 	} else {
3244 		if (!priv->irq[1]) {
3245 			tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3246 				     (unsigned long)dev);
3247 		} else {
3248 			tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3249 				     (unsigned long)dev);
3250 			tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3251 				     (unsigned long)dev);
3252 		}
3253 	}
3254 
3255 	priv->chan = kzalloc(sizeof(struct talitos_channel) *
3256 			     priv->num_channels, GFP_KERNEL);
3257 	if (!priv->chan) {
3258 		dev_err(dev, "failed to allocate channel management space\n");
3259 		err = -ENOMEM;
3260 		goto err_out;
3261 	}
3262 
3263 	priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3264 
3265 	for (i = 0; i < priv->num_channels; i++) {
3266 		priv->chan[i].reg = priv->reg + stride * (i + 1);
3267 		if (!priv->irq[1] || !(i & 1))
3268 			priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3269 
3270 		spin_lock_init(&priv->chan[i].head_lock);
3271 		spin_lock_init(&priv->chan[i].tail_lock);
3272 
3273 		priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3274 					     priv->fifo_len, GFP_KERNEL);
3275 		if (!priv->chan[i].fifo) {
3276 			dev_err(dev, "failed to allocate request fifo %d\n", i);
3277 			err = -ENOMEM;
3278 			goto err_out;
3279 		}
3280 
3281 		atomic_set(&priv->chan[i].submit_count,
3282 			   -(priv->chfifo_len - 1));
3283 	}
3284 
3285 	dma_set_mask(dev, DMA_BIT_MASK(36));
3286 
3287 	/* reset and initialize the h/w */
3288 	err = init_device(dev);
3289 	if (err) {
3290 		dev_err(dev, "failed to initialize device\n");
3291 		goto err_out;
3292 	}
3293 
3294 	/* register the RNG, if available */
3295 	if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3296 		err = talitos_register_rng(dev);
3297 		if (err) {
3298 			dev_err(dev, "failed to register hwrng: %d\n", err);
3299 			goto err_out;
3300 		} else
3301 			dev_info(dev, "hwrng\n");
3302 	}
3303 
3304 	/* register crypto algorithms the device supports */
3305 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3306 		if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3307 			struct talitos_crypto_alg *t_alg;
3308 			struct crypto_alg *alg = NULL;
3309 
3310 			t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3311 			if (IS_ERR(t_alg)) {
3312 				err = PTR_ERR(t_alg);
3313 				if (err == -ENOTSUPP)
3314 					continue;
3315 				goto err_out;
3316 			}
3317 
3318 			switch (t_alg->algt.type) {
3319 			case CRYPTO_ALG_TYPE_ABLKCIPHER:
3320 				err = crypto_register_alg(
3321 						&t_alg->algt.alg.crypto);
3322 				alg = &t_alg->algt.alg.crypto;
3323 				break;
3324 
3325 			case CRYPTO_ALG_TYPE_AEAD:
3326 				err = crypto_register_aead(
3327 					&t_alg->algt.alg.aead);
3328 				alg = &t_alg->algt.alg.aead.base;
3329 				break;
3330 
3331 			case CRYPTO_ALG_TYPE_AHASH:
3332 				err = crypto_register_ahash(
3333 						&t_alg->algt.alg.hash);
3334 				alg = &t_alg->algt.alg.hash.halg.base;
3335 				break;
3336 			}
3337 			if (err) {
3338 				dev_err(dev, "%s alg registration failed\n",
3339 					alg->cra_driver_name);
3340 				kfree(t_alg);
3341 			} else
3342 				list_add_tail(&t_alg->entry, &priv->alg_list);
3343 		}
3344 	}
3345 	if (!list_empty(&priv->alg_list))
3346 		dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3347 			 (char *)of_get_property(np, "compatible", NULL));
3348 
3349 	return 0;
3350 
3351 err_out:
3352 	talitos_remove(ofdev);
3353 
3354 	return err;
3355 }
3356 
3357 static const struct of_device_id talitos_match[] = {
3358 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3359 	{
3360 		.compatible = "fsl,sec1.0",
3361 	},
3362 #endif
3363 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3364 	{
3365 		.compatible = "fsl,sec2.0",
3366 	},
3367 #endif
3368 	{},
3369 };
3370 MODULE_DEVICE_TABLE(of, talitos_match);
3371 
3372 static struct platform_driver talitos_driver = {
3373 	.driver = {
3374 		.name = "talitos",
3375 		.of_match_table = talitos_match,
3376 	},
3377 	.probe = talitos_probe,
3378 	.remove = talitos_remove,
3379 };
3380 
3381 module_platform_driver(talitos_driver);
3382 
3383 MODULE_LICENSE("GPL");
3384 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3385 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
3386