xref: /openbmc/linux/drivers/crypto/ccp/ccp-dev-v3.c (revision 83bff109)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * AMD Cryptographic Coprocessor (CCP) driver
4  *
5  * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
6  *
7  * Author: Tom Lendacky <thomas.lendacky@amd.com>
8  * Author: Gary R Hook <gary.hook@amd.com>
9  */
10 
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/kthread.h>
14 #include <linux/interrupt.h>
15 #include <linux/ccp.h>
16 
17 #include "ccp-dev.h"
18 
ccp_alloc_ksb(struct ccp_cmd_queue * cmd_q,unsigned int count)19 static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
20 {
21 	int start;
22 	struct ccp_device *ccp = cmd_q->ccp;
23 
24 	for (;;) {
25 		mutex_lock(&ccp->sb_mutex);
26 
27 		start = (u32)bitmap_find_next_zero_area(ccp->sb,
28 							ccp->sb_count,
29 							ccp->sb_start,
30 							count, 0);
31 		if (start <= ccp->sb_count) {
32 			bitmap_set(ccp->sb, start, count);
33 
34 			mutex_unlock(&ccp->sb_mutex);
35 			break;
36 		}
37 
38 		ccp->sb_avail = 0;
39 
40 		mutex_unlock(&ccp->sb_mutex);
41 
42 		/* Wait for KSB entries to become available */
43 		if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
44 			return 0;
45 	}
46 
47 	return KSB_START + start;
48 }
49 
ccp_free_ksb(struct ccp_cmd_queue * cmd_q,unsigned int start,unsigned int count)50 static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
51 			 unsigned int count)
52 {
53 	struct ccp_device *ccp = cmd_q->ccp;
54 
55 	if (!start)
56 		return;
57 
58 	mutex_lock(&ccp->sb_mutex);
59 
60 	bitmap_clear(ccp->sb, start - KSB_START, count);
61 
62 	ccp->sb_avail = 1;
63 
64 	mutex_unlock(&ccp->sb_mutex);
65 
66 	wake_up_interruptible_all(&ccp->sb_queue);
67 }
68 
ccp_get_free_slots(struct ccp_cmd_queue * cmd_q)69 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
70 {
71 	return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
72 }
73 
ccp_do_cmd(struct ccp_op * op,u32 * cr,unsigned int cr_count)74 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
75 {
76 	struct ccp_cmd_queue *cmd_q = op->cmd_q;
77 	struct ccp_device *ccp = cmd_q->ccp;
78 	void __iomem *cr_addr;
79 	u32 cr0, cmd;
80 	unsigned int i;
81 	int ret = 0;
82 
83 	/* We could read a status register to see how many free slots
84 	 * are actually available, but reading that register resets it
85 	 * and you could lose some error information.
86 	 */
87 	cmd_q->free_slots--;
88 
89 	cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
90 	      | (op->jobid << REQ0_JOBID_SHIFT)
91 	      | REQ0_WAIT_FOR_WRITE;
92 
93 	if (op->soc)
94 		cr0 |= REQ0_STOP_ON_COMPLETE
95 		       | REQ0_INT_ON_COMPLETE;
96 
97 	if (op->ioc || !cmd_q->free_slots)
98 		cr0 |= REQ0_INT_ON_COMPLETE;
99 
100 	/* Start at CMD_REQ1 */
101 	cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
102 
103 	mutex_lock(&ccp->req_mutex);
104 
105 	/* Write CMD_REQ1 through CMD_REQx first */
106 	for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
107 		iowrite32(*(cr + i), cr_addr);
108 
109 	/* Tell the CCP to start */
110 	wmb();
111 	iowrite32(cr0, ccp->io_regs + CMD_REQ0);
112 
113 	mutex_unlock(&ccp->req_mutex);
114 
115 	if (cr0 & REQ0_INT_ON_COMPLETE) {
116 		/* Wait for the job to complete */
117 		ret = wait_event_interruptible(cmd_q->int_queue,
118 					       cmd_q->int_rcvd);
119 		if (ret || cmd_q->cmd_error) {
120 			/* On error delete all related jobs from the queue */
121 			cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
122 			      | op->jobid;
123 			if (cmd_q->cmd_error)
124 				ccp_log_error(cmd_q->ccp,
125 					      cmd_q->cmd_error);
126 
127 			iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
128 
129 			if (!ret)
130 				ret = -EIO;
131 		} else if (op->soc) {
132 			/* Delete just head job from the queue on SoC */
133 			cmd = DEL_Q_ACTIVE
134 			      | (cmd_q->id << DEL_Q_ID_SHIFT)
135 			      | op->jobid;
136 
137 			iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
138 		}
139 
140 		cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
141 
142 		cmd_q->int_rcvd = 0;
143 	}
144 
145 	return ret;
146 }
147 
ccp_perform_aes(struct ccp_op * op)148 static int ccp_perform_aes(struct ccp_op *op)
149 {
150 	u32 cr[6];
151 
152 	/* Fill out the register contents for REQ1 through REQ6 */
153 	cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
154 		| (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
155 		| (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
156 		| (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
157 		| (op->sb_key << REQ1_KEY_KSB_SHIFT);
158 	cr[1] = op->src.u.dma.length - 1;
159 	cr[2] = ccp_addr_lo(&op->src.u.dma);
160 	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
161 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
162 		| ccp_addr_hi(&op->src.u.dma);
163 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
164 	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
165 		| ccp_addr_hi(&op->dst.u.dma);
166 
167 	if (op->u.aes.mode == CCP_AES_MODE_CFB)
168 		cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
169 
170 	if (op->eom)
171 		cr[0] |= REQ1_EOM;
172 
173 	if (op->init)
174 		cr[0] |= REQ1_INIT;
175 
176 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
177 }
178 
ccp_perform_xts_aes(struct ccp_op * op)179 static int ccp_perform_xts_aes(struct ccp_op *op)
180 {
181 	u32 cr[6];
182 
183 	/* Fill out the register contents for REQ1 through REQ6 */
184 	cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
185 		| (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
186 		| (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
187 		| (op->sb_key << REQ1_KEY_KSB_SHIFT);
188 	cr[1] = op->src.u.dma.length - 1;
189 	cr[2] = ccp_addr_lo(&op->src.u.dma);
190 	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
191 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
192 		| ccp_addr_hi(&op->src.u.dma);
193 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
194 	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
195 		| ccp_addr_hi(&op->dst.u.dma);
196 
197 	if (op->eom)
198 		cr[0] |= REQ1_EOM;
199 
200 	if (op->init)
201 		cr[0] |= REQ1_INIT;
202 
203 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
204 }
205 
ccp_perform_sha(struct ccp_op * op)206 static int ccp_perform_sha(struct ccp_op *op)
207 {
208 	u32 cr[6];
209 
210 	/* Fill out the register contents for REQ1 through REQ6 */
211 	cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
212 		| (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
213 		| REQ1_INIT;
214 	cr[1] = op->src.u.dma.length - 1;
215 	cr[2] = ccp_addr_lo(&op->src.u.dma);
216 	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
217 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
218 		| ccp_addr_hi(&op->src.u.dma);
219 
220 	if (op->eom) {
221 		cr[0] |= REQ1_EOM;
222 		cr[4] = lower_32_bits(op->u.sha.msg_bits);
223 		cr[5] = upper_32_bits(op->u.sha.msg_bits);
224 	} else {
225 		cr[4] = 0;
226 		cr[5] = 0;
227 	}
228 
229 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
230 }
231 
ccp_perform_rsa(struct ccp_op * op)232 static int ccp_perform_rsa(struct ccp_op *op)
233 {
234 	u32 cr[6];
235 
236 	/* Fill out the register contents for REQ1 through REQ6 */
237 	cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
238 		| (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
239 		| (op->sb_key << REQ1_KEY_KSB_SHIFT)
240 		| REQ1_EOM;
241 	cr[1] = op->u.rsa.input_len - 1;
242 	cr[2] = ccp_addr_lo(&op->src.u.dma);
243 	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
244 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
245 		| ccp_addr_hi(&op->src.u.dma);
246 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
247 	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
248 		| ccp_addr_hi(&op->dst.u.dma);
249 
250 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
251 }
252 
ccp_perform_passthru(struct ccp_op * op)253 static int ccp_perform_passthru(struct ccp_op *op)
254 {
255 	u32 cr[6];
256 
257 	/* Fill out the register contents for REQ1 through REQ6 */
258 	cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
259 		| (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
260 		| (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
261 
262 	if (op->src.type == CCP_MEMTYPE_SYSTEM)
263 		cr[1] = op->src.u.dma.length - 1;
264 	else
265 		cr[1] = op->dst.u.dma.length - 1;
266 
267 	if (op->src.type == CCP_MEMTYPE_SYSTEM) {
268 		cr[2] = ccp_addr_lo(&op->src.u.dma);
269 		cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
270 			| ccp_addr_hi(&op->src.u.dma);
271 
272 		if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
273 			cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);
274 	} else {
275 		cr[2] = op->src.u.sb * CCP_SB_BYTES;
276 		cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);
277 	}
278 
279 	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
280 		cr[4] = ccp_addr_lo(&op->dst.u.dma);
281 		cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
282 			| ccp_addr_hi(&op->dst.u.dma);
283 	} else {
284 		cr[4] = op->dst.u.sb * CCP_SB_BYTES;
285 		cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
286 	}
287 
288 	if (op->eom)
289 		cr[0] |= REQ1_EOM;
290 
291 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
292 }
293 
ccp_perform_ecc(struct ccp_op * op)294 static int ccp_perform_ecc(struct ccp_op *op)
295 {
296 	u32 cr[6];
297 
298 	/* Fill out the register contents for REQ1 through REQ6 */
299 	cr[0] = REQ1_ECC_AFFINE_CONVERT
300 		| (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
301 		| (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
302 		| REQ1_EOM;
303 	cr[1] = op->src.u.dma.length - 1;
304 	cr[2] = ccp_addr_lo(&op->src.u.dma);
305 	cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
306 		| ccp_addr_hi(&op->src.u.dma);
307 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
308 	cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
309 		| ccp_addr_hi(&op->dst.u.dma);
310 
311 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
312 }
313 
ccp_disable_queue_interrupts(struct ccp_device * ccp)314 static void ccp_disable_queue_interrupts(struct ccp_device *ccp)
315 {
316 	iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
317 }
318 
ccp_enable_queue_interrupts(struct ccp_device * ccp)319 static void ccp_enable_queue_interrupts(struct ccp_device *ccp)
320 {
321 	iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG);
322 }
323 
ccp_irq_bh(unsigned long data)324 static void ccp_irq_bh(unsigned long data)
325 {
326 	struct ccp_device *ccp = (struct ccp_device *)data;
327 	struct ccp_cmd_queue *cmd_q;
328 	u32 q_int, status;
329 	unsigned int i;
330 
331 	status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
332 
333 	for (i = 0; i < ccp->cmd_q_count; i++) {
334 		cmd_q = &ccp->cmd_q[i];
335 
336 		q_int = status & (cmd_q->int_ok | cmd_q->int_err);
337 		if (q_int) {
338 			cmd_q->int_status = status;
339 			cmd_q->q_status = ioread32(cmd_q->reg_status);
340 			cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
341 
342 			/* On error, only save the first error value */
343 			if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
344 				cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
345 
346 			cmd_q->int_rcvd = 1;
347 
348 			/* Acknowledge the interrupt and wake the kthread */
349 			iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
350 			wake_up_interruptible(&cmd_q->int_queue);
351 		}
352 	}
353 	ccp_enable_queue_interrupts(ccp);
354 }
355 
ccp_irq_handler(int irq,void * data)356 static irqreturn_t ccp_irq_handler(int irq, void *data)
357 {
358 	struct ccp_device *ccp = (struct ccp_device *)data;
359 
360 	ccp_disable_queue_interrupts(ccp);
361 	if (ccp->use_tasklet)
362 		tasklet_schedule(&ccp->irq_tasklet);
363 	else
364 		ccp_irq_bh((unsigned long)ccp);
365 
366 	return IRQ_HANDLED;
367 }
368 
ccp_init(struct ccp_device * ccp)369 static int ccp_init(struct ccp_device *ccp)
370 {
371 	struct device *dev = ccp->dev;
372 	struct ccp_cmd_queue *cmd_q;
373 	struct dma_pool *dma_pool;
374 	char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
375 	unsigned int qmr, i;
376 	int ret;
377 
378 	/* Find available queues */
379 	ccp->qim = 0;
380 	qmr = ioread32(ccp->io_regs + Q_MASK_REG);
381 	for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
382 		if (!(qmr & (1 << i)))
383 			continue;
384 
385 		/* Allocate a dma pool for this queue */
386 		snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
387 			 ccp->name, i);
388 		dma_pool = dma_pool_create(dma_pool_name, dev,
389 					   CCP_DMAPOOL_MAX_SIZE,
390 					   CCP_DMAPOOL_ALIGN, 0);
391 		if (!dma_pool) {
392 			dev_err(dev, "unable to allocate dma pool\n");
393 			ret = -ENOMEM;
394 			goto e_pool;
395 		}
396 
397 		cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
398 		ccp->cmd_q_count++;
399 
400 		cmd_q->ccp = ccp;
401 		cmd_q->id = i;
402 		cmd_q->dma_pool = dma_pool;
403 
404 		/* Reserve 2 KSB regions for the queue */
405 		cmd_q->sb_key = KSB_START + ccp->sb_start++;
406 		cmd_q->sb_ctx = KSB_START + ccp->sb_start++;
407 		ccp->sb_count -= 2;
408 
409 		/* Preset some register values and masks that are queue
410 		 * number dependent
411 		 */
412 		cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
413 				    (CMD_Q_STATUS_INCR * i);
414 		cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
415 					(CMD_Q_STATUS_INCR * i);
416 		cmd_q->int_ok = 1 << (i * 2);
417 		cmd_q->int_err = 1 << ((i * 2) + 1);
418 
419 		cmd_q->free_slots = ccp_get_free_slots(cmd_q);
420 
421 		init_waitqueue_head(&cmd_q->int_queue);
422 
423 		/* Build queue interrupt mask (two interrupts per queue) */
424 		ccp->qim |= cmd_q->int_ok | cmd_q->int_err;
425 
426 #ifdef CONFIG_ARM64
427 		/* For arm64 set the recommended queue cache settings */
428 		iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
429 			  (CMD_Q_CACHE_INC * i));
430 #endif
431 
432 		dev_dbg(dev, "queue #%u available\n", i);
433 	}
434 	if (ccp->cmd_q_count == 0) {
435 		dev_notice(dev, "no command queues available\n");
436 		ret = -EIO;
437 		goto e_pool;
438 	}
439 	dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
440 
441 	/* Disable and clear interrupts until ready */
442 	ccp_disable_queue_interrupts(ccp);
443 	for (i = 0; i < ccp->cmd_q_count; i++) {
444 		cmd_q = &ccp->cmd_q[i];
445 
446 		ioread32(cmd_q->reg_int_status);
447 		ioread32(cmd_q->reg_status);
448 	}
449 	iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
450 
451 	/* Request an irq */
452 	ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
453 	if (ret) {
454 		dev_err(dev, "unable to allocate an IRQ\n");
455 		goto e_pool;
456 	}
457 
458 	/* Initialize the ISR tasklet? */
459 	if (ccp->use_tasklet)
460 		tasklet_init(&ccp->irq_tasklet, ccp_irq_bh,
461 			     (unsigned long)ccp);
462 
463 	dev_dbg(dev, "Starting threads...\n");
464 	/* Create a kthread for each queue */
465 	for (i = 0; i < ccp->cmd_q_count; i++) {
466 		struct task_struct *kthread;
467 
468 		cmd_q = &ccp->cmd_q[i];
469 
470 		kthread = kthread_run(ccp_cmd_queue_thread, cmd_q,
471 				      "%s-q%u", ccp->name, cmd_q->id);
472 		if (IS_ERR(kthread)) {
473 			dev_err(dev, "error creating queue thread (%ld)\n",
474 				PTR_ERR(kthread));
475 			ret = PTR_ERR(kthread);
476 			goto e_kthread;
477 		}
478 
479 		cmd_q->kthread = kthread;
480 	}
481 
482 	dev_dbg(dev, "Enabling interrupts...\n");
483 	/* Enable interrupts */
484 	ccp_enable_queue_interrupts(ccp);
485 
486 	dev_dbg(dev, "Registering device...\n");
487 	ccp_add_device(ccp);
488 
489 	ret = ccp_register_rng(ccp);
490 	if (ret)
491 		goto e_kthread;
492 
493 	/* Register the DMA engine support */
494 	ret = ccp_dmaengine_register(ccp);
495 	if (ret)
496 		goto e_hwrng;
497 
498 	return 0;
499 
500 e_hwrng:
501 	ccp_unregister_rng(ccp);
502 
503 e_kthread:
504 	for (i = 0; i < ccp->cmd_q_count; i++)
505 		if (ccp->cmd_q[i].kthread)
506 			kthread_stop(ccp->cmd_q[i].kthread);
507 
508 	sp_free_ccp_irq(ccp->sp, ccp);
509 
510 e_pool:
511 	for (i = 0; i < ccp->cmd_q_count; i++)
512 		dma_pool_destroy(ccp->cmd_q[i].dma_pool);
513 
514 	return ret;
515 }
516 
ccp_destroy(struct ccp_device * ccp)517 static void ccp_destroy(struct ccp_device *ccp)
518 {
519 	struct ccp_cmd_queue *cmd_q;
520 	struct ccp_cmd *cmd;
521 	unsigned int i;
522 
523 	/* Unregister the DMA engine */
524 	ccp_dmaengine_unregister(ccp);
525 
526 	/* Unregister the RNG */
527 	ccp_unregister_rng(ccp);
528 
529 	/* Remove this device from the list of available units */
530 	ccp_del_device(ccp);
531 
532 	/* Disable and clear interrupts */
533 	ccp_disable_queue_interrupts(ccp);
534 	for (i = 0; i < ccp->cmd_q_count; i++) {
535 		cmd_q = &ccp->cmd_q[i];
536 
537 		ioread32(cmd_q->reg_int_status);
538 		ioread32(cmd_q->reg_status);
539 	}
540 	iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
541 
542 	/* Stop the queue kthreads */
543 	for (i = 0; i < ccp->cmd_q_count; i++)
544 		if (ccp->cmd_q[i].kthread)
545 			kthread_stop(ccp->cmd_q[i].kthread);
546 
547 	sp_free_ccp_irq(ccp->sp, ccp);
548 
549 	for (i = 0; i < ccp->cmd_q_count; i++)
550 		dma_pool_destroy(ccp->cmd_q[i].dma_pool);
551 
552 	/* Flush the cmd and backlog queue */
553 	while (!list_empty(&ccp->cmd)) {
554 		/* Invoke the callback directly with an error code */
555 		cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
556 		list_del(&cmd->entry);
557 		cmd->callback(cmd->data, -ENODEV);
558 	}
559 	while (!list_empty(&ccp->backlog)) {
560 		/* Invoke the callback directly with an error code */
561 		cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
562 		list_del(&cmd->entry);
563 		cmd->callback(cmd->data, -ENODEV);
564 	}
565 }
566 
567 static const struct ccp_actions ccp3_actions = {
568 	.aes = ccp_perform_aes,
569 	.xts_aes = ccp_perform_xts_aes,
570 	.des3 = NULL,
571 	.sha = ccp_perform_sha,
572 	.rsa = ccp_perform_rsa,
573 	.passthru = ccp_perform_passthru,
574 	.ecc = ccp_perform_ecc,
575 	.sballoc = ccp_alloc_ksb,
576 	.sbfree = ccp_free_ksb,
577 	.init = ccp_init,
578 	.destroy = ccp_destroy,
579 	.get_free_slots = ccp_get_free_slots,
580 	.irqhandler = ccp_irq_handler,
581 };
582 
583 const struct ccp_vdata ccpv3_platform = {
584 	.version = CCP_VERSION(3, 0),
585 	.setup = NULL,
586 	.perform = &ccp3_actions,
587 	.offset = 0,
588 	.rsamax = CCP_RSA_MAX_WIDTH,
589 };
590 
591 const struct ccp_vdata ccpv3 = {
592 	.version = CCP_VERSION(3, 0),
593 	.setup = NULL,
594 	.perform = &ccp3_actions,
595 	.offset = 0x20000,
596 	.rsamax = CCP_RSA_MAX_WIDTH,
597 };
598