1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2ea0375afSGary R Hook /*
3ea0375afSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver
4ea0375afSGary R Hook *
568cc652fSGary R Hook * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
6ea0375afSGary R Hook *
7ea0375afSGary R Hook * Author: Tom Lendacky <thomas.lendacky@amd.com>
8fba8855cSGary R Hook * Author: Gary R Hook <gary.hook@amd.com>
9ea0375afSGary R Hook */
10ea0375afSGary R Hook
11ea0375afSGary R Hook #include <linux/module.h>
12ea0375afSGary R Hook #include <linux/kernel.h>
13ea0375afSGary R Hook #include <linux/kthread.h>
14ea0375afSGary R Hook #include <linux/interrupt.h>
15ea0375afSGary R Hook #include <linux/ccp.h>
16ea0375afSGary R Hook
17ea0375afSGary R Hook #include "ccp-dev.h"
18ea0375afSGary R Hook
ccp_alloc_ksb(struct ccp_cmd_queue * cmd_q,unsigned int count)1958a690b7SGary R Hook static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
2058a690b7SGary R Hook {
2158a690b7SGary R Hook int start;
2258a690b7SGary R Hook struct ccp_device *ccp = cmd_q->ccp;
2358a690b7SGary R Hook
2458a690b7SGary R Hook for (;;) {
2558a690b7SGary R Hook mutex_lock(&ccp->sb_mutex);
2658a690b7SGary R Hook
2758a690b7SGary R Hook start = (u32)bitmap_find_next_zero_area(ccp->sb,
2858a690b7SGary R Hook ccp->sb_count,
2958a690b7SGary R Hook ccp->sb_start,
3058a690b7SGary R Hook count, 0);
3158a690b7SGary R Hook if (start <= ccp->sb_count) {
3258a690b7SGary R Hook bitmap_set(ccp->sb, start, count);
3358a690b7SGary R Hook
3458a690b7SGary R Hook mutex_unlock(&ccp->sb_mutex);
3558a690b7SGary R Hook break;
3658a690b7SGary R Hook }
3758a690b7SGary R Hook
3858a690b7SGary R Hook ccp->sb_avail = 0;
3958a690b7SGary R Hook
4058a690b7SGary R Hook mutex_unlock(&ccp->sb_mutex);
4158a690b7SGary R Hook
4258a690b7SGary R Hook /* Wait for KSB entries to become available */
4358a690b7SGary R Hook if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
4458a690b7SGary R Hook return 0;
4558a690b7SGary R Hook }
4658a690b7SGary R Hook
4758a690b7SGary R Hook return KSB_START + start;
4858a690b7SGary R Hook }
4958a690b7SGary R Hook
ccp_free_ksb(struct ccp_cmd_queue * cmd_q,unsigned int start,unsigned int count)5058a690b7SGary R Hook static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
5158a690b7SGary R Hook unsigned int count)
5258a690b7SGary R Hook {
5358a690b7SGary R Hook struct ccp_device *ccp = cmd_q->ccp;
5458a690b7SGary R Hook
5558a690b7SGary R Hook if (!start)
5658a690b7SGary R Hook return;
5758a690b7SGary R Hook
5858a690b7SGary R Hook mutex_lock(&ccp->sb_mutex);
5958a690b7SGary R Hook
6058a690b7SGary R Hook bitmap_clear(ccp->sb, start - KSB_START, count);
6158a690b7SGary R Hook
6258a690b7SGary R Hook ccp->sb_avail = 1;
6358a690b7SGary R Hook
6458a690b7SGary R Hook mutex_unlock(&ccp->sb_mutex);
6558a690b7SGary R Hook
6658a690b7SGary R Hook wake_up_interruptible_all(&ccp->sb_queue);
6758a690b7SGary R Hook }
6858a690b7SGary R Hook
ccp_get_free_slots(struct ccp_cmd_queue * cmd_q)69bb4e89b3SGary R Hook static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
70bb4e89b3SGary R Hook {
71bb4e89b3SGary R Hook return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
72bb4e89b3SGary R Hook }
73bb4e89b3SGary R Hook
ccp_do_cmd(struct ccp_op * op,u32 * cr,unsigned int cr_count)74ea0375afSGary R Hook static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
75ea0375afSGary R Hook {
76ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q = op->cmd_q;
77ea0375afSGary R Hook struct ccp_device *ccp = cmd_q->ccp;
78ea0375afSGary R Hook void __iomem *cr_addr;
79ea0375afSGary R Hook u32 cr0, cmd;
80ea0375afSGary R Hook unsigned int i;
81ea0375afSGary R Hook int ret = 0;
82ea0375afSGary R Hook
83ea0375afSGary R Hook /* We could read a status register to see how many free slots
84ea0375afSGary R Hook * are actually available, but reading that register resets it
85ea0375afSGary R Hook * and you could lose some error information.
86ea0375afSGary R Hook */
87ea0375afSGary R Hook cmd_q->free_slots--;
88ea0375afSGary R Hook
89ea0375afSGary R Hook cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
90ea0375afSGary R Hook | (op->jobid << REQ0_JOBID_SHIFT)
91ea0375afSGary R Hook | REQ0_WAIT_FOR_WRITE;
92ea0375afSGary R Hook
93ea0375afSGary R Hook if (op->soc)
94ea0375afSGary R Hook cr0 |= REQ0_STOP_ON_COMPLETE
95ea0375afSGary R Hook | REQ0_INT_ON_COMPLETE;
96ea0375afSGary R Hook
97ea0375afSGary R Hook if (op->ioc || !cmd_q->free_slots)
98ea0375afSGary R Hook cr0 |= REQ0_INT_ON_COMPLETE;
99ea0375afSGary R Hook
100ea0375afSGary R Hook /* Start at CMD_REQ1 */
101ea0375afSGary R Hook cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
102ea0375afSGary R Hook
103ea0375afSGary R Hook mutex_lock(&ccp->req_mutex);
104ea0375afSGary R Hook
105ea0375afSGary R Hook /* Write CMD_REQ1 through CMD_REQx first */
106ea0375afSGary R Hook for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
107ea0375afSGary R Hook iowrite32(*(cr + i), cr_addr);
108ea0375afSGary R Hook
109ea0375afSGary R Hook /* Tell the CCP to start */
110ea0375afSGary R Hook wmb();
111ea0375afSGary R Hook iowrite32(cr0, ccp->io_regs + CMD_REQ0);
112ea0375afSGary R Hook
113ea0375afSGary R Hook mutex_unlock(&ccp->req_mutex);
114ea0375afSGary R Hook
115ea0375afSGary R Hook if (cr0 & REQ0_INT_ON_COMPLETE) {
116ea0375afSGary R Hook /* Wait for the job to complete */
117ea0375afSGary R Hook ret = wait_event_interruptible(cmd_q->int_queue,
118ea0375afSGary R Hook cmd_q->int_rcvd);
119ea0375afSGary R Hook if (ret || cmd_q->cmd_error) {
120ea0375afSGary R Hook /* On error delete all related jobs from the queue */
121ea0375afSGary R Hook cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
122ea0375afSGary R Hook | op->jobid;
12381422badSGary R Hook if (cmd_q->cmd_error)
12481422badSGary R Hook ccp_log_error(cmd_q->ccp,
12581422badSGary R Hook cmd_q->cmd_error);
126ea0375afSGary R Hook
127ea0375afSGary R Hook iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
128ea0375afSGary R Hook
129ea0375afSGary R Hook if (!ret)
130ea0375afSGary R Hook ret = -EIO;
131ea0375afSGary R Hook } else if (op->soc) {
132ea0375afSGary R Hook /* Delete just head job from the queue on SoC */
133ea0375afSGary R Hook cmd = DEL_Q_ACTIVE
134ea0375afSGary R Hook | (cmd_q->id << DEL_Q_ID_SHIFT)
135ea0375afSGary R Hook | op->jobid;
136ea0375afSGary R Hook
137ea0375afSGary R Hook iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
138ea0375afSGary R Hook }
139ea0375afSGary R Hook
140ea0375afSGary R Hook cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
141ea0375afSGary R Hook
142ea0375afSGary R Hook cmd_q->int_rcvd = 0;
143ea0375afSGary R Hook }
144ea0375afSGary R Hook
145ea0375afSGary R Hook return ret;
146ea0375afSGary R Hook }
147ea0375afSGary R Hook
ccp_perform_aes(struct ccp_op * op)148ea0375afSGary R Hook static int ccp_perform_aes(struct ccp_op *op)
149ea0375afSGary R Hook {
150ea0375afSGary R Hook u32 cr[6];
151ea0375afSGary R Hook
152ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */
153ea0375afSGary R Hook cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
154ea0375afSGary R Hook | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
155ea0375afSGary R Hook | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
156ea0375afSGary R Hook | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
157956ee21aSGary R Hook | (op->sb_key << REQ1_KEY_KSB_SHIFT);
158ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1;
159ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma);
160956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
161ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
162ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma);
163ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma);
164ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
165ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma);
166ea0375afSGary R Hook
167ea0375afSGary R Hook if (op->u.aes.mode == CCP_AES_MODE_CFB)
168ea0375afSGary R Hook cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
169ea0375afSGary R Hook
170ea0375afSGary R Hook if (op->eom)
171ea0375afSGary R Hook cr[0] |= REQ1_EOM;
172ea0375afSGary R Hook
173ea0375afSGary R Hook if (op->init)
174ea0375afSGary R Hook cr[0] |= REQ1_INIT;
175ea0375afSGary R Hook
176ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
177ea0375afSGary R Hook }
178ea0375afSGary R Hook
ccp_perform_xts_aes(struct ccp_op * op)179ea0375afSGary R Hook static int ccp_perform_xts_aes(struct ccp_op *op)
180ea0375afSGary R Hook {
181ea0375afSGary R Hook u32 cr[6];
182ea0375afSGary R Hook
183ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */
184ea0375afSGary R Hook cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
185ea0375afSGary R Hook | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
186ea0375afSGary R Hook | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
187956ee21aSGary R Hook | (op->sb_key << REQ1_KEY_KSB_SHIFT);
188ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1;
189ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma);
190956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
191ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
192ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma);
193ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma);
194ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
195ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma);
196ea0375afSGary R Hook
197ea0375afSGary R Hook if (op->eom)
198ea0375afSGary R Hook cr[0] |= REQ1_EOM;
199ea0375afSGary R Hook
200ea0375afSGary R Hook if (op->init)
201ea0375afSGary R Hook cr[0] |= REQ1_INIT;
202ea0375afSGary R Hook
203ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
204ea0375afSGary R Hook }
205ea0375afSGary R Hook
ccp_perform_sha(struct ccp_op * op)206ea0375afSGary R Hook static int ccp_perform_sha(struct ccp_op *op)
207ea0375afSGary R Hook {
208ea0375afSGary R Hook u32 cr[6];
209ea0375afSGary R Hook
210ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */
211ea0375afSGary R Hook cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
212ea0375afSGary R Hook | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
213ea0375afSGary R Hook | REQ1_INIT;
214ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1;
215ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma);
216956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
217ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
218ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma);
219ea0375afSGary R Hook
220ea0375afSGary R Hook if (op->eom) {
221ea0375afSGary R Hook cr[0] |= REQ1_EOM;
222ea0375afSGary R Hook cr[4] = lower_32_bits(op->u.sha.msg_bits);
223ea0375afSGary R Hook cr[5] = upper_32_bits(op->u.sha.msg_bits);
224ea0375afSGary R Hook } else {
225ea0375afSGary R Hook cr[4] = 0;
226ea0375afSGary R Hook cr[5] = 0;
227ea0375afSGary R Hook }
228ea0375afSGary R Hook
229ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
230ea0375afSGary R Hook }
231ea0375afSGary R Hook
ccp_perform_rsa(struct ccp_op * op)232ea0375afSGary R Hook static int ccp_perform_rsa(struct ccp_op *op)
233ea0375afSGary R Hook {
234ea0375afSGary R Hook u32 cr[6];
235ea0375afSGary R Hook
236ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */
237ea0375afSGary R Hook cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
238ea0375afSGary R Hook | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
239956ee21aSGary R Hook | (op->sb_key << REQ1_KEY_KSB_SHIFT)
240ea0375afSGary R Hook | REQ1_EOM;
241ea0375afSGary R Hook cr[1] = op->u.rsa.input_len - 1;
242ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma);
243956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
244ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
245ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma);
246ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma);
247ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
248ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma);
249ea0375afSGary R Hook
250ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
251ea0375afSGary R Hook }
252ea0375afSGary R Hook
ccp_perform_passthru(struct ccp_op * op)253ea0375afSGary R Hook static int ccp_perform_passthru(struct ccp_op *op)
254ea0375afSGary R Hook {
255ea0375afSGary R Hook u32 cr[6];
256ea0375afSGary R Hook
257ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */
258ea0375afSGary R Hook cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
259ea0375afSGary R Hook | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
260ea0375afSGary R Hook | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
261ea0375afSGary R Hook
262ea0375afSGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM)
263ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1;
264ea0375afSGary R Hook else
265ea0375afSGary R Hook cr[1] = op->dst.u.dma.length - 1;
266ea0375afSGary R Hook
267ea0375afSGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM) {
268ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma);
269ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
270ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma);
271ea0375afSGary R Hook
272ea0375afSGary R Hook if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
273956ee21aSGary R Hook cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);
274ea0375afSGary R Hook } else {
275956ee21aSGary R Hook cr[2] = op->src.u.sb * CCP_SB_BYTES;
276956ee21aSGary R Hook cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);
277ea0375afSGary R Hook }
278ea0375afSGary R Hook
279ea0375afSGary R Hook if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
280ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma);
281ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
282ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma);
283ea0375afSGary R Hook } else {
284956ee21aSGary R Hook cr[4] = op->dst.u.sb * CCP_SB_BYTES;
285956ee21aSGary R Hook cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
286ea0375afSGary R Hook }
287ea0375afSGary R Hook
288ea0375afSGary R Hook if (op->eom)
289ea0375afSGary R Hook cr[0] |= REQ1_EOM;
290ea0375afSGary R Hook
291ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
292ea0375afSGary R Hook }
293ea0375afSGary R Hook
ccp_perform_ecc(struct ccp_op * op)294ea0375afSGary R Hook static int ccp_perform_ecc(struct ccp_op *op)
295ea0375afSGary R Hook {
296ea0375afSGary R Hook u32 cr[6];
297ea0375afSGary R Hook
298ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */
299ea0375afSGary R Hook cr[0] = REQ1_ECC_AFFINE_CONVERT
300ea0375afSGary R Hook | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
301ea0375afSGary R Hook | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
302ea0375afSGary R Hook | REQ1_EOM;
303ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1;
304ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma);
305ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
306ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma);
307ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma);
308ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
309ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma);
310ea0375afSGary R Hook
311ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
312ea0375afSGary R Hook }
313ea0375afSGary R Hook
ccp_disable_queue_interrupts(struct ccp_device * ccp)3147b537b24SGary R Hook static void ccp_disable_queue_interrupts(struct ccp_device *ccp)
3157b537b24SGary R Hook {
3167b537b24SGary R Hook iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
3177b537b24SGary R Hook }
3187b537b24SGary R Hook
ccp_enable_queue_interrupts(struct ccp_device * ccp)3197b537b24SGary R Hook static void ccp_enable_queue_interrupts(struct ccp_device *ccp)
3207b537b24SGary R Hook {
3217b537b24SGary R Hook iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG);
3227b537b24SGary R Hook }
3237b537b24SGary R Hook
ccp_irq_bh(unsigned long data)3247b537b24SGary R Hook static void ccp_irq_bh(unsigned long data)
3257b537b24SGary R Hook {
3267b537b24SGary R Hook struct ccp_device *ccp = (struct ccp_device *)data;
3277b537b24SGary R Hook struct ccp_cmd_queue *cmd_q;
3287b537b24SGary R Hook u32 q_int, status;
3297b537b24SGary R Hook unsigned int i;
3307b537b24SGary R Hook
3317b537b24SGary R Hook status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
3327b537b24SGary R Hook
3337b537b24SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
3347b537b24SGary R Hook cmd_q = &ccp->cmd_q[i];
3357b537b24SGary R Hook
3367b537b24SGary R Hook q_int = status & (cmd_q->int_ok | cmd_q->int_err);
3377b537b24SGary R Hook if (q_int) {
3387b537b24SGary R Hook cmd_q->int_status = status;
3397b537b24SGary R Hook cmd_q->q_status = ioread32(cmd_q->reg_status);
3407b537b24SGary R Hook cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
3417b537b24SGary R Hook
3427b537b24SGary R Hook /* On error, only save the first error value */
3437b537b24SGary R Hook if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
3447b537b24SGary R Hook cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
3457b537b24SGary R Hook
3467b537b24SGary R Hook cmd_q->int_rcvd = 1;
3477b537b24SGary R Hook
3487b537b24SGary R Hook /* Acknowledge the interrupt and wake the kthread */
3497b537b24SGary R Hook iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
3507b537b24SGary R Hook wake_up_interruptible(&cmd_q->int_queue);
3517b537b24SGary R Hook }
3527b537b24SGary R Hook }
3537b537b24SGary R Hook ccp_enable_queue_interrupts(ccp);
3547b537b24SGary R Hook }
3557b537b24SGary R Hook
ccp_irq_handler(int irq,void * data)3567b537b24SGary R Hook static irqreturn_t ccp_irq_handler(int irq, void *data)
3577b537b24SGary R Hook {
358720419f0SBrijesh Singh struct ccp_device *ccp = (struct ccp_device *)data;
3597b537b24SGary R Hook
3607b537b24SGary R Hook ccp_disable_queue_interrupts(ccp);
3617b537b24SGary R Hook if (ccp->use_tasklet)
3627b537b24SGary R Hook tasklet_schedule(&ccp->irq_tasklet);
3637b537b24SGary R Hook else
3647b537b24SGary R Hook ccp_irq_bh((unsigned long)ccp);
3657b537b24SGary R Hook
3667b537b24SGary R Hook return IRQ_HANDLED;
3677b537b24SGary R Hook }
3687b537b24SGary R Hook
ccp_init(struct ccp_device * ccp)369ea0375afSGary R Hook static int ccp_init(struct ccp_device *ccp)
370ea0375afSGary R Hook {
371ea0375afSGary R Hook struct device *dev = ccp->dev;
372ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q;
373ea0375afSGary R Hook struct dma_pool *dma_pool;
374ea0375afSGary R Hook char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
3757b537b24SGary R Hook unsigned int qmr, i;
376ea0375afSGary R Hook int ret;
377ea0375afSGary R Hook
378ea0375afSGary R Hook /* Find available queues */
3797b537b24SGary R Hook ccp->qim = 0;
380ea0375afSGary R Hook qmr = ioread32(ccp->io_regs + Q_MASK_REG);
381c4a89279SHook, Gary for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
382ea0375afSGary R Hook if (!(qmr & (1 << i)))
383ea0375afSGary R Hook continue;
384ea0375afSGary R Hook
385ea0375afSGary R Hook /* Allocate a dma pool for this queue */
386ea0375afSGary R Hook snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
387ea0375afSGary R Hook ccp->name, i);
388ea0375afSGary R Hook dma_pool = dma_pool_create(dma_pool_name, dev,
389ea0375afSGary R Hook CCP_DMAPOOL_MAX_SIZE,
390ea0375afSGary R Hook CCP_DMAPOOL_ALIGN, 0);
391ea0375afSGary R Hook if (!dma_pool) {
392ea0375afSGary R Hook dev_err(dev, "unable to allocate dma pool\n");
393ea0375afSGary R Hook ret = -ENOMEM;
394ea0375afSGary R Hook goto e_pool;
395ea0375afSGary R Hook }
396ea0375afSGary R Hook
397ea0375afSGary R Hook cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
398ea0375afSGary R Hook ccp->cmd_q_count++;
399ea0375afSGary R Hook
400ea0375afSGary R Hook cmd_q->ccp = ccp;
401ea0375afSGary R Hook cmd_q->id = i;
402ea0375afSGary R Hook cmd_q->dma_pool = dma_pool;
403ea0375afSGary R Hook
404ea0375afSGary R Hook /* Reserve 2 KSB regions for the queue */
405956ee21aSGary R Hook cmd_q->sb_key = KSB_START + ccp->sb_start++;
406956ee21aSGary R Hook cmd_q->sb_ctx = KSB_START + ccp->sb_start++;
407956ee21aSGary R Hook ccp->sb_count -= 2;
408ea0375afSGary R Hook
409ea0375afSGary R Hook /* Preset some register values and masks that are queue
410ea0375afSGary R Hook * number dependent
411ea0375afSGary R Hook */
412ea0375afSGary R Hook cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
413ea0375afSGary R Hook (CMD_Q_STATUS_INCR * i);
414ea0375afSGary R Hook cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
415ea0375afSGary R Hook (CMD_Q_STATUS_INCR * i);
416ea0375afSGary R Hook cmd_q->int_ok = 1 << (i * 2);
417ea0375afSGary R Hook cmd_q->int_err = 1 << ((i * 2) + 1);
418ea0375afSGary R Hook
419bb4e89b3SGary R Hook cmd_q->free_slots = ccp_get_free_slots(cmd_q);
420ea0375afSGary R Hook
421ea0375afSGary R Hook init_waitqueue_head(&cmd_q->int_queue);
422ea0375afSGary R Hook
423ea0375afSGary R Hook /* Build queue interrupt mask (two interrupts per queue) */
4247b537b24SGary R Hook ccp->qim |= cmd_q->int_ok | cmd_q->int_err;
425ea0375afSGary R Hook
426ea0375afSGary R Hook #ifdef CONFIG_ARM64
427ea0375afSGary R Hook /* For arm64 set the recommended queue cache settings */
428ea0375afSGary R Hook iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
429ea0375afSGary R Hook (CMD_Q_CACHE_INC * i));
430ea0375afSGary R Hook #endif
431ea0375afSGary R Hook
432ea0375afSGary R Hook dev_dbg(dev, "queue #%u available\n", i);
433ea0375afSGary R Hook }
434ea0375afSGary R Hook if (ccp->cmd_q_count == 0) {
435ea0375afSGary R Hook dev_notice(dev, "no command queues available\n");
436ea0375afSGary R Hook ret = -EIO;
437ea0375afSGary R Hook goto e_pool;
438ea0375afSGary R Hook }
439ea0375afSGary R Hook dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
440ea0375afSGary R Hook
441ea0375afSGary R Hook /* Disable and clear interrupts until ready */
4427b537b24SGary R Hook ccp_disable_queue_interrupts(ccp);
443ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
444ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i];
445ea0375afSGary R Hook
446ea0375afSGary R Hook ioread32(cmd_q->reg_int_status);
447ea0375afSGary R Hook ioread32(cmd_q->reg_status);
448ea0375afSGary R Hook }
4497b537b24SGary R Hook iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
450ea0375afSGary R Hook
451ea0375afSGary R Hook /* Request an irq */
452f4d18d65SBrijesh Singh ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
453ea0375afSGary R Hook if (ret) {
454ea0375afSGary R Hook dev_err(dev, "unable to allocate an IRQ\n");
455ea0375afSGary R Hook goto e_pool;
456ea0375afSGary R Hook }
457ea0375afSGary R Hook
4587b537b24SGary R Hook /* Initialize the ISR tasklet? */
4597b537b24SGary R Hook if (ccp->use_tasklet)
4607b537b24SGary R Hook tasklet_init(&ccp->irq_tasklet, ccp_irq_bh,
4617b537b24SGary R Hook (unsigned long)ccp);
4627b537b24SGary R Hook
4634b394a23SGary R Hook dev_dbg(dev, "Starting threads...\n");
464ea0375afSGary R Hook /* Create a kthread for each queue */
465ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
466ea0375afSGary R Hook struct task_struct *kthread;
467ea0375afSGary R Hook
468ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i];
469ea0375afSGary R Hook
470*83bff109SCai Huoqing kthread = kthread_run(ccp_cmd_queue_thread, cmd_q,
471ea0375afSGary R Hook "%s-q%u", ccp->name, cmd_q->id);
472ea0375afSGary R Hook if (IS_ERR(kthread)) {
473ea0375afSGary R Hook dev_err(dev, "error creating queue thread (%ld)\n",
474ea0375afSGary R Hook PTR_ERR(kthread));
475ea0375afSGary R Hook ret = PTR_ERR(kthread);
476ea0375afSGary R Hook goto e_kthread;
477ea0375afSGary R Hook }
478ea0375afSGary R Hook
479ea0375afSGary R Hook cmd_q->kthread = kthread;
480ea0375afSGary R Hook }
481ea0375afSGary R Hook
4824b394a23SGary R Hook dev_dbg(dev, "Enabling interrupts...\n");
4834b394a23SGary R Hook /* Enable interrupts */
4847b537b24SGary R Hook ccp_enable_queue_interrupts(ccp);
4854b394a23SGary R Hook
4864b394a23SGary R Hook dev_dbg(dev, "Registering device...\n");
4874b394a23SGary R Hook ccp_add_device(ccp);
4884b394a23SGary R Hook
489084935b2SGary R Hook ret = ccp_register_rng(ccp);
490084935b2SGary R Hook if (ret)
491ea0375afSGary R Hook goto e_kthread;
492ea0375afSGary R Hook
49358ea8abfSGary R Hook /* Register the DMA engine support */
49458ea8abfSGary R Hook ret = ccp_dmaengine_register(ccp);
49558ea8abfSGary R Hook if (ret)
49658ea8abfSGary R Hook goto e_hwrng;
49758ea8abfSGary R Hook
498ea0375afSGary R Hook return 0;
499ea0375afSGary R Hook
50058ea8abfSGary R Hook e_hwrng:
501084935b2SGary R Hook ccp_unregister_rng(ccp);
50258ea8abfSGary R Hook
503ea0375afSGary R Hook e_kthread:
504ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
505ea0375afSGary R Hook if (ccp->cmd_q[i].kthread)
506ea0375afSGary R Hook kthread_stop(ccp->cmd_q[i].kthread);
507ea0375afSGary R Hook
508f4d18d65SBrijesh Singh sp_free_ccp_irq(ccp->sp, ccp);
509ea0375afSGary R Hook
510ea0375afSGary R Hook e_pool:
511ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
512ea0375afSGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool);
513ea0375afSGary R Hook
514ea0375afSGary R Hook return ret;
515ea0375afSGary R Hook }
516ea0375afSGary R Hook
ccp_destroy(struct ccp_device * ccp)517ea0375afSGary R Hook static void ccp_destroy(struct ccp_device *ccp)
518ea0375afSGary R Hook {
519ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q;
520ea0375afSGary R Hook struct ccp_cmd *cmd;
5217b537b24SGary R Hook unsigned int i;
522ea0375afSGary R Hook
5234b394a23SGary R Hook /* Unregister the DMA engine */
5244b394a23SGary R Hook ccp_dmaengine_unregister(ccp);
5254b394a23SGary R Hook
5264b394a23SGary R Hook /* Unregister the RNG */
527084935b2SGary R Hook ccp_unregister_rng(ccp);
5284b394a23SGary R Hook
5294b394a23SGary R Hook /* Remove this device from the list of available units */
530ea0375afSGary R Hook ccp_del_device(ccp);
531ea0375afSGary R Hook
532ea0375afSGary R Hook /* Disable and clear interrupts */
5337b537b24SGary R Hook ccp_disable_queue_interrupts(ccp);
534ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
535ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i];
536ea0375afSGary R Hook
537ea0375afSGary R Hook ioread32(cmd_q->reg_int_status);
538ea0375afSGary R Hook ioread32(cmd_q->reg_status);
539ea0375afSGary R Hook }
5407b537b24SGary R Hook iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
541ea0375afSGary R Hook
5428256e683SGary R Hook /* Stop the queue kthreads */
5438256e683SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
5448256e683SGary R Hook if (ccp->cmd_q[i].kthread)
5458256e683SGary R Hook kthread_stop(ccp->cmd_q[i].kthread);
5468256e683SGary R Hook
547f4d18d65SBrijesh Singh sp_free_ccp_irq(ccp->sp, ccp);
548ea0375afSGary R Hook
549ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
550ea0375afSGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool);
551ea0375afSGary R Hook
552ea0375afSGary R Hook /* Flush the cmd and backlog queue */
553ea0375afSGary R Hook while (!list_empty(&ccp->cmd)) {
554ea0375afSGary R Hook /* Invoke the callback directly with an error code */
555ea0375afSGary R Hook cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
556ea0375afSGary R Hook list_del(&cmd->entry);
557ea0375afSGary R Hook cmd->callback(cmd->data, -ENODEV);
558ea0375afSGary R Hook }
559ea0375afSGary R Hook while (!list_empty(&ccp->backlog)) {
560ea0375afSGary R Hook /* Invoke the callback directly with an error code */
561ea0375afSGary R Hook cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
562ea0375afSGary R Hook list_del(&cmd->entry);
563ea0375afSGary R Hook cmd->callback(cmd->data, -ENODEV);
564ea0375afSGary R Hook }
565ea0375afSGary R Hook }
566ea0375afSGary R Hook
567bc197b2aSJulia Lawall static const struct ccp_actions ccp3_actions = {
568a43eb985SGary R Hook .aes = ccp_perform_aes,
569a43eb985SGary R Hook .xts_aes = ccp_perform_xts_aes,
570990672d4SGary R Hook .des3 = NULL,
571a43eb985SGary R Hook .sha = ccp_perform_sha,
572a43eb985SGary R Hook .rsa = ccp_perform_rsa,
573a43eb985SGary R Hook .passthru = ccp_perform_passthru,
574a43eb985SGary R Hook .ecc = ccp_perform_ecc,
57558a690b7SGary R Hook .sballoc = ccp_alloc_ksb,
57658a690b7SGary R Hook .sbfree = ccp_free_ksb,
577ea0375afSGary R Hook .init = ccp_init,
578ea0375afSGary R Hook .destroy = ccp_destroy,
579bb4e89b3SGary R Hook .get_free_slots = ccp_get_free_slots,
580ea0375afSGary R Hook .irqhandler = ccp_irq_handler,
581ea0375afSGary R Hook };
582ea0375afSGary R Hook
583970e8303SBrijesh Singh const struct ccp_vdata ccpv3_platform = {
584970e8303SBrijesh Singh .version = CCP_VERSION(3, 0),
585970e8303SBrijesh Singh .setup = NULL,
586970e8303SBrijesh Singh .perform = &ccp3_actions,
587970e8303SBrijesh Singh .offset = 0,
58811548f5aSArd Biesheuvel .rsamax = CCP_RSA_MAX_WIDTH,
589970e8303SBrijesh Singh };
590970e8303SBrijesh Singh
5919ddb9dc6SGary R Hook const struct ccp_vdata ccpv3 = {
592ea0375afSGary R Hook .version = CCP_VERSION(3, 0),
5934b394a23SGary R Hook .setup = NULL,
594ea0375afSGary R Hook .perform = &ccp3_actions,
595fba8855cSGary R Hook .offset = 0x20000,
596e28c190dSGary R Hook .rsamax = CCP_RSA_MAX_WIDTH,
597ea0375afSGary R Hook };
598