1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2ea0375afSGary R Hook /* 3ea0375afSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver 4ea0375afSGary R Hook * 568cc652fSGary R Hook * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. 6ea0375afSGary R Hook * 7ea0375afSGary R Hook * Author: Tom Lendacky <thomas.lendacky@amd.com> 8fba8855cSGary R Hook * Author: Gary R Hook <gary.hook@amd.com> 9ea0375afSGary R Hook */ 10ea0375afSGary R Hook 11ea0375afSGary R Hook #include <linux/module.h> 12ea0375afSGary R Hook #include <linux/kernel.h> 13ea0375afSGary R Hook #include <linux/kthread.h> 14ea0375afSGary R Hook #include <linux/interrupt.h> 15ea0375afSGary R Hook #include <linux/ccp.h> 16ea0375afSGary R Hook 17ea0375afSGary R Hook #include "ccp-dev.h" 18ea0375afSGary R Hook 1958a690b7SGary R Hook static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) 2058a690b7SGary R Hook { 2158a690b7SGary R Hook int start; 2258a690b7SGary R Hook struct ccp_device *ccp = cmd_q->ccp; 2358a690b7SGary R Hook 2458a690b7SGary R Hook for (;;) { 2558a690b7SGary R Hook mutex_lock(&ccp->sb_mutex); 2658a690b7SGary R Hook 2758a690b7SGary R Hook start = (u32)bitmap_find_next_zero_area(ccp->sb, 2858a690b7SGary R Hook ccp->sb_count, 2958a690b7SGary R Hook ccp->sb_start, 3058a690b7SGary R Hook count, 0); 3158a690b7SGary R Hook if (start <= ccp->sb_count) { 3258a690b7SGary R Hook bitmap_set(ccp->sb, start, count); 3358a690b7SGary R Hook 3458a690b7SGary R Hook mutex_unlock(&ccp->sb_mutex); 3558a690b7SGary R Hook break; 3658a690b7SGary R Hook } 3758a690b7SGary R Hook 3858a690b7SGary R Hook ccp->sb_avail = 0; 3958a690b7SGary R Hook 4058a690b7SGary R Hook mutex_unlock(&ccp->sb_mutex); 4158a690b7SGary R Hook 4258a690b7SGary R Hook /* Wait for KSB entries to become available */ 4358a690b7SGary R Hook if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 4458a690b7SGary R Hook return 0; 4558a690b7SGary R Hook } 4658a690b7SGary R Hook 4758a690b7SGary R Hook return KSB_START + start; 4858a690b7SGary R Hook } 4958a690b7SGary R Hook 5058a690b7SGary R Hook static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, 5158a690b7SGary R Hook unsigned int count) 5258a690b7SGary R Hook { 5358a690b7SGary R Hook struct ccp_device *ccp = cmd_q->ccp; 5458a690b7SGary R Hook 5558a690b7SGary R Hook if (!start) 5658a690b7SGary R Hook return; 5758a690b7SGary R Hook 5858a690b7SGary R Hook mutex_lock(&ccp->sb_mutex); 5958a690b7SGary R Hook 6058a690b7SGary R Hook bitmap_clear(ccp->sb, start - KSB_START, count); 6158a690b7SGary R Hook 6258a690b7SGary R Hook ccp->sb_avail = 1; 6358a690b7SGary R Hook 6458a690b7SGary R Hook mutex_unlock(&ccp->sb_mutex); 6558a690b7SGary R Hook 6658a690b7SGary R Hook wake_up_interruptible_all(&ccp->sb_queue); 6758a690b7SGary R Hook } 6858a690b7SGary R Hook 69bb4e89b3SGary R Hook static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) 70bb4e89b3SGary R Hook { 71bb4e89b3SGary R Hook return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 72bb4e89b3SGary R Hook } 73bb4e89b3SGary R Hook 74ea0375afSGary R Hook static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 75ea0375afSGary R Hook { 76ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q = op->cmd_q; 77ea0375afSGary R Hook struct ccp_device *ccp = cmd_q->ccp; 78ea0375afSGary R Hook void __iomem *cr_addr; 79ea0375afSGary R Hook u32 cr0, cmd; 80ea0375afSGary R Hook unsigned int i; 81ea0375afSGary R Hook int ret = 0; 82ea0375afSGary R Hook 83ea0375afSGary R Hook /* We could read a status register to see how many free slots 84ea0375afSGary R Hook * are actually available, but reading that register resets it 85ea0375afSGary R Hook * and you could lose some error information. 86ea0375afSGary R Hook */ 87ea0375afSGary R Hook cmd_q->free_slots--; 88ea0375afSGary R Hook 89ea0375afSGary R Hook cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 90ea0375afSGary R Hook | (op->jobid << REQ0_JOBID_SHIFT) 91ea0375afSGary R Hook | REQ0_WAIT_FOR_WRITE; 92ea0375afSGary R Hook 93ea0375afSGary R Hook if (op->soc) 94ea0375afSGary R Hook cr0 |= REQ0_STOP_ON_COMPLETE 95ea0375afSGary R Hook | REQ0_INT_ON_COMPLETE; 96ea0375afSGary R Hook 97ea0375afSGary R Hook if (op->ioc || !cmd_q->free_slots) 98ea0375afSGary R Hook cr0 |= REQ0_INT_ON_COMPLETE; 99ea0375afSGary R Hook 100ea0375afSGary R Hook /* Start at CMD_REQ1 */ 101ea0375afSGary R Hook cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 102ea0375afSGary R Hook 103ea0375afSGary R Hook mutex_lock(&ccp->req_mutex); 104ea0375afSGary R Hook 105ea0375afSGary R Hook /* Write CMD_REQ1 through CMD_REQx first */ 106ea0375afSGary R Hook for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 107ea0375afSGary R Hook iowrite32(*(cr + i), cr_addr); 108ea0375afSGary R Hook 109ea0375afSGary R Hook /* Tell the CCP to start */ 110ea0375afSGary R Hook wmb(); 111ea0375afSGary R Hook iowrite32(cr0, ccp->io_regs + CMD_REQ0); 112ea0375afSGary R Hook 113ea0375afSGary R Hook mutex_unlock(&ccp->req_mutex); 114ea0375afSGary R Hook 115ea0375afSGary R Hook if (cr0 & REQ0_INT_ON_COMPLETE) { 116ea0375afSGary R Hook /* Wait for the job to complete */ 117ea0375afSGary R Hook ret = wait_event_interruptible(cmd_q->int_queue, 118ea0375afSGary R Hook cmd_q->int_rcvd); 119ea0375afSGary R Hook if (ret || cmd_q->cmd_error) { 120ea0375afSGary R Hook /* On error delete all related jobs from the queue */ 121ea0375afSGary R Hook cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 122ea0375afSGary R Hook | op->jobid; 12381422badSGary R Hook if (cmd_q->cmd_error) 12481422badSGary R Hook ccp_log_error(cmd_q->ccp, 12581422badSGary R Hook cmd_q->cmd_error); 126ea0375afSGary R Hook 127ea0375afSGary R Hook iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 128ea0375afSGary R Hook 129ea0375afSGary R Hook if (!ret) 130ea0375afSGary R Hook ret = -EIO; 131ea0375afSGary R Hook } else if (op->soc) { 132ea0375afSGary R Hook /* Delete just head job from the queue on SoC */ 133ea0375afSGary R Hook cmd = DEL_Q_ACTIVE 134ea0375afSGary R Hook | (cmd_q->id << DEL_Q_ID_SHIFT) 135ea0375afSGary R Hook | op->jobid; 136ea0375afSGary R Hook 137ea0375afSGary R Hook iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 138ea0375afSGary R Hook } 139ea0375afSGary R Hook 140ea0375afSGary R Hook cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 141ea0375afSGary R Hook 142ea0375afSGary R Hook cmd_q->int_rcvd = 0; 143ea0375afSGary R Hook } 144ea0375afSGary R Hook 145ea0375afSGary R Hook return ret; 146ea0375afSGary R Hook } 147ea0375afSGary R Hook 148ea0375afSGary R Hook static int ccp_perform_aes(struct ccp_op *op) 149ea0375afSGary R Hook { 150ea0375afSGary R Hook u32 cr[6]; 151ea0375afSGary R Hook 152ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 153ea0375afSGary R Hook cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 154ea0375afSGary R Hook | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 155ea0375afSGary R Hook | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 156ea0375afSGary R Hook | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 157956ee21aSGary R Hook | (op->sb_key << REQ1_KEY_KSB_SHIFT); 158ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 159ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 160956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 161ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 162ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 163ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 164ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 165ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 166ea0375afSGary R Hook 167ea0375afSGary R Hook if (op->u.aes.mode == CCP_AES_MODE_CFB) 168ea0375afSGary R Hook cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 169ea0375afSGary R Hook 170ea0375afSGary R Hook if (op->eom) 171ea0375afSGary R Hook cr[0] |= REQ1_EOM; 172ea0375afSGary R Hook 173ea0375afSGary R Hook if (op->init) 174ea0375afSGary R Hook cr[0] |= REQ1_INIT; 175ea0375afSGary R Hook 176ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 177ea0375afSGary R Hook } 178ea0375afSGary R Hook 179ea0375afSGary R Hook static int ccp_perform_xts_aes(struct ccp_op *op) 180ea0375afSGary R Hook { 181ea0375afSGary R Hook u32 cr[6]; 182ea0375afSGary R Hook 183ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 184ea0375afSGary R Hook cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 185ea0375afSGary R Hook | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 186ea0375afSGary R Hook | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 187956ee21aSGary R Hook | (op->sb_key << REQ1_KEY_KSB_SHIFT); 188ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 189ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 190956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 191ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 192ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 193ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 194ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 195ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 196ea0375afSGary R Hook 197ea0375afSGary R Hook if (op->eom) 198ea0375afSGary R Hook cr[0] |= REQ1_EOM; 199ea0375afSGary R Hook 200ea0375afSGary R Hook if (op->init) 201ea0375afSGary R Hook cr[0] |= REQ1_INIT; 202ea0375afSGary R Hook 203ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 204ea0375afSGary R Hook } 205ea0375afSGary R Hook 206ea0375afSGary R Hook static int ccp_perform_sha(struct ccp_op *op) 207ea0375afSGary R Hook { 208ea0375afSGary R Hook u32 cr[6]; 209ea0375afSGary R Hook 210ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 211ea0375afSGary R Hook cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 212ea0375afSGary R Hook | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 213ea0375afSGary R Hook | REQ1_INIT; 214ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 215ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 216956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 217ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 218ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 219ea0375afSGary R Hook 220ea0375afSGary R Hook if (op->eom) { 221ea0375afSGary R Hook cr[0] |= REQ1_EOM; 222ea0375afSGary R Hook cr[4] = lower_32_bits(op->u.sha.msg_bits); 223ea0375afSGary R Hook cr[5] = upper_32_bits(op->u.sha.msg_bits); 224ea0375afSGary R Hook } else { 225ea0375afSGary R Hook cr[4] = 0; 226ea0375afSGary R Hook cr[5] = 0; 227ea0375afSGary R Hook } 228ea0375afSGary R Hook 229ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 230ea0375afSGary R Hook } 231ea0375afSGary R Hook 232ea0375afSGary R Hook static int ccp_perform_rsa(struct ccp_op *op) 233ea0375afSGary R Hook { 234ea0375afSGary R Hook u32 cr[6]; 235ea0375afSGary R Hook 236ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 237ea0375afSGary R Hook cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 238ea0375afSGary R Hook | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 239956ee21aSGary R Hook | (op->sb_key << REQ1_KEY_KSB_SHIFT) 240ea0375afSGary R Hook | REQ1_EOM; 241ea0375afSGary R Hook cr[1] = op->u.rsa.input_len - 1; 242ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 243956ee21aSGary R Hook cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) 244ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 245ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 246ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 247ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 248ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 249ea0375afSGary R Hook 250ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 251ea0375afSGary R Hook } 252ea0375afSGary R Hook 253ea0375afSGary R Hook static int ccp_perform_passthru(struct ccp_op *op) 254ea0375afSGary R Hook { 255ea0375afSGary R Hook u32 cr[6]; 256ea0375afSGary R Hook 257ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 258ea0375afSGary R Hook cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 259ea0375afSGary R Hook | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 260ea0375afSGary R Hook | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 261ea0375afSGary R Hook 262ea0375afSGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM) 263ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 264ea0375afSGary R Hook else 265ea0375afSGary R Hook cr[1] = op->dst.u.dma.length - 1; 266ea0375afSGary R Hook 267ea0375afSGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM) { 268ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 269ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 270ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 271ea0375afSGary R Hook 272ea0375afSGary R Hook if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 273956ee21aSGary R Hook cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); 274ea0375afSGary R Hook } else { 275956ee21aSGary R Hook cr[2] = op->src.u.sb * CCP_SB_BYTES; 276956ee21aSGary R Hook cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); 277ea0375afSGary R Hook } 278ea0375afSGary R Hook 279ea0375afSGary R Hook if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 280ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 281ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 282ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 283ea0375afSGary R Hook } else { 284956ee21aSGary R Hook cr[4] = op->dst.u.sb * CCP_SB_BYTES; 285956ee21aSGary R Hook cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); 286ea0375afSGary R Hook } 287ea0375afSGary R Hook 288ea0375afSGary R Hook if (op->eom) 289ea0375afSGary R Hook cr[0] |= REQ1_EOM; 290ea0375afSGary R Hook 291ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 292ea0375afSGary R Hook } 293ea0375afSGary R Hook 294ea0375afSGary R Hook static int ccp_perform_ecc(struct ccp_op *op) 295ea0375afSGary R Hook { 296ea0375afSGary R Hook u32 cr[6]; 297ea0375afSGary R Hook 298ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 299ea0375afSGary R Hook cr[0] = REQ1_ECC_AFFINE_CONVERT 300ea0375afSGary R Hook | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 301ea0375afSGary R Hook | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 302ea0375afSGary R Hook | REQ1_EOM; 303ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 304ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 305ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 306ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 307ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 308ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 309ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 310ea0375afSGary R Hook 311ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 312ea0375afSGary R Hook } 313ea0375afSGary R Hook 3147b537b24SGary R Hook static void ccp_disable_queue_interrupts(struct ccp_device *ccp) 3157b537b24SGary R Hook { 3167b537b24SGary R Hook iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); 3177b537b24SGary R Hook } 3187b537b24SGary R Hook 3197b537b24SGary R Hook static void ccp_enable_queue_interrupts(struct ccp_device *ccp) 3207b537b24SGary R Hook { 3217b537b24SGary R Hook iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); 3227b537b24SGary R Hook } 3237b537b24SGary R Hook 3247b537b24SGary R Hook static void ccp_irq_bh(unsigned long data) 3257b537b24SGary R Hook { 3267b537b24SGary R Hook struct ccp_device *ccp = (struct ccp_device *)data; 3277b537b24SGary R Hook struct ccp_cmd_queue *cmd_q; 3287b537b24SGary R Hook u32 q_int, status; 3297b537b24SGary R Hook unsigned int i; 3307b537b24SGary R Hook 3317b537b24SGary R Hook status = ioread32(ccp->io_regs + IRQ_STATUS_REG); 3327b537b24SGary R Hook 3337b537b24SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 3347b537b24SGary R Hook cmd_q = &ccp->cmd_q[i]; 3357b537b24SGary R Hook 3367b537b24SGary R Hook q_int = status & (cmd_q->int_ok | cmd_q->int_err); 3377b537b24SGary R Hook if (q_int) { 3387b537b24SGary R Hook cmd_q->int_status = status; 3397b537b24SGary R Hook cmd_q->q_status = ioread32(cmd_q->reg_status); 3407b537b24SGary R Hook cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 3417b537b24SGary R Hook 3427b537b24SGary R Hook /* On error, only save the first error value */ 3437b537b24SGary R Hook if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) 3447b537b24SGary R Hook cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 3457b537b24SGary R Hook 3467b537b24SGary R Hook cmd_q->int_rcvd = 1; 3477b537b24SGary R Hook 3487b537b24SGary R Hook /* Acknowledge the interrupt and wake the kthread */ 3497b537b24SGary R Hook iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); 3507b537b24SGary R Hook wake_up_interruptible(&cmd_q->int_queue); 3517b537b24SGary R Hook } 3527b537b24SGary R Hook } 3537b537b24SGary R Hook ccp_enable_queue_interrupts(ccp); 3547b537b24SGary R Hook } 3557b537b24SGary R Hook 3567b537b24SGary R Hook static irqreturn_t ccp_irq_handler(int irq, void *data) 3577b537b24SGary R Hook { 358720419f0SBrijesh Singh struct ccp_device *ccp = (struct ccp_device *)data; 3597b537b24SGary R Hook 3607b537b24SGary R Hook ccp_disable_queue_interrupts(ccp); 3617b537b24SGary R Hook if (ccp->use_tasklet) 3627b537b24SGary R Hook tasklet_schedule(&ccp->irq_tasklet); 3637b537b24SGary R Hook else 3647b537b24SGary R Hook ccp_irq_bh((unsigned long)ccp); 3657b537b24SGary R Hook 3667b537b24SGary R Hook return IRQ_HANDLED; 3677b537b24SGary R Hook } 3687b537b24SGary R Hook 369ea0375afSGary R Hook static int ccp_init(struct ccp_device *ccp) 370ea0375afSGary R Hook { 371ea0375afSGary R Hook struct device *dev = ccp->dev; 372ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q; 373ea0375afSGary R Hook struct dma_pool *dma_pool; 374ea0375afSGary R Hook char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 3757b537b24SGary R Hook unsigned int qmr, i; 376ea0375afSGary R Hook int ret; 377ea0375afSGary R Hook 378ea0375afSGary R Hook /* Find available queues */ 3797b537b24SGary R Hook ccp->qim = 0; 380ea0375afSGary R Hook qmr = ioread32(ccp->io_regs + Q_MASK_REG); 381c4a89279SHook, Gary for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { 382ea0375afSGary R Hook if (!(qmr & (1 << i))) 383ea0375afSGary R Hook continue; 384ea0375afSGary R Hook 385ea0375afSGary R Hook /* Allocate a dma pool for this queue */ 386ea0375afSGary R Hook snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 387ea0375afSGary R Hook ccp->name, i); 388ea0375afSGary R Hook dma_pool = dma_pool_create(dma_pool_name, dev, 389ea0375afSGary R Hook CCP_DMAPOOL_MAX_SIZE, 390ea0375afSGary R Hook CCP_DMAPOOL_ALIGN, 0); 391ea0375afSGary R Hook if (!dma_pool) { 392ea0375afSGary R Hook dev_err(dev, "unable to allocate dma pool\n"); 393ea0375afSGary R Hook ret = -ENOMEM; 394ea0375afSGary R Hook goto e_pool; 395ea0375afSGary R Hook } 396ea0375afSGary R Hook 397ea0375afSGary R Hook cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 398ea0375afSGary R Hook ccp->cmd_q_count++; 399ea0375afSGary R Hook 400ea0375afSGary R Hook cmd_q->ccp = ccp; 401ea0375afSGary R Hook cmd_q->id = i; 402ea0375afSGary R Hook cmd_q->dma_pool = dma_pool; 403ea0375afSGary R Hook 404ea0375afSGary R Hook /* Reserve 2 KSB regions for the queue */ 405956ee21aSGary R Hook cmd_q->sb_key = KSB_START + ccp->sb_start++; 406956ee21aSGary R Hook cmd_q->sb_ctx = KSB_START + ccp->sb_start++; 407956ee21aSGary R Hook ccp->sb_count -= 2; 408ea0375afSGary R Hook 409ea0375afSGary R Hook /* Preset some register values and masks that are queue 410ea0375afSGary R Hook * number dependent 411ea0375afSGary R Hook */ 412ea0375afSGary R Hook cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + 413ea0375afSGary R Hook (CMD_Q_STATUS_INCR * i); 414ea0375afSGary R Hook cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + 415ea0375afSGary R Hook (CMD_Q_STATUS_INCR * i); 416ea0375afSGary R Hook cmd_q->int_ok = 1 << (i * 2); 417ea0375afSGary R Hook cmd_q->int_err = 1 << ((i * 2) + 1); 418ea0375afSGary R Hook 419bb4e89b3SGary R Hook cmd_q->free_slots = ccp_get_free_slots(cmd_q); 420ea0375afSGary R Hook 421ea0375afSGary R Hook init_waitqueue_head(&cmd_q->int_queue); 422ea0375afSGary R Hook 423ea0375afSGary R Hook /* Build queue interrupt mask (two interrupts per queue) */ 4247b537b24SGary R Hook ccp->qim |= cmd_q->int_ok | cmd_q->int_err; 425ea0375afSGary R Hook 426ea0375afSGary R Hook #ifdef CONFIG_ARM64 427ea0375afSGary R Hook /* For arm64 set the recommended queue cache settings */ 428ea0375afSGary R Hook iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + 429ea0375afSGary R Hook (CMD_Q_CACHE_INC * i)); 430ea0375afSGary R Hook #endif 431ea0375afSGary R Hook 432ea0375afSGary R Hook dev_dbg(dev, "queue #%u available\n", i); 433ea0375afSGary R Hook } 434ea0375afSGary R Hook if (ccp->cmd_q_count == 0) { 435ea0375afSGary R Hook dev_notice(dev, "no command queues available\n"); 436ea0375afSGary R Hook ret = -EIO; 437ea0375afSGary R Hook goto e_pool; 438ea0375afSGary R Hook } 439ea0375afSGary R Hook dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); 440ea0375afSGary R Hook 441ea0375afSGary R Hook /* Disable and clear interrupts until ready */ 4427b537b24SGary R Hook ccp_disable_queue_interrupts(ccp); 443ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 444ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 445ea0375afSGary R Hook 446ea0375afSGary R Hook ioread32(cmd_q->reg_int_status); 447ea0375afSGary R Hook ioread32(cmd_q->reg_status); 448ea0375afSGary R Hook } 4497b537b24SGary R Hook iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); 450ea0375afSGary R Hook 451ea0375afSGary R Hook /* Request an irq */ 452f4d18d65SBrijesh Singh ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp); 453ea0375afSGary R Hook if (ret) { 454ea0375afSGary R Hook dev_err(dev, "unable to allocate an IRQ\n"); 455ea0375afSGary R Hook goto e_pool; 456ea0375afSGary R Hook } 457ea0375afSGary R Hook 4587b537b24SGary R Hook /* Initialize the ISR tasklet? */ 4597b537b24SGary R Hook if (ccp->use_tasklet) 4607b537b24SGary R Hook tasklet_init(&ccp->irq_tasklet, ccp_irq_bh, 4617b537b24SGary R Hook (unsigned long)ccp); 4627b537b24SGary R Hook 4634b394a23SGary R Hook dev_dbg(dev, "Starting threads...\n"); 464ea0375afSGary R Hook /* Create a kthread for each queue */ 465ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 466ea0375afSGary R Hook struct task_struct *kthread; 467ea0375afSGary R Hook 468ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 469ea0375afSGary R Hook 470ea0375afSGary R Hook kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, 471ea0375afSGary R Hook "%s-q%u", ccp->name, cmd_q->id); 472ea0375afSGary R Hook if (IS_ERR(kthread)) { 473ea0375afSGary R Hook dev_err(dev, "error creating queue thread (%ld)\n", 474ea0375afSGary R Hook PTR_ERR(kthread)); 475ea0375afSGary R Hook ret = PTR_ERR(kthread); 476ea0375afSGary R Hook goto e_kthread; 477ea0375afSGary R Hook } 478ea0375afSGary R Hook 479ea0375afSGary R Hook cmd_q->kthread = kthread; 480ea0375afSGary R Hook wake_up_process(kthread); 481ea0375afSGary R Hook } 482ea0375afSGary R Hook 4834b394a23SGary R Hook dev_dbg(dev, "Enabling interrupts...\n"); 4844b394a23SGary R Hook /* Enable interrupts */ 4857b537b24SGary R Hook ccp_enable_queue_interrupts(ccp); 4864b394a23SGary R Hook 4874b394a23SGary R Hook dev_dbg(dev, "Registering device...\n"); 4884b394a23SGary R Hook ccp_add_device(ccp); 4894b394a23SGary R Hook 490084935b2SGary R Hook ret = ccp_register_rng(ccp); 491084935b2SGary R Hook if (ret) 492ea0375afSGary R Hook goto e_kthread; 493ea0375afSGary R Hook 49458ea8abfSGary R Hook /* Register the DMA engine support */ 49558ea8abfSGary R Hook ret = ccp_dmaengine_register(ccp); 49658ea8abfSGary R Hook if (ret) 49758ea8abfSGary R Hook goto e_hwrng; 49858ea8abfSGary R Hook 499ea0375afSGary R Hook return 0; 500ea0375afSGary R Hook 50158ea8abfSGary R Hook e_hwrng: 502084935b2SGary R Hook ccp_unregister_rng(ccp); 50358ea8abfSGary R Hook 504ea0375afSGary R Hook e_kthread: 505ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 506ea0375afSGary R Hook if (ccp->cmd_q[i].kthread) 507ea0375afSGary R Hook kthread_stop(ccp->cmd_q[i].kthread); 508ea0375afSGary R Hook 509f4d18d65SBrijesh Singh sp_free_ccp_irq(ccp->sp, ccp); 510ea0375afSGary R Hook 511ea0375afSGary R Hook e_pool: 512ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 513ea0375afSGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool); 514ea0375afSGary R Hook 515ea0375afSGary R Hook return ret; 516ea0375afSGary R Hook } 517ea0375afSGary R Hook 518ea0375afSGary R Hook static void ccp_destroy(struct ccp_device *ccp) 519ea0375afSGary R Hook { 520ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q; 521ea0375afSGary R Hook struct ccp_cmd *cmd; 5227b537b24SGary R Hook unsigned int i; 523ea0375afSGary R Hook 5244b394a23SGary R Hook /* Unregister the DMA engine */ 5254b394a23SGary R Hook ccp_dmaengine_unregister(ccp); 5264b394a23SGary R Hook 5274b394a23SGary R Hook /* Unregister the RNG */ 528084935b2SGary R Hook ccp_unregister_rng(ccp); 5294b394a23SGary R Hook 5304b394a23SGary R Hook /* Remove this device from the list of available units */ 531ea0375afSGary R Hook ccp_del_device(ccp); 532ea0375afSGary R Hook 533ea0375afSGary R Hook /* Disable and clear interrupts */ 5347b537b24SGary R Hook ccp_disable_queue_interrupts(ccp); 535ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 536ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 537ea0375afSGary R Hook 538ea0375afSGary R Hook ioread32(cmd_q->reg_int_status); 539ea0375afSGary R Hook ioread32(cmd_q->reg_status); 540ea0375afSGary R Hook } 5417b537b24SGary R Hook iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); 542ea0375afSGary R Hook 5438256e683SGary R Hook /* Stop the queue kthreads */ 5448256e683SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 5458256e683SGary R Hook if (ccp->cmd_q[i].kthread) 5468256e683SGary R Hook kthread_stop(ccp->cmd_q[i].kthread); 5478256e683SGary R Hook 548f4d18d65SBrijesh Singh sp_free_ccp_irq(ccp->sp, ccp); 549ea0375afSGary R Hook 550ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 551ea0375afSGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool); 552ea0375afSGary R Hook 553ea0375afSGary R Hook /* Flush the cmd and backlog queue */ 554ea0375afSGary R Hook while (!list_empty(&ccp->cmd)) { 555ea0375afSGary R Hook /* Invoke the callback directly with an error code */ 556ea0375afSGary R Hook cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 557ea0375afSGary R Hook list_del(&cmd->entry); 558ea0375afSGary R Hook cmd->callback(cmd->data, -ENODEV); 559ea0375afSGary R Hook } 560ea0375afSGary R Hook while (!list_empty(&ccp->backlog)) { 561ea0375afSGary R Hook /* Invoke the callback directly with an error code */ 562ea0375afSGary R Hook cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 563ea0375afSGary R Hook list_del(&cmd->entry); 564ea0375afSGary R Hook cmd->callback(cmd->data, -ENODEV); 565ea0375afSGary R Hook } 566ea0375afSGary R Hook } 567ea0375afSGary R Hook 568bc197b2aSJulia Lawall static const struct ccp_actions ccp3_actions = { 569a43eb985SGary R Hook .aes = ccp_perform_aes, 570a43eb985SGary R Hook .xts_aes = ccp_perform_xts_aes, 571990672d4SGary R Hook .des3 = NULL, 572a43eb985SGary R Hook .sha = ccp_perform_sha, 573a43eb985SGary R Hook .rsa = ccp_perform_rsa, 574a43eb985SGary R Hook .passthru = ccp_perform_passthru, 575a43eb985SGary R Hook .ecc = ccp_perform_ecc, 57658a690b7SGary R Hook .sballoc = ccp_alloc_ksb, 57758a690b7SGary R Hook .sbfree = ccp_free_ksb, 578ea0375afSGary R Hook .init = ccp_init, 579ea0375afSGary R Hook .destroy = ccp_destroy, 580bb4e89b3SGary R Hook .get_free_slots = ccp_get_free_slots, 581ea0375afSGary R Hook .irqhandler = ccp_irq_handler, 582ea0375afSGary R Hook }; 583ea0375afSGary R Hook 584970e8303SBrijesh Singh const struct ccp_vdata ccpv3_platform = { 585970e8303SBrijesh Singh .version = CCP_VERSION(3, 0), 586970e8303SBrijesh Singh .setup = NULL, 587970e8303SBrijesh Singh .perform = &ccp3_actions, 588970e8303SBrijesh Singh .offset = 0, 589*11548f5aSArd Biesheuvel .rsamax = CCP_RSA_MAX_WIDTH, 590970e8303SBrijesh Singh }; 591970e8303SBrijesh Singh 5929ddb9dc6SGary R Hook const struct ccp_vdata ccpv3 = { 593ea0375afSGary R Hook .version = CCP_VERSION(3, 0), 5944b394a23SGary R Hook .setup = NULL, 595ea0375afSGary R Hook .perform = &ccp3_actions, 596fba8855cSGary R Hook .offset = 0x20000, 597e28c190dSGary R Hook .rsamax = CCP_RSA_MAX_WIDTH, 598ea0375afSGary R Hook }; 599