1ea0375afSGary R Hook /* 2ea0375afSGary R Hook * AMD Cryptographic Coprocessor (CCP) driver 3ea0375afSGary R Hook * 4ea0375afSGary R Hook * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 5ea0375afSGary R Hook * 6ea0375afSGary R Hook * Author: Tom Lendacky <thomas.lendacky@amd.com> 7fba8855cSGary R Hook * Author: Gary R Hook <gary.hook@amd.com> 8ea0375afSGary R Hook * 9ea0375afSGary R Hook * This program is free software; you can redistribute it and/or modify 10ea0375afSGary R Hook * it under the terms of the GNU General Public License version 2 as 11ea0375afSGary R Hook * published by the Free Software Foundation. 12ea0375afSGary R Hook */ 13ea0375afSGary R Hook 14ea0375afSGary R Hook #include <linux/module.h> 15ea0375afSGary R Hook #include <linux/kernel.h> 16ea0375afSGary R Hook #include <linux/pci.h> 17ea0375afSGary R Hook #include <linux/kthread.h> 18ea0375afSGary R Hook #include <linux/interrupt.h> 19ea0375afSGary R Hook #include <linux/ccp.h> 20ea0375afSGary R Hook 21ea0375afSGary R Hook #include "ccp-dev.h" 22ea0375afSGary R Hook 23ea0375afSGary R Hook static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) 24ea0375afSGary R Hook { 25ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q = op->cmd_q; 26ea0375afSGary R Hook struct ccp_device *ccp = cmd_q->ccp; 27ea0375afSGary R Hook void __iomem *cr_addr; 28ea0375afSGary R Hook u32 cr0, cmd; 29ea0375afSGary R Hook unsigned int i; 30ea0375afSGary R Hook int ret = 0; 31ea0375afSGary R Hook 32ea0375afSGary R Hook /* We could read a status register to see how many free slots 33ea0375afSGary R Hook * are actually available, but reading that register resets it 34ea0375afSGary R Hook * and you could lose some error information. 35ea0375afSGary R Hook */ 36ea0375afSGary R Hook cmd_q->free_slots--; 37ea0375afSGary R Hook 38ea0375afSGary R Hook cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) 39ea0375afSGary R Hook | (op->jobid << REQ0_JOBID_SHIFT) 40ea0375afSGary R Hook | REQ0_WAIT_FOR_WRITE; 41ea0375afSGary R Hook 42ea0375afSGary R Hook if (op->soc) 43ea0375afSGary R Hook cr0 |= REQ0_STOP_ON_COMPLETE 44ea0375afSGary R Hook | REQ0_INT_ON_COMPLETE; 45ea0375afSGary R Hook 46ea0375afSGary R Hook if (op->ioc || !cmd_q->free_slots) 47ea0375afSGary R Hook cr0 |= REQ0_INT_ON_COMPLETE; 48ea0375afSGary R Hook 49ea0375afSGary R Hook /* Start at CMD_REQ1 */ 50ea0375afSGary R Hook cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; 51ea0375afSGary R Hook 52ea0375afSGary R Hook mutex_lock(&ccp->req_mutex); 53ea0375afSGary R Hook 54ea0375afSGary R Hook /* Write CMD_REQ1 through CMD_REQx first */ 55ea0375afSGary R Hook for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) 56ea0375afSGary R Hook iowrite32(*(cr + i), cr_addr); 57ea0375afSGary R Hook 58ea0375afSGary R Hook /* Tell the CCP to start */ 59ea0375afSGary R Hook wmb(); 60ea0375afSGary R Hook iowrite32(cr0, ccp->io_regs + CMD_REQ0); 61ea0375afSGary R Hook 62ea0375afSGary R Hook mutex_unlock(&ccp->req_mutex); 63ea0375afSGary R Hook 64ea0375afSGary R Hook if (cr0 & REQ0_INT_ON_COMPLETE) { 65ea0375afSGary R Hook /* Wait for the job to complete */ 66ea0375afSGary R Hook ret = wait_event_interruptible(cmd_q->int_queue, 67ea0375afSGary R Hook cmd_q->int_rcvd); 68ea0375afSGary R Hook if (ret || cmd_q->cmd_error) { 69ea0375afSGary R Hook /* On error delete all related jobs from the queue */ 70ea0375afSGary R Hook cmd = (cmd_q->id << DEL_Q_ID_SHIFT) 71ea0375afSGary R Hook | op->jobid; 72ea0375afSGary R Hook 73ea0375afSGary R Hook iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 74ea0375afSGary R Hook 75ea0375afSGary R Hook if (!ret) 76ea0375afSGary R Hook ret = -EIO; 77ea0375afSGary R Hook } else if (op->soc) { 78ea0375afSGary R Hook /* Delete just head job from the queue on SoC */ 79ea0375afSGary R Hook cmd = DEL_Q_ACTIVE 80ea0375afSGary R Hook | (cmd_q->id << DEL_Q_ID_SHIFT) 81ea0375afSGary R Hook | op->jobid; 82ea0375afSGary R Hook 83ea0375afSGary R Hook iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); 84ea0375afSGary R Hook } 85ea0375afSGary R Hook 86ea0375afSGary R Hook cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); 87ea0375afSGary R Hook 88ea0375afSGary R Hook cmd_q->int_rcvd = 0; 89ea0375afSGary R Hook } 90ea0375afSGary R Hook 91ea0375afSGary R Hook return ret; 92ea0375afSGary R Hook } 93ea0375afSGary R Hook 94ea0375afSGary R Hook static int ccp_perform_aes(struct ccp_op *op) 95ea0375afSGary R Hook { 96ea0375afSGary R Hook u32 cr[6]; 97ea0375afSGary R Hook 98ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 99ea0375afSGary R Hook cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) 100ea0375afSGary R Hook | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) 101ea0375afSGary R Hook | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) 102ea0375afSGary R Hook | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) 103ea0375afSGary R Hook | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 104ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 105ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 106ea0375afSGary R Hook cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 107ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 108ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 109ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 110ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 111ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 112ea0375afSGary R Hook 113ea0375afSGary R Hook if (op->u.aes.mode == CCP_AES_MODE_CFB) 114ea0375afSGary R Hook cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); 115ea0375afSGary R Hook 116ea0375afSGary R Hook if (op->eom) 117ea0375afSGary R Hook cr[0] |= REQ1_EOM; 118ea0375afSGary R Hook 119ea0375afSGary R Hook if (op->init) 120ea0375afSGary R Hook cr[0] |= REQ1_INIT; 121ea0375afSGary R Hook 122ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 123ea0375afSGary R Hook } 124ea0375afSGary R Hook 125ea0375afSGary R Hook static int ccp_perform_xts_aes(struct ccp_op *op) 126ea0375afSGary R Hook { 127ea0375afSGary R Hook u32 cr[6]; 128ea0375afSGary R Hook 129ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 130ea0375afSGary R Hook cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) 131ea0375afSGary R Hook | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) 132ea0375afSGary R Hook | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) 133ea0375afSGary R Hook | (op->ksb_key << REQ1_KEY_KSB_SHIFT); 134ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 135ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 136ea0375afSGary R Hook cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 137ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 138ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 139ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 140ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 141ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 142ea0375afSGary R Hook 143ea0375afSGary R Hook if (op->eom) 144ea0375afSGary R Hook cr[0] |= REQ1_EOM; 145ea0375afSGary R Hook 146ea0375afSGary R Hook if (op->init) 147ea0375afSGary R Hook cr[0] |= REQ1_INIT; 148ea0375afSGary R Hook 149ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 150ea0375afSGary R Hook } 151ea0375afSGary R Hook 152ea0375afSGary R Hook static int ccp_perform_sha(struct ccp_op *op) 153ea0375afSGary R Hook { 154ea0375afSGary R Hook u32 cr[6]; 155ea0375afSGary R Hook 156ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 157ea0375afSGary R Hook cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) 158ea0375afSGary R Hook | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) 159ea0375afSGary R Hook | REQ1_INIT; 160ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 161ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 162ea0375afSGary R Hook cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 163ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 164ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 165ea0375afSGary R Hook 166ea0375afSGary R Hook if (op->eom) { 167ea0375afSGary R Hook cr[0] |= REQ1_EOM; 168ea0375afSGary R Hook cr[4] = lower_32_bits(op->u.sha.msg_bits); 169ea0375afSGary R Hook cr[5] = upper_32_bits(op->u.sha.msg_bits); 170ea0375afSGary R Hook } else { 171ea0375afSGary R Hook cr[4] = 0; 172ea0375afSGary R Hook cr[5] = 0; 173ea0375afSGary R Hook } 174ea0375afSGary R Hook 175ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 176ea0375afSGary R Hook } 177ea0375afSGary R Hook 178ea0375afSGary R Hook static int ccp_perform_rsa(struct ccp_op *op) 179ea0375afSGary R Hook { 180ea0375afSGary R Hook u32 cr[6]; 181ea0375afSGary R Hook 182ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 183ea0375afSGary R Hook cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) 184ea0375afSGary R Hook | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) 185ea0375afSGary R Hook | (op->ksb_key << REQ1_KEY_KSB_SHIFT) 186ea0375afSGary R Hook | REQ1_EOM; 187ea0375afSGary R Hook cr[1] = op->u.rsa.input_len - 1; 188ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 189ea0375afSGary R Hook cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) 190ea0375afSGary R Hook | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 191ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 192ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 193ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 194ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 195ea0375afSGary R Hook 196ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 197ea0375afSGary R Hook } 198ea0375afSGary R Hook 199ea0375afSGary R Hook static int ccp_perform_passthru(struct ccp_op *op) 200ea0375afSGary R Hook { 201ea0375afSGary R Hook u32 cr[6]; 202ea0375afSGary R Hook 203ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 204ea0375afSGary R Hook cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) 205ea0375afSGary R Hook | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) 206ea0375afSGary R Hook | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); 207ea0375afSGary R Hook 208ea0375afSGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM) 209ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 210ea0375afSGary R Hook else 211ea0375afSGary R Hook cr[1] = op->dst.u.dma.length - 1; 212ea0375afSGary R Hook 213ea0375afSGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM) { 214ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 215ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 216ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 217ea0375afSGary R Hook 218ea0375afSGary R Hook if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 219ea0375afSGary R Hook cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); 220ea0375afSGary R Hook } else { 221ea0375afSGary R Hook cr[2] = op->src.u.ksb * CCP_KSB_BYTES; 222ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); 223ea0375afSGary R Hook } 224ea0375afSGary R Hook 225ea0375afSGary R Hook if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 226ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 227ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 228ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 229ea0375afSGary R Hook } else { 230ea0375afSGary R Hook cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; 231ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); 232ea0375afSGary R Hook } 233ea0375afSGary R Hook 234ea0375afSGary R Hook if (op->eom) 235ea0375afSGary R Hook cr[0] |= REQ1_EOM; 236ea0375afSGary R Hook 237ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 238ea0375afSGary R Hook } 239ea0375afSGary R Hook 240ea0375afSGary R Hook static int ccp_perform_ecc(struct ccp_op *op) 241ea0375afSGary R Hook { 242ea0375afSGary R Hook u32 cr[6]; 243ea0375afSGary R Hook 244ea0375afSGary R Hook /* Fill out the register contents for REQ1 through REQ6 */ 245ea0375afSGary R Hook cr[0] = REQ1_ECC_AFFINE_CONVERT 246ea0375afSGary R Hook | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) 247ea0375afSGary R Hook | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) 248ea0375afSGary R Hook | REQ1_EOM; 249ea0375afSGary R Hook cr[1] = op->src.u.dma.length - 1; 250ea0375afSGary R Hook cr[2] = ccp_addr_lo(&op->src.u.dma); 251ea0375afSGary R Hook cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) 252ea0375afSGary R Hook | ccp_addr_hi(&op->src.u.dma); 253ea0375afSGary R Hook cr[4] = ccp_addr_lo(&op->dst.u.dma); 254ea0375afSGary R Hook cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) 255ea0375afSGary R Hook | ccp_addr_hi(&op->dst.u.dma); 256ea0375afSGary R Hook 257ea0375afSGary R Hook return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); 258ea0375afSGary R Hook } 259ea0375afSGary R Hook 260ea0375afSGary R Hook static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) 261ea0375afSGary R Hook { 262ea0375afSGary R Hook struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); 263ea0375afSGary R Hook u32 trng_value; 264ea0375afSGary R Hook int len = min_t(int, sizeof(trng_value), max); 265ea0375afSGary R Hook 266ea0375afSGary R Hook /* 267ea0375afSGary R Hook * Locking is provided by the caller so we can update device 268ea0375afSGary R Hook * hwrng-related fields safely 269ea0375afSGary R Hook */ 270ea0375afSGary R Hook trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); 271ea0375afSGary R Hook if (!trng_value) { 272ea0375afSGary R Hook /* Zero is returned if not data is available or if a 273ea0375afSGary R Hook * bad-entropy error is present. Assume an error if 274ea0375afSGary R Hook * we exceed TRNG_RETRIES reads of zero. 275ea0375afSGary R Hook */ 276ea0375afSGary R Hook if (ccp->hwrng_retries++ > TRNG_RETRIES) 277ea0375afSGary R Hook return -EIO; 278ea0375afSGary R Hook 279ea0375afSGary R Hook return 0; 280ea0375afSGary R Hook } 281ea0375afSGary R Hook 282ea0375afSGary R Hook /* Reset the counter and save the rng value */ 283ea0375afSGary R Hook ccp->hwrng_retries = 0; 284ea0375afSGary R Hook memcpy(data, &trng_value, len); 285ea0375afSGary R Hook 286ea0375afSGary R Hook return len; 287ea0375afSGary R Hook } 288ea0375afSGary R Hook 289ea0375afSGary R Hook static int ccp_init(struct ccp_device *ccp) 290ea0375afSGary R Hook { 291ea0375afSGary R Hook struct device *dev = ccp->dev; 292ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q; 293ea0375afSGary R Hook struct dma_pool *dma_pool; 294ea0375afSGary R Hook char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 295ea0375afSGary R Hook unsigned int qmr, qim, i; 296ea0375afSGary R Hook int ret; 297ea0375afSGary R Hook 298ea0375afSGary R Hook /* Find available queues */ 299ea0375afSGary R Hook qim = 0; 300ea0375afSGary R Hook qmr = ioread32(ccp->io_regs + Q_MASK_REG); 301ea0375afSGary R Hook for (i = 0; i < MAX_HW_QUEUES; i++) { 302ea0375afSGary R Hook if (!(qmr & (1 << i))) 303ea0375afSGary R Hook continue; 304ea0375afSGary R Hook 305ea0375afSGary R Hook /* Allocate a dma pool for this queue */ 306ea0375afSGary R Hook snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 307ea0375afSGary R Hook ccp->name, i); 308ea0375afSGary R Hook dma_pool = dma_pool_create(dma_pool_name, dev, 309ea0375afSGary R Hook CCP_DMAPOOL_MAX_SIZE, 310ea0375afSGary R Hook CCP_DMAPOOL_ALIGN, 0); 311ea0375afSGary R Hook if (!dma_pool) { 312ea0375afSGary R Hook dev_err(dev, "unable to allocate dma pool\n"); 313ea0375afSGary R Hook ret = -ENOMEM; 314ea0375afSGary R Hook goto e_pool; 315ea0375afSGary R Hook } 316ea0375afSGary R Hook 317ea0375afSGary R Hook cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 318ea0375afSGary R Hook ccp->cmd_q_count++; 319ea0375afSGary R Hook 320ea0375afSGary R Hook cmd_q->ccp = ccp; 321ea0375afSGary R Hook cmd_q->id = i; 322ea0375afSGary R Hook cmd_q->dma_pool = dma_pool; 323ea0375afSGary R Hook 324ea0375afSGary R Hook /* Reserve 2 KSB regions for the queue */ 325ea0375afSGary R Hook cmd_q->ksb_key = KSB_START + ccp->ksb_start++; 326ea0375afSGary R Hook cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++; 327ea0375afSGary R Hook ccp->ksb_count -= 2; 328ea0375afSGary R Hook 329ea0375afSGary R Hook /* Preset some register values and masks that are queue 330ea0375afSGary R Hook * number dependent 331ea0375afSGary R Hook */ 332ea0375afSGary R Hook cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + 333ea0375afSGary R Hook (CMD_Q_STATUS_INCR * i); 334ea0375afSGary R Hook cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + 335ea0375afSGary R Hook (CMD_Q_STATUS_INCR * i); 336ea0375afSGary R Hook cmd_q->int_ok = 1 << (i * 2); 337ea0375afSGary R Hook cmd_q->int_err = 1 << ((i * 2) + 1); 338ea0375afSGary R Hook 339ea0375afSGary R Hook cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); 340ea0375afSGary R Hook 341ea0375afSGary R Hook init_waitqueue_head(&cmd_q->int_queue); 342ea0375afSGary R Hook 343ea0375afSGary R Hook /* Build queue interrupt mask (two interrupts per queue) */ 344ea0375afSGary R Hook qim |= cmd_q->int_ok | cmd_q->int_err; 345ea0375afSGary R Hook 346ea0375afSGary R Hook #ifdef CONFIG_ARM64 347ea0375afSGary R Hook /* For arm64 set the recommended queue cache settings */ 348ea0375afSGary R Hook iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + 349ea0375afSGary R Hook (CMD_Q_CACHE_INC * i)); 350ea0375afSGary R Hook #endif 351ea0375afSGary R Hook 352ea0375afSGary R Hook dev_dbg(dev, "queue #%u available\n", i); 353ea0375afSGary R Hook } 354ea0375afSGary R Hook if (ccp->cmd_q_count == 0) { 355ea0375afSGary R Hook dev_notice(dev, "no command queues available\n"); 356ea0375afSGary R Hook ret = -EIO; 357ea0375afSGary R Hook goto e_pool; 358ea0375afSGary R Hook } 359ea0375afSGary R Hook dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); 360ea0375afSGary R Hook 361ea0375afSGary R Hook /* Disable and clear interrupts until ready */ 362ea0375afSGary R Hook iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); 363ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 364ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 365ea0375afSGary R Hook 366ea0375afSGary R Hook ioread32(cmd_q->reg_int_status); 367ea0375afSGary R Hook ioread32(cmd_q->reg_status); 368ea0375afSGary R Hook } 369ea0375afSGary R Hook iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); 370ea0375afSGary R Hook 371ea0375afSGary R Hook /* Request an irq */ 372ea0375afSGary R Hook ret = ccp->get_irq(ccp); 373ea0375afSGary R Hook if (ret) { 374ea0375afSGary R Hook dev_err(dev, "unable to allocate an IRQ\n"); 375ea0375afSGary R Hook goto e_pool; 376ea0375afSGary R Hook } 377ea0375afSGary R Hook 378ea0375afSGary R Hook /* Initialize the queues used to wait for KSB space and suspend */ 379ea0375afSGary R Hook init_waitqueue_head(&ccp->ksb_queue); 380ea0375afSGary R Hook init_waitqueue_head(&ccp->suspend_queue); 381ea0375afSGary R Hook 382ea0375afSGary R Hook /* Create a kthread for each queue */ 383ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 384ea0375afSGary R Hook struct task_struct *kthread; 385ea0375afSGary R Hook 386ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 387ea0375afSGary R Hook 388ea0375afSGary R Hook kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, 389ea0375afSGary R Hook "%s-q%u", ccp->name, cmd_q->id); 390ea0375afSGary R Hook if (IS_ERR(kthread)) { 391ea0375afSGary R Hook dev_err(dev, "error creating queue thread (%ld)\n", 392ea0375afSGary R Hook PTR_ERR(kthread)); 393ea0375afSGary R Hook ret = PTR_ERR(kthread); 394ea0375afSGary R Hook goto e_kthread; 395ea0375afSGary R Hook } 396ea0375afSGary R Hook 397ea0375afSGary R Hook cmd_q->kthread = kthread; 398ea0375afSGary R Hook wake_up_process(kthread); 399ea0375afSGary R Hook } 400ea0375afSGary R Hook 401ea0375afSGary R Hook /* Register the RNG */ 402ea0375afSGary R Hook ccp->hwrng.name = ccp->rngname; 403ea0375afSGary R Hook ccp->hwrng.read = ccp_trng_read; 404ea0375afSGary R Hook ret = hwrng_register(&ccp->hwrng); 405ea0375afSGary R Hook if (ret) { 406ea0375afSGary R Hook dev_err(dev, "error registering hwrng (%d)\n", ret); 407ea0375afSGary R Hook goto e_kthread; 408ea0375afSGary R Hook } 409ea0375afSGary R Hook 41058ea8abfSGary R Hook /* Register the DMA engine support */ 41158ea8abfSGary R Hook ret = ccp_dmaengine_register(ccp); 41258ea8abfSGary R Hook if (ret) 41358ea8abfSGary R Hook goto e_hwrng; 41458ea8abfSGary R Hook 415ea0375afSGary R Hook ccp_add_device(ccp); 416ea0375afSGary R Hook 417ea0375afSGary R Hook /* Enable interrupts */ 418ea0375afSGary R Hook iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); 419ea0375afSGary R Hook 420ea0375afSGary R Hook return 0; 421ea0375afSGary R Hook 42258ea8abfSGary R Hook e_hwrng: 42358ea8abfSGary R Hook hwrng_unregister(&ccp->hwrng); 42458ea8abfSGary R Hook 425ea0375afSGary R Hook e_kthread: 426ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 427ea0375afSGary R Hook if (ccp->cmd_q[i].kthread) 428ea0375afSGary R Hook kthread_stop(ccp->cmd_q[i].kthread); 429ea0375afSGary R Hook 430ea0375afSGary R Hook ccp->free_irq(ccp); 431ea0375afSGary R Hook 432ea0375afSGary R Hook e_pool: 433ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 434ea0375afSGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool); 435ea0375afSGary R Hook 436ea0375afSGary R Hook return ret; 437ea0375afSGary R Hook } 438ea0375afSGary R Hook 439ea0375afSGary R Hook static void ccp_destroy(struct ccp_device *ccp) 440ea0375afSGary R Hook { 441ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q; 442ea0375afSGary R Hook struct ccp_cmd *cmd; 443ea0375afSGary R Hook unsigned int qim, i; 444ea0375afSGary R Hook 445ea0375afSGary R Hook /* Remove this device from the list of available units first */ 446ea0375afSGary R Hook ccp_del_device(ccp); 447ea0375afSGary R Hook 44858ea8abfSGary R Hook /* Unregister the DMA engine */ 44958ea8abfSGary R Hook ccp_dmaengine_unregister(ccp); 45058ea8abfSGary R Hook 451ea0375afSGary R Hook /* Unregister the RNG */ 452ea0375afSGary R Hook hwrng_unregister(&ccp->hwrng); 453ea0375afSGary R Hook 454ea0375afSGary R Hook /* Stop the queue kthreads */ 455ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 456ea0375afSGary R Hook if (ccp->cmd_q[i].kthread) 457ea0375afSGary R Hook kthread_stop(ccp->cmd_q[i].kthread); 458ea0375afSGary R Hook 459ea0375afSGary R Hook /* Build queue interrupt mask (two interrupt masks per queue) */ 460ea0375afSGary R Hook qim = 0; 461ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 462ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 463ea0375afSGary R Hook qim |= cmd_q->int_ok | cmd_q->int_err; 464ea0375afSGary R Hook } 465ea0375afSGary R Hook 466ea0375afSGary R Hook /* Disable and clear interrupts */ 467ea0375afSGary R Hook iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); 468ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 469ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 470ea0375afSGary R Hook 471ea0375afSGary R Hook ioread32(cmd_q->reg_int_status); 472ea0375afSGary R Hook ioread32(cmd_q->reg_status); 473ea0375afSGary R Hook } 474ea0375afSGary R Hook iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); 475ea0375afSGary R Hook 476ea0375afSGary R Hook ccp->free_irq(ccp); 477ea0375afSGary R Hook 478ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) 479ea0375afSGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool); 480ea0375afSGary R Hook 481ea0375afSGary R Hook /* Flush the cmd and backlog queue */ 482ea0375afSGary R Hook while (!list_empty(&ccp->cmd)) { 483ea0375afSGary R Hook /* Invoke the callback directly with an error code */ 484ea0375afSGary R Hook cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 485ea0375afSGary R Hook list_del(&cmd->entry); 486ea0375afSGary R Hook cmd->callback(cmd->data, -ENODEV); 487ea0375afSGary R Hook } 488ea0375afSGary R Hook while (!list_empty(&ccp->backlog)) { 489ea0375afSGary R Hook /* Invoke the callback directly with an error code */ 490ea0375afSGary R Hook cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 491ea0375afSGary R Hook list_del(&cmd->entry); 492ea0375afSGary R Hook cmd->callback(cmd->data, -ENODEV); 493ea0375afSGary R Hook } 494ea0375afSGary R Hook } 495ea0375afSGary R Hook 496ea0375afSGary R Hook static irqreturn_t ccp_irq_handler(int irq, void *data) 497ea0375afSGary R Hook { 498ea0375afSGary R Hook struct device *dev = data; 499ea0375afSGary R Hook struct ccp_device *ccp = dev_get_drvdata(dev); 500ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q; 501ea0375afSGary R Hook u32 q_int, status; 502ea0375afSGary R Hook unsigned int i; 503ea0375afSGary R Hook 504ea0375afSGary R Hook status = ioread32(ccp->io_regs + IRQ_STATUS_REG); 505ea0375afSGary R Hook 506ea0375afSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) { 507ea0375afSGary R Hook cmd_q = &ccp->cmd_q[i]; 508ea0375afSGary R Hook 509ea0375afSGary R Hook q_int = status & (cmd_q->int_ok | cmd_q->int_err); 510ea0375afSGary R Hook if (q_int) { 511ea0375afSGary R Hook cmd_q->int_status = status; 512ea0375afSGary R Hook cmd_q->q_status = ioread32(cmd_q->reg_status); 513ea0375afSGary R Hook cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 514ea0375afSGary R Hook 515ea0375afSGary R Hook /* On error, only save the first error value */ 516ea0375afSGary R Hook if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) 517ea0375afSGary R Hook cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 518ea0375afSGary R Hook 519ea0375afSGary R Hook cmd_q->int_rcvd = 1; 520ea0375afSGary R Hook 521ea0375afSGary R Hook /* Acknowledge the interrupt and wake the kthread */ 522ea0375afSGary R Hook iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); 523ea0375afSGary R Hook wake_up_interruptible(&cmd_q->int_queue); 524ea0375afSGary R Hook } 525ea0375afSGary R Hook } 526ea0375afSGary R Hook 527ea0375afSGary R Hook return IRQ_HANDLED; 528ea0375afSGary R Hook } 529ea0375afSGary R Hook 530bc197b2aSJulia Lawall static const struct ccp_actions ccp3_actions = { 531*a43eb985SGary R Hook .aes = ccp_perform_aes, 532*a43eb985SGary R Hook .xts_aes = ccp_perform_xts_aes, 533*a43eb985SGary R Hook .sha = ccp_perform_sha, 534*a43eb985SGary R Hook .rsa = ccp_perform_rsa, 535*a43eb985SGary R Hook .passthru = ccp_perform_passthru, 536*a43eb985SGary R Hook .ecc = ccp_perform_ecc, 537ea0375afSGary R Hook .init = ccp_init, 538ea0375afSGary R Hook .destroy = ccp_destroy, 539ea0375afSGary R Hook .irqhandler = ccp_irq_handler, 540ea0375afSGary R Hook }; 541ea0375afSGary R Hook 542ea0375afSGary R Hook struct ccp_vdata ccpv3 = { 543ea0375afSGary R Hook .version = CCP_VERSION(3, 0), 544ea0375afSGary R Hook .perform = &ccp3_actions, 545fba8855cSGary R Hook .bar = 2, 546fba8855cSGary R Hook .offset = 0x20000, 547ea0375afSGary R Hook }; 548