1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
24b394a23SGary R Hook /*
34b394a23SGary R Hook * AMD Cryptographic Coprocessor (CCP) driver
44b394a23SGary R Hook *
5c4a89279SHook, Gary * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
64b394a23SGary R Hook *
74b394a23SGary R Hook * Author: Gary R Hook <gary.hook@amd.com>
84b394a23SGary R Hook */
94b394a23SGary R Hook
104b394a23SGary R Hook #include <linux/kernel.h>
114b394a23SGary R Hook #include <linux/kthread.h>
124b394a23SGary R Hook #include <linux/dma-mapping.h>
134b394a23SGary R Hook #include <linux/interrupt.h>
144b394a23SGary R Hook #include <linux/compiler.h>
154b394a23SGary R Hook #include <linux/ccp.h>
164b394a23SGary R Hook
174b394a23SGary R Hook #include "ccp-dev.h"
184b394a23SGary R Hook
19103600abSGary R Hook /* Allocate the requested number of contiguous LSB slots
20103600abSGary R Hook * from the LSB bitmap. Look in the private range for this
21103600abSGary R Hook * queue first; failing that, check the public area.
22103600abSGary R Hook * If no space is available, wait around.
23103600abSGary R Hook * Return: first slot number
24103600abSGary R Hook */
ccp_lsb_alloc(struct ccp_cmd_queue * cmd_q,unsigned int count)254b394a23SGary R Hook static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
264b394a23SGary R Hook {
274b394a23SGary R Hook struct ccp_device *ccp;
284b394a23SGary R Hook int start;
294b394a23SGary R Hook
304b394a23SGary R Hook /* First look at the map for the queue */
314b394a23SGary R Hook if (cmd_q->lsb >= 0) {
324b394a23SGary R Hook start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
334b394a23SGary R Hook LSB_SIZE,
344b394a23SGary R Hook 0, count, 0);
354b394a23SGary R Hook if (start < LSB_SIZE) {
364b394a23SGary R Hook bitmap_set(cmd_q->lsbmap, start, count);
374b394a23SGary R Hook return start + cmd_q->lsb * LSB_SIZE;
384b394a23SGary R Hook }
394b394a23SGary R Hook }
404b394a23SGary R Hook
414b394a23SGary R Hook /* No joy; try to get an entry from the shared blocks */
424b394a23SGary R Hook ccp = cmd_q->ccp;
434b394a23SGary R Hook for (;;) {
444b394a23SGary R Hook mutex_lock(&ccp->sb_mutex);
454b394a23SGary R Hook
464b394a23SGary R Hook start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
474b394a23SGary R Hook MAX_LSB_CNT * LSB_SIZE,
484b394a23SGary R Hook 0,
494b394a23SGary R Hook count, 0);
504b394a23SGary R Hook if (start <= MAX_LSB_CNT * LSB_SIZE) {
514b394a23SGary R Hook bitmap_set(ccp->lsbmap, start, count);
524b394a23SGary R Hook
534b394a23SGary R Hook mutex_unlock(&ccp->sb_mutex);
54103600abSGary R Hook return start;
554b394a23SGary R Hook }
564b394a23SGary R Hook
574b394a23SGary R Hook ccp->sb_avail = 0;
584b394a23SGary R Hook
594b394a23SGary R Hook mutex_unlock(&ccp->sb_mutex);
604b394a23SGary R Hook
614b394a23SGary R Hook /* Wait for KSB entries to become available */
624b394a23SGary R Hook if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
634b394a23SGary R Hook return 0;
644b394a23SGary R Hook }
654b394a23SGary R Hook }
664b394a23SGary R Hook
67103600abSGary R Hook /* Free a number of LSB slots from the bitmap, starting at
68103600abSGary R Hook * the indicated starting slot number.
69103600abSGary R Hook */
ccp_lsb_free(struct ccp_cmd_queue * cmd_q,unsigned int start,unsigned int count)704b394a23SGary R Hook static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
714b394a23SGary R Hook unsigned int count)
724b394a23SGary R Hook {
734b394a23SGary R Hook if (!start)
744b394a23SGary R Hook return;
754b394a23SGary R Hook
76103600abSGary R Hook if (cmd_q->lsb == start) {
774b394a23SGary R Hook /* An entry from the private LSB */
78103600abSGary R Hook bitmap_clear(cmd_q->lsbmap, start, count);
794b394a23SGary R Hook } else {
804b394a23SGary R Hook /* From the shared LSBs */
814b394a23SGary R Hook struct ccp_device *ccp = cmd_q->ccp;
824b394a23SGary R Hook
834b394a23SGary R Hook mutex_lock(&ccp->sb_mutex);
844b394a23SGary R Hook bitmap_clear(ccp->lsbmap, start, count);
854b394a23SGary R Hook ccp->sb_avail = 1;
864b394a23SGary R Hook mutex_unlock(&ccp->sb_mutex);
874b394a23SGary R Hook wake_up_interruptible_all(&ccp->sb_queue);
884b394a23SGary R Hook }
894b394a23SGary R Hook }
904b394a23SGary R Hook
914b394a23SGary R Hook /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
924b394a23SGary R Hook union ccp_function {
934b394a23SGary R Hook struct {
944b394a23SGary R Hook u16 size:7;
954b394a23SGary R Hook u16 encrypt:1;
964b394a23SGary R Hook u16 mode:5;
974b394a23SGary R Hook u16 type:2;
984b394a23SGary R Hook } aes;
994b394a23SGary R Hook struct {
1004b394a23SGary R Hook u16 size:7;
1014b394a23SGary R Hook u16 encrypt:1;
1024b394a23SGary R Hook u16 rsvd:5;
1034b394a23SGary R Hook u16 type:2;
1044b394a23SGary R Hook } aes_xts;
1054b394a23SGary R Hook struct {
106990672d4SGary R Hook u16 size:7;
107990672d4SGary R Hook u16 encrypt:1;
108990672d4SGary R Hook u16 mode:5;
109990672d4SGary R Hook u16 type:2;
110990672d4SGary R Hook } des3;
111990672d4SGary R Hook struct {
1124b394a23SGary R Hook u16 rsvd1:10;
1134b394a23SGary R Hook u16 type:4;
1144b394a23SGary R Hook u16 rsvd2:1;
1154b394a23SGary R Hook } sha;
1164b394a23SGary R Hook struct {
1174b394a23SGary R Hook u16 mode:3;
1184b394a23SGary R Hook u16 size:12;
1194b394a23SGary R Hook } rsa;
1204b394a23SGary R Hook struct {
1214b394a23SGary R Hook u16 byteswap:2;
1224b394a23SGary R Hook u16 bitwise:3;
1234b394a23SGary R Hook u16 reflect:2;
1244b394a23SGary R Hook u16 rsvd:8;
1254b394a23SGary R Hook } pt;
1264b394a23SGary R Hook struct {
1274b394a23SGary R Hook u16 rsvd:13;
1284b394a23SGary R Hook } zlib;
1294b394a23SGary R Hook struct {
1304b394a23SGary R Hook u16 size:10;
1314b394a23SGary R Hook u16 type:2;
1324b394a23SGary R Hook u16 mode:3;
1334b394a23SGary R Hook } ecc;
1344b394a23SGary R Hook u16 raw;
1354b394a23SGary R Hook };
1364b394a23SGary R Hook
1374b394a23SGary R Hook #define CCP_AES_SIZE(p) ((p)->aes.size)
1384b394a23SGary R Hook #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
1394b394a23SGary R Hook #define CCP_AES_MODE(p) ((p)->aes.mode)
1404b394a23SGary R Hook #define CCP_AES_TYPE(p) ((p)->aes.type)
1414b394a23SGary R Hook #define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
142e652399eSGary R Hook #define CCP_XTS_TYPE(p) ((p)->aes_xts.type)
1434b394a23SGary R Hook #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
144990672d4SGary R Hook #define CCP_DES3_SIZE(p) ((p)->des3.size)
145990672d4SGary R Hook #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt)
146990672d4SGary R Hook #define CCP_DES3_MODE(p) ((p)->des3.mode)
147990672d4SGary R Hook #define CCP_DES3_TYPE(p) ((p)->des3.type)
1484b394a23SGary R Hook #define CCP_SHA_TYPE(p) ((p)->sha.type)
1494b394a23SGary R Hook #define CCP_RSA_SIZE(p) ((p)->rsa.size)
1504b394a23SGary R Hook #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
1514b394a23SGary R Hook #define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
1524b394a23SGary R Hook #define CCP_ECC_MODE(p) ((p)->ecc.mode)
1534b394a23SGary R Hook #define CCP_ECC_AFFINE(p) ((p)->ecc.one)
1544b394a23SGary R Hook
1554b394a23SGary R Hook /* Word 0 */
1564b394a23SGary R Hook #define CCP5_CMD_DW0(p) ((p)->dw0)
1574b394a23SGary R Hook #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
1584b394a23SGary R Hook #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
1594b394a23SGary R Hook #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
1604b394a23SGary R Hook #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
1614b394a23SGary R Hook #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
1624b394a23SGary R Hook #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
1634b394a23SGary R Hook #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
1644b394a23SGary R Hook
1654b394a23SGary R Hook /* Word 1 */
1664b394a23SGary R Hook #define CCP5_CMD_DW1(p) ((p)->length)
1674b394a23SGary R Hook #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
1684b394a23SGary R Hook
1694b394a23SGary R Hook /* Word 2 */
1704b394a23SGary R Hook #define CCP5_CMD_DW2(p) ((p)->src_lo)
1714b394a23SGary R Hook #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
1724b394a23SGary R Hook
1734b394a23SGary R Hook /* Word 3 */
1744b394a23SGary R Hook #define CCP5_CMD_DW3(p) ((p)->dw3)
1754b394a23SGary R Hook #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
1764b394a23SGary R Hook #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
1774b394a23SGary R Hook #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
1784b394a23SGary R Hook #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
1794b394a23SGary R Hook
1804b394a23SGary R Hook /* Words 4/5 */
1814b394a23SGary R Hook #define CCP5_CMD_DW4(p) ((p)->dw4)
1824b394a23SGary R Hook #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
1834b394a23SGary R Hook #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
1844b394a23SGary R Hook #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
1854b394a23SGary R Hook #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
1864b394a23SGary R Hook #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
1874b394a23SGary R Hook #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
1884b394a23SGary R Hook #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
1894b394a23SGary R Hook
1904b394a23SGary R Hook /* Word 6/7 */
1914b394a23SGary R Hook #define CCP5_CMD_DW6(p) ((p)->key_lo)
1924b394a23SGary R Hook #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
1934b394a23SGary R Hook #define CCP5_CMD_DW7(p) ((p)->dw7)
1944b394a23SGary R Hook #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
1954b394a23SGary R Hook #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
1964b394a23SGary R Hook
low_address(unsigned long addr)1974b394a23SGary R Hook static inline u32 low_address(unsigned long addr)
1984b394a23SGary R Hook {
1994b394a23SGary R Hook return (u64)addr & 0x0ffffffff;
2004b394a23SGary R Hook }
2014b394a23SGary R Hook
high_address(unsigned long addr)2024b394a23SGary R Hook static inline u32 high_address(unsigned long addr)
2034b394a23SGary R Hook {
2044b394a23SGary R Hook return ((u64)addr >> 32) & 0x00000ffff;
2054b394a23SGary R Hook }
2064b394a23SGary R Hook
ccp5_get_free_slots(struct ccp_cmd_queue * cmd_q)2074b394a23SGary R Hook static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
2084b394a23SGary R Hook {
2094b394a23SGary R Hook unsigned int head_idx, n;
2104b394a23SGary R Hook u32 head_lo, queue_start;
2114b394a23SGary R Hook
2124b394a23SGary R Hook queue_start = low_address(cmd_q->qdma_tail);
2134b394a23SGary R Hook head_lo = ioread32(cmd_q->reg_head_lo);
2144b394a23SGary R Hook head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
2154b394a23SGary R Hook
2164b394a23SGary R Hook n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
2174b394a23SGary R Hook
2184b394a23SGary R Hook return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
2194b394a23SGary R Hook }
2204b394a23SGary R Hook
ccp5_do_cmd(struct ccp5_desc * desc,struct ccp_cmd_queue * cmd_q)2214b394a23SGary R Hook static int ccp5_do_cmd(struct ccp5_desc *desc,
2224b394a23SGary R Hook struct ccp_cmd_queue *cmd_q)
2234b394a23SGary R Hook {
224d9dd5ef3SHerbert Xu __le32 *mP;
225d9dd5ef3SHerbert Xu u32 *dP;
2264b394a23SGary R Hook u32 tail;
2274b394a23SGary R Hook int i;
2284b394a23SGary R Hook int ret = 0;
2294b394a23SGary R Hook
2303cdbe346SGary R Hook cmd_q->total_ops++;
2313cdbe346SGary R Hook
2324b394a23SGary R Hook if (CCP5_CMD_SOC(desc)) {
2334b394a23SGary R Hook CCP5_CMD_IOC(desc) = 1;
2344b394a23SGary R Hook CCP5_CMD_SOC(desc) = 0;
2354b394a23SGary R Hook }
2364b394a23SGary R Hook mutex_lock(&cmd_q->q_mutex);
2374b394a23SGary R Hook
238d9dd5ef3SHerbert Xu mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx];
239d9dd5ef3SHerbert Xu dP = (u32 *)desc;
2404b394a23SGary R Hook for (i = 0; i < 8; i++)
2414b394a23SGary R Hook mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
2424b394a23SGary R Hook
2434b394a23SGary R Hook cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2444b394a23SGary R Hook
2454b394a23SGary R Hook /* The data used by this command must be flushed to memory */
2464b394a23SGary R Hook wmb();
2474b394a23SGary R Hook
2484b394a23SGary R Hook /* Write the new tail address back to the queue register */
2494b394a23SGary R Hook tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
2504b394a23SGary R Hook iowrite32(tail, cmd_q->reg_tail_lo);
2514b394a23SGary R Hook
2524b394a23SGary R Hook /* Turn the queue back on using our cached control register */
2534b394a23SGary R Hook iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
2544b394a23SGary R Hook mutex_unlock(&cmd_q->q_mutex);
2554b394a23SGary R Hook
2564b394a23SGary R Hook if (CCP5_CMD_IOC(desc)) {
2574b394a23SGary R Hook /* Wait for the job to complete */
2584b394a23SGary R Hook ret = wait_event_interruptible(cmd_q->int_queue,
2594b394a23SGary R Hook cmd_q->int_rcvd);
2604b394a23SGary R Hook if (ret || cmd_q->cmd_error) {
2614cdf101eSGary R Hook /* Log the error and flush the queue by
2624cdf101eSGary R Hook * moving the head pointer
2634cdf101eSGary R Hook */
26481422badSGary R Hook if (cmd_q->cmd_error)
26581422badSGary R Hook ccp_log_error(cmd_q->ccp,
26681422badSGary R Hook cmd_q->cmd_error);
2674cdf101eSGary R Hook iowrite32(tail, cmd_q->reg_head_lo);
2684b394a23SGary R Hook if (!ret)
2694b394a23SGary R Hook ret = -EIO;
2704b394a23SGary R Hook }
2714b394a23SGary R Hook cmd_q->int_rcvd = 0;
2724b394a23SGary R Hook }
2734b394a23SGary R Hook
2744cdf101eSGary R Hook return ret;
2754b394a23SGary R Hook }
2764b394a23SGary R Hook
ccp5_perform_aes(struct ccp_op * op)2774b394a23SGary R Hook static int ccp5_perform_aes(struct ccp_op *op)
2784b394a23SGary R Hook {
2794b394a23SGary R Hook struct ccp5_desc desc;
2804b394a23SGary R Hook union ccp_function function;
2814b394a23SGary R Hook u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
2824b394a23SGary R Hook
2833cdbe346SGary R Hook op->cmd_q->total_aes_ops++;
2843cdbe346SGary R Hook
2854b394a23SGary R Hook /* Zero out all the fields of the command desc */
2864b394a23SGary R Hook memset(&desc, 0, Q_DESC_SIZE);
2874b394a23SGary R Hook
2884b394a23SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
2894b394a23SGary R Hook
2904b394a23SGary R Hook CCP5_CMD_SOC(&desc) = op->soc;
2914b394a23SGary R Hook CCP5_CMD_IOC(&desc) = 1;
2924b394a23SGary R Hook CCP5_CMD_INIT(&desc) = op->init;
2934b394a23SGary R Hook CCP5_CMD_EOM(&desc) = op->eom;
2944b394a23SGary R Hook CCP5_CMD_PROT(&desc) = 0;
2954b394a23SGary R Hook
2964b394a23SGary R Hook function.raw = 0;
2974b394a23SGary R Hook CCP_AES_ENCRYPT(&function) = op->u.aes.action;
2984b394a23SGary R Hook CCP_AES_MODE(&function) = op->u.aes.mode;
2994b394a23SGary R Hook CCP_AES_TYPE(&function) = op->u.aes.type;
300f7cc02b3SGary R Hook CCP_AES_SIZE(&function) = op->u.aes.size;
3014b394a23SGary R Hook
3024b394a23SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
3034b394a23SGary R Hook
3044b394a23SGary R Hook CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
3054b394a23SGary R Hook
3064b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
3074b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
3084b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
3094b394a23SGary R Hook
3104b394a23SGary R Hook CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
3114b394a23SGary R Hook CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
3124b394a23SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
3134b394a23SGary R Hook
3144b394a23SGary R Hook CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
3154b394a23SGary R Hook CCP5_CMD_KEY_HI(&desc) = 0;
3164b394a23SGary R Hook CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
3174b394a23SGary R Hook CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
3184b394a23SGary R Hook
3194b394a23SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
3204b394a23SGary R Hook }
3214b394a23SGary R Hook
ccp5_perform_xts_aes(struct ccp_op * op)3224b394a23SGary R Hook static int ccp5_perform_xts_aes(struct ccp_op *op)
3234b394a23SGary R Hook {
3244b394a23SGary R Hook struct ccp5_desc desc;
3254b394a23SGary R Hook union ccp_function function;
3264b394a23SGary R Hook u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
3274b394a23SGary R Hook
3283cdbe346SGary R Hook op->cmd_q->total_xts_aes_ops++;
3293cdbe346SGary R Hook
3304b394a23SGary R Hook /* Zero out all the fields of the command desc */
3314b394a23SGary R Hook memset(&desc, 0, Q_DESC_SIZE);
3324b394a23SGary R Hook
3334b394a23SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
3344b394a23SGary R Hook
3354b394a23SGary R Hook CCP5_CMD_SOC(&desc) = op->soc;
3364b394a23SGary R Hook CCP5_CMD_IOC(&desc) = 1;
3374b394a23SGary R Hook CCP5_CMD_INIT(&desc) = op->init;
3384b394a23SGary R Hook CCP5_CMD_EOM(&desc) = op->eom;
3394b394a23SGary R Hook CCP5_CMD_PROT(&desc) = 0;
3404b394a23SGary R Hook
3414b394a23SGary R Hook function.raw = 0;
342e652399eSGary R Hook CCP_XTS_TYPE(&function) = op->u.xts.type;
3434b394a23SGary R Hook CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
3444b394a23SGary R Hook CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
3454b394a23SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
3464b394a23SGary R Hook
3474b394a23SGary R Hook CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
3484b394a23SGary R Hook
3494b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
3504b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
3514b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
3524b394a23SGary R Hook
3534b394a23SGary R Hook CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
3544b394a23SGary R Hook CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
3554b394a23SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
3564b394a23SGary R Hook
3574b394a23SGary R Hook CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
3584b394a23SGary R Hook CCP5_CMD_KEY_HI(&desc) = 0;
3594b394a23SGary R Hook CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
3604b394a23SGary R Hook CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
3614b394a23SGary R Hook
3624b394a23SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
3634b394a23SGary R Hook }
3644b394a23SGary R Hook
ccp5_perform_sha(struct ccp_op * op)3654b394a23SGary R Hook static int ccp5_perform_sha(struct ccp_op *op)
3664b394a23SGary R Hook {
3674b394a23SGary R Hook struct ccp5_desc desc;
3684b394a23SGary R Hook union ccp_function function;
3694b394a23SGary R Hook
3703cdbe346SGary R Hook op->cmd_q->total_sha_ops++;
3713cdbe346SGary R Hook
3724b394a23SGary R Hook /* Zero out all the fields of the command desc */
3734b394a23SGary R Hook memset(&desc, 0, Q_DESC_SIZE);
3744b394a23SGary R Hook
3754b394a23SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
3764b394a23SGary R Hook
3774b394a23SGary R Hook CCP5_CMD_SOC(&desc) = op->soc;
3784b394a23SGary R Hook CCP5_CMD_IOC(&desc) = 1;
3794b394a23SGary R Hook CCP5_CMD_INIT(&desc) = 1;
3804b394a23SGary R Hook CCP5_CMD_EOM(&desc) = op->eom;
3814b394a23SGary R Hook CCP5_CMD_PROT(&desc) = 0;
3824b394a23SGary R Hook
3834b394a23SGary R Hook function.raw = 0;
3844b394a23SGary R Hook CCP_SHA_TYPE(&function) = op->u.sha.type;
3854b394a23SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
3864b394a23SGary R Hook
3874b394a23SGary R Hook CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
3884b394a23SGary R Hook
3894b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
3904b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
3914b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
3924b394a23SGary R Hook
3934b394a23SGary R Hook CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
3944b394a23SGary R Hook
3954b394a23SGary R Hook if (op->eom) {
3964b394a23SGary R Hook CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
3974b394a23SGary R Hook CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
3984b394a23SGary R Hook } else {
3994b394a23SGary R Hook CCP5_CMD_SHA_LO(&desc) = 0;
4004b394a23SGary R Hook CCP5_CMD_SHA_HI(&desc) = 0;
4014b394a23SGary R Hook }
4024b394a23SGary R Hook
4034b394a23SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
4044b394a23SGary R Hook }
4054b394a23SGary R Hook
ccp5_perform_des3(struct ccp_op * op)406990672d4SGary R Hook static int ccp5_perform_des3(struct ccp_op *op)
407990672d4SGary R Hook {
408990672d4SGary R Hook struct ccp5_desc desc;
409990672d4SGary R Hook union ccp_function function;
410990672d4SGary R Hook u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
411990672d4SGary R Hook
4123cdbe346SGary R Hook op->cmd_q->total_3des_ops++;
4133cdbe346SGary R Hook
414990672d4SGary R Hook /* Zero out all the fields of the command desc */
415990672d4SGary R Hook memset(&desc, 0, sizeof(struct ccp5_desc));
416990672d4SGary R Hook
417990672d4SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3;
418990672d4SGary R Hook
419990672d4SGary R Hook CCP5_CMD_SOC(&desc) = op->soc;
420990672d4SGary R Hook CCP5_CMD_IOC(&desc) = 1;
421990672d4SGary R Hook CCP5_CMD_INIT(&desc) = op->init;
422990672d4SGary R Hook CCP5_CMD_EOM(&desc) = op->eom;
423990672d4SGary R Hook CCP5_CMD_PROT(&desc) = 0;
424990672d4SGary R Hook
425990672d4SGary R Hook function.raw = 0;
426990672d4SGary R Hook CCP_DES3_ENCRYPT(&function) = op->u.des3.action;
427990672d4SGary R Hook CCP_DES3_MODE(&function) = op->u.des3.mode;
428990672d4SGary R Hook CCP_DES3_TYPE(&function) = op->u.des3.type;
42951de7dd0SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
430990672d4SGary R Hook
43151de7dd0SGary R Hook CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
432990672d4SGary R Hook
43351de7dd0SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
43451de7dd0SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
43551de7dd0SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
436990672d4SGary R Hook
43751de7dd0SGary R Hook CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
43851de7dd0SGary R Hook CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
43951de7dd0SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
440990672d4SGary R Hook
44151de7dd0SGary R Hook CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
442990672d4SGary R Hook CCP5_CMD_KEY_HI(&desc) = 0;
44351de7dd0SGary R Hook CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
44451de7dd0SGary R Hook CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
445990672d4SGary R Hook
446990672d4SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
447990672d4SGary R Hook }
448990672d4SGary R Hook
ccp5_perform_rsa(struct ccp_op * op)4494b394a23SGary R Hook static int ccp5_perform_rsa(struct ccp_op *op)
4504b394a23SGary R Hook {
4514b394a23SGary R Hook struct ccp5_desc desc;
4524b394a23SGary R Hook union ccp_function function;
4534b394a23SGary R Hook
4543cdbe346SGary R Hook op->cmd_q->total_rsa_ops++;
4553cdbe346SGary R Hook
4564b394a23SGary R Hook /* Zero out all the fields of the command desc */
4574b394a23SGary R Hook memset(&desc, 0, Q_DESC_SIZE);
4584b394a23SGary R Hook
4594b394a23SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
4604b394a23SGary R Hook
4614b394a23SGary R Hook CCP5_CMD_SOC(&desc) = op->soc;
4624b394a23SGary R Hook CCP5_CMD_IOC(&desc) = 1;
4634b394a23SGary R Hook CCP5_CMD_INIT(&desc) = 0;
4644b394a23SGary R Hook CCP5_CMD_EOM(&desc) = 1;
4654b394a23SGary R Hook CCP5_CMD_PROT(&desc) = 0;
4664b394a23SGary R Hook
4674b394a23SGary R Hook function.raw = 0;
4686ba46c7dSGary R Hook CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
4694b394a23SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
4704b394a23SGary R Hook
4714b394a23SGary R Hook CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
4724b394a23SGary R Hook
4734b394a23SGary R Hook /* Source is from external memory */
4744b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
4754b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
4764b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
4774b394a23SGary R Hook
4784b394a23SGary R Hook /* Destination is in external memory */
4794b394a23SGary R Hook CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
4804b394a23SGary R Hook CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
4814b394a23SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
4824b394a23SGary R Hook
4836ba46c7dSGary R Hook /* Key (Exponent) is in external memory */
4846ba46c7dSGary R Hook CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
4856ba46c7dSGary R Hook CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
4866ba46c7dSGary R Hook CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
4874b394a23SGary R Hook
4884b394a23SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
4894b394a23SGary R Hook }
4904b394a23SGary R Hook
ccp5_perform_passthru(struct ccp_op * op)4914b394a23SGary R Hook static int ccp5_perform_passthru(struct ccp_op *op)
4924b394a23SGary R Hook {
4934b394a23SGary R Hook struct ccp5_desc desc;
4944b394a23SGary R Hook union ccp_function function;
4954b394a23SGary R Hook struct ccp_dma_info *saddr = &op->src.u.dma;
4964b394a23SGary R Hook struct ccp_dma_info *daddr = &op->dst.u.dma;
4974b394a23SGary R Hook
498990672d4SGary R Hook
4993cdbe346SGary R Hook op->cmd_q->total_pt_ops++;
5003cdbe346SGary R Hook
5014b394a23SGary R Hook memset(&desc, 0, Q_DESC_SIZE);
5024b394a23SGary R Hook
5034b394a23SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
5044b394a23SGary R Hook
5054b394a23SGary R Hook CCP5_CMD_SOC(&desc) = 0;
5064b394a23SGary R Hook CCP5_CMD_IOC(&desc) = 1;
5074b394a23SGary R Hook CCP5_CMD_INIT(&desc) = 0;
5084b394a23SGary R Hook CCP5_CMD_EOM(&desc) = op->eom;
5094b394a23SGary R Hook CCP5_CMD_PROT(&desc) = 0;
5104b394a23SGary R Hook
5114b394a23SGary R Hook function.raw = 0;
5124b394a23SGary R Hook CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
5134b394a23SGary R Hook CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
5144b394a23SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
5154b394a23SGary R Hook
5164b394a23SGary R Hook /* Length of source data is always 256 bytes */
5174b394a23SGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM)
5184b394a23SGary R Hook CCP5_CMD_LEN(&desc) = saddr->length;
5194b394a23SGary R Hook else
5204b394a23SGary R Hook CCP5_CMD_LEN(&desc) = daddr->length;
5214b394a23SGary R Hook
5224b394a23SGary R Hook if (op->src.type == CCP_MEMTYPE_SYSTEM) {
5234b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
5244b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
5254b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
5264b394a23SGary R Hook
5274b394a23SGary R Hook if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
5284b394a23SGary R Hook CCP5_CMD_LSB_ID(&desc) = op->sb_key;
5294b394a23SGary R Hook } else {
5304b394a23SGary R Hook u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
5314b394a23SGary R Hook
5324b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
5334b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = 0;
5344b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
5354b394a23SGary R Hook }
5364b394a23SGary R Hook
5374b394a23SGary R Hook if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
5384b394a23SGary R Hook CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
5394b394a23SGary R Hook CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
5404b394a23SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
5414b394a23SGary R Hook } else {
5424b394a23SGary R Hook u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
5434b394a23SGary R Hook
5444b394a23SGary R Hook CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
5454b394a23SGary R Hook CCP5_CMD_DST_HI(&desc) = 0;
5464b394a23SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
5474b394a23SGary R Hook }
5484b394a23SGary R Hook
5494b394a23SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
5504b394a23SGary R Hook }
5514b394a23SGary R Hook
ccp5_perform_ecc(struct ccp_op * op)5524b394a23SGary R Hook static int ccp5_perform_ecc(struct ccp_op *op)
5534b394a23SGary R Hook {
5544b394a23SGary R Hook struct ccp5_desc desc;
5554b394a23SGary R Hook union ccp_function function;
5564b394a23SGary R Hook
5573cdbe346SGary R Hook op->cmd_q->total_ecc_ops++;
5583cdbe346SGary R Hook
5594b394a23SGary R Hook /* Zero out all the fields of the command desc */
5604b394a23SGary R Hook memset(&desc, 0, Q_DESC_SIZE);
5614b394a23SGary R Hook
5624b394a23SGary R Hook CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
5634b394a23SGary R Hook
5644b394a23SGary R Hook CCP5_CMD_SOC(&desc) = 0;
5654b394a23SGary R Hook CCP5_CMD_IOC(&desc) = 1;
5664b394a23SGary R Hook CCP5_CMD_INIT(&desc) = 0;
5674b394a23SGary R Hook CCP5_CMD_EOM(&desc) = 1;
5684b394a23SGary R Hook CCP5_CMD_PROT(&desc) = 0;
5694b394a23SGary R Hook
5704b394a23SGary R Hook function.raw = 0;
5714b394a23SGary R Hook function.ecc.mode = op->u.ecc.function;
5724b394a23SGary R Hook CCP5_CMD_FUNCTION(&desc) = function.raw;
5734b394a23SGary R Hook
5744b394a23SGary R Hook CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
5754b394a23SGary R Hook
5764b394a23SGary R Hook CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
5774b394a23SGary R Hook CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
5784b394a23SGary R Hook CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
5794b394a23SGary R Hook
5804b394a23SGary R Hook CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
5814b394a23SGary R Hook CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
5824b394a23SGary R Hook CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
5834b394a23SGary R Hook
5844b394a23SGary R Hook return ccp5_do_cmd(&desc, op->cmd_q);
5854b394a23SGary R Hook }
5864b394a23SGary R Hook
ccp_find_lsb_regions(struct ccp_cmd_queue * cmd_q,u64 status)5874b394a23SGary R Hook static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
5884b394a23SGary R Hook {
5894b394a23SGary R Hook int q_mask = 1 << cmd_q->id;
5904b394a23SGary R Hook int queues = 0;
5914b394a23SGary R Hook int j;
5924b394a23SGary R Hook
5934b394a23SGary R Hook /* Build a bit mask to know which LSBs this queue has access to.
5944b394a23SGary R Hook * Don't bother with segment 0 as it has special privileges.
5954b394a23SGary R Hook */
5964b394a23SGary R Hook for (j = 1; j < MAX_LSB_CNT; j++) {
5974b394a23SGary R Hook if (status & q_mask)
5984b394a23SGary R Hook bitmap_set(cmd_q->lsbmask, j, 1);
5994b394a23SGary R Hook status >>= LSB_REGION_WIDTH;
6004b394a23SGary R Hook }
6014b394a23SGary R Hook queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
602a60496a0SGary R Hook dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
6034b394a23SGary R Hook cmd_q->id, queues);
6044b394a23SGary R Hook
6054b394a23SGary R Hook return queues ? 0 : -EINVAL;
6064b394a23SGary R Hook }
6074b394a23SGary R Hook
ccp_find_and_assign_lsb_to_q(struct ccp_device * ccp,int lsb_cnt,int n_lsbs,unsigned long * lsb_pub)6084b394a23SGary R Hook static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
6094b394a23SGary R Hook int lsb_cnt, int n_lsbs,
6104b394a23SGary R Hook unsigned long *lsb_pub)
6114b394a23SGary R Hook {
6124b394a23SGary R Hook DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
6134b394a23SGary R Hook int bitno;
6144b394a23SGary R Hook int qlsb_wgt;
6154b394a23SGary R Hook int i;
6164b394a23SGary R Hook
6174b394a23SGary R Hook /* For each queue:
6184b394a23SGary R Hook * If the count of potential LSBs available to a queue matches the
6194b394a23SGary R Hook * ordinal given to us in lsb_cnt:
6204b394a23SGary R Hook * Copy the mask of possible LSBs for this queue into "qlsb";
6214b394a23SGary R Hook * For each bit in qlsb, see if the corresponding bit in the
6224b394a23SGary R Hook * aggregation mask is set; if so, we have a match.
6234b394a23SGary R Hook * If we have a match, clear the bit in the aggregation to
6244b394a23SGary R Hook * mark it as no longer available.
6254b394a23SGary R Hook * If there is no match, clear the bit in qlsb and keep looking.
6264b394a23SGary R Hook */
6274b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
6284b394a23SGary R Hook struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
6294b394a23SGary R Hook
6304b394a23SGary R Hook qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
6314b394a23SGary R Hook
6324b394a23SGary R Hook if (qlsb_wgt == lsb_cnt) {
6334b394a23SGary R Hook bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
6344b394a23SGary R Hook
6354b394a23SGary R Hook bitno = find_first_bit(qlsb, MAX_LSB_CNT);
6364b394a23SGary R Hook while (bitno < MAX_LSB_CNT) {
6374b394a23SGary R Hook if (test_bit(bitno, lsb_pub)) {
6384b394a23SGary R Hook /* We found an available LSB
6394b394a23SGary R Hook * that this queue can access
6404b394a23SGary R Hook */
6414b394a23SGary R Hook cmd_q->lsb = bitno;
6424b394a23SGary R Hook bitmap_clear(lsb_pub, bitno, 1);
643a60496a0SGary R Hook dev_dbg(ccp->dev,
6444b394a23SGary R Hook "Queue %d gets LSB %d\n",
6454b394a23SGary R Hook i, bitno);
6464b394a23SGary R Hook break;
6474b394a23SGary R Hook }
6484b394a23SGary R Hook bitmap_clear(qlsb, bitno, 1);
6494b394a23SGary R Hook bitno = find_first_bit(qlsb, MAX_LSB_CNT);
6504b394a23SGary R Hook }
6514b394a23SGary R Hook if (bitno >= MAX_LSB_CNT)
6524b394a23SGary R Hook return -EINVAL;
6534b394a23SGary R Hook n_lsbs--;
6544b394a23SGary R Hook }
6554b394a23SGary R Hook }
6564b394a23SGary R Hook return n_lsbs;
6574b394a23SGary R Hook }
6584b394a23SGary R Hook
6594b394a23SGary R Hook /* For each queue, from the most- to least-constrained:
6604b394a23SGary R Hook * find an LSB that can be assigned to the queue. If there are N queues that
6614b394a23SGary R Hook * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
6624b394a23SGary R Hook * dedicated LSB. Remaining LSB regions become a shared resource.
6634b394a23SGary R Hook * If we have fewer LSBs than queues, all LSB regions become shared resources.
6644b394a23SGary R Hook */
ccp_assign_lsbs(struct ccp_device * ccp)6654b394a23SGary R Hook static int ccp_assign_lsbs(struct ccp_device *ccp)
6664b394a23SGary R Hook {
6674b394a23SGary R Hook DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
6684b394a23SGary R Hook DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
6694b394a23SGary R Hook int n_lsbs = 0;
6704b394a23SGary R Hook int bitno;
6714b394a23SGary R Hook int i, lsb_cnt;
6724b394a23SGary R Hook int rc = 0;
6734b394a23SGary R Hook
6744b394a23SGary R Hook bitmap_zero(lsb_pub, MAX_LSB_CNT);
6754b394a23SGary R Hook
6764b394a23SGary R Hook /* Create an aggregate bitmap to get a total count of available LSBs */
6774b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
6784b394a23SGary R Hook bitmap_or(lsb_pub,
6794b394a23SGary R Hook lsb_pub, ccp->cmd_q[i].lsbmask,
6804b394a23SGary R Hook MAX_LSB_CNT);
6814b394a23SGary R Hook
6824b394a23SGary R Hook n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
6834b394a23SGary R Hook
6844b394a23SGary R Hook if (n_lsbs >= ccp->cmd_q_count) {
6854b394a23SGary R Hook /* We have enough LSBS to give every queue a private LSB.
6864b394a23SGary R Hook * Brute force search to start with the queues that are more
6874b394a23SGary R Hook * constrained in LSB choice. When an LSB is privately
6884b394a23SGary R Hook * assigned, it is removed from the public mask.
6894b394a23SGary R Hook * This is an ugly N squared algorithm with some optimization.
6904b394a23SGary R Hook */
6914b394a23SGary R Hook for (lsb_cnt = 1;
6924b394a23SGary R Hook n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
6934b394a23SGary R Hook lsb_cnt++) {
6944b394a23SGary R Hook rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
6954b394a23SGary R Hook lsb_pub);
6964b394a23SGary R Hook if (rc < 0)
6974b394a23SGary R Hook return -EINVAL;
6984b394a23SGary R Hook n_lsbs = rc;
6994b394a23SGary R Hook }
7004b394a23SGary R Hook }
7014b394a23SGary R Hook
7024b394a23SGary R Hook rc = 0;
7034b394a23SGary R Hook /* What's left of the LSBs, according to the public mask, now become
7044b394a23SGary R Hook * shared. Any zero bits in the lsb_pub mask represent an LSB region
7054b394a23SGary R Hook * that can't be used as a shared resource, so mark the LSB slots for
7064b394a23SGary R Hook * them as "in use".
7074b394a23SGary R Hook */
7084b394a23SGary R Hook bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
7094b394a23SGary R Hook
7104b394a23SGary R Hook bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
7114b394a23SGary R Hook while (bitno < MAX_LSB_CNT) {
7124b394a23SGary R Hook bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
7134b394a23SGary R Hook bitmap_set(qlsb, bitno, 1);
7144b394a23SGary R Hook bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
7154b394a23SGary R Hook }
7164b394a23SGary R Hook
7174b394a23SGary R Hook return rc;
7184b394a23SGary R Hook }
7194b394a23SGary R Hook
ccp5_disable_queue_interrupts(struct ccp_device * ccp)7206263b51eSGary R Hook static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
7216263b51eSGary R Hook {
7226263b51eSGary R Hook unsigned int i;
7236263b51eSGary R Hook
7246263b51eSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
7256263b51eSGary R Hook iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
7266263b51eSGary R Hook }
7276263b51eSGary R Hook
ccp5_enable_queue_interrupts(struct ccp_device * ccp)7286263b51eSGary R Hook static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
7296263b51eSGary R Hook {
7306263b51eSGary R Hook unsigned int i;
7316263b51eSGary R Hook
7326263b51eSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
7336263b51eSGary R Hook iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
7346263b51eSGary R Hook }
7356263b51eSGary R Hook
ccp5_irq_bh(unsigned long data)7366263b51eSGary R Hook static void ccp5_irq_bh(unsigned long data)
7376263b51eSGary R Hook {
7386263b51eSGary R Hook struct ccp_device *ccp = (struct ccp_device *)data;
7396263b51eSGary R Hook u32 status;
7406263b51eSGary R Hook unsigned int i;
7416263b51eSGary R Hook
7426263b51eSGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
7436263b51eSGary R Hook struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
7446263b51eSGary R Hook
7456263b51eSGary R Hook status = ioread32(cmd_q->reg_interrupt_status);
7466263b51eSGary R Hook
7476263b51eSGary R Hook if (status) {
7486263b51eSGary R Hook cmd_q->int_status = status;
7496263b51eSGary R Hook cmd_q->q_status = ioread32(cmd_q->reg_status);
7506263b51eSGary R Hook cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
7516263b51eSGary R Hook
7526263b51eSGary R Hook /* On error, only save the first error value */
7536263b51eSGary R Hook if ((status & INT_ERROR) && !cmd_q->cmd_error)
7546263b51eSGary R Hook cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
7556263b51eSGary R Hook
7566263b51eSGary R Hook cmd_q->int_rcvd = 1;
7576263b51eSGary R Hook
7586263b51eSGary R Hook /* Acknowledge the interrupt and wake the kthread */
7596263b51eSGary R Hook iowrite32(status, cmd_q->reg_interrupt_status);
7606263b51eSGary R Hook wake_up_interruptible(&cmd_q->int_queue);
7616263b51eSGary R Hook }
7626263b51eSGary R Hook }
7636263b51eSGary R Hook ccp5_enable_queue_interrupts(ccp);
7646263b51eSGary R Hook }
7656263b51eSGary R Hook
ccp5_irq_handler(int irq,void * data)7666263b51eSGary R Hook static irqreturn_t ccp5_irq_handler(int irq, void *data)
7676263b51eSGary R Hook {
768720419f0SBrijesh Singh struct ccp_device *ccp = (struct ccp_device *)data;
7696263b51eSGary R Hook
7706263b51eSGary R Hook ccp5_disable_queue_interrupts(ccp);
7713cdbe346SGary R Hook ccp->total_interrupts++;
7726263b51eSGary R Hook if (ccp->use_tasklet)
7736263b51eSGary R Hook tasklet_schedule(&ccp->irq_tasklet);
7746263b51eSGary R Hook else
7756263b51eSGary R Hook ccp5_irq_bh((unsigned long)ccp);
7766263b51eSGary R Hook return IRQ_HANDLED;
7776263b51eSGary R Hook }
7786263b51eSGary R Hook
ccp5_init(struct ccp_device * ccp)7794b394a23SGary R Hook static int ccp5_init(struct ccp_device *ccp)
7804b394a23SGary R Hook {
7814b394a23SGary R Hook struct device *dev = ccp->dev;
7824b394a23SGary R Hook struct ccp_cmd_queue *cmd_q;
7834b394a23SGary R Hook struct dma_pool *dma_pool;
7844b394a23SGary R Hook char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
785745952aeSColin Ian King unsigned int qmr, i;
7864b394a23SGary R Hook u64 status;
7874b394a23SGary R Hook u32 status_lo, status_hi;
7884b394a23SGary R Hook int ret;
7894b394a23SGary R Hook
7904b394a23SGary R Hook /* Find available queues */
7914b394a23SGary R Hook qmr = ioread32(ccp->io_regs + Q_MASK_REG);
79203f008c5SHook, Gary /*
79303f008c5SHook, Gary * Check for a access to the registers. If this read returns
79403f008c5SHook, Gary * 0xffffffff, it's likely that the system is running a broken
79503f008c5SHook, Gary * BIOS which disallows access to the device. Stop here and fail
79603f008c5SHook, Gary * the initialization (but not the load, as the PSP could get
79703f008c5SHook, Gary * properly initialized).
79803f008c5SHook, Gary */
79903f008c5SHook, Gary if (qmr == 0xffffffff) {
80003f008c5SHook, Gary dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n");
80103f008c5SHook, Gary return 1;
80203f008c5SHook, Gary }
80303f008c5SHook, Gary
804c4a89279SHook, Gary for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
8054b394a23SGary R Hook if (!(qmr & (1 << i)))
8064b394a23SGary R Hook continue;
8074b394a23SGary R Hook
8084b394a23SGary R Hook /* Allocate a dma pool for this queue */
8094b394a23SGary R Hook snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
8104b394a23SGary R Hook ccp->name, i);
8114b394a23SGary R Hook dma_pool = dma_pool_create(dma_pool_name, dev,
8124b394a23SGary R Hook CCP_DMAPOOL_MAX_SIZE,
8134b394a23SGary R Hook CCP_DMAPOOL_ALIGN, 0);
8144b394a23SGary R Hook if (!dma_pool) {
8154b394a23SGary R Hook dev_err(dev, "unable to allocate dma pool\n");
8164b394a23SGary R Hook ret = -ENOMEM;
817114e2ccdSGary R Hook goto e_pool;
8184b394a23SGary R Hook }
8194b394a23SGary R Hook
8204b394a23SGary R Hook cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
8214b394a23SGary R Hook ccp->cmd_q_count++;
8224b394a23SGary R Hook
8234b394a23SGary R Hook cmd_q->ccp = ccp;
8244b394a23SGary R Hook cmd_q->id = i;
8254b394a23SGary R Hook cmd_q->dma_pool = dma_pool;
8264b394a23SGary R Hook mutex_init(&cmd_q->q_mutex);
8274b394a23SGary R Hook
8284b394a23SGary R Hook /* Page alignment satisfies our needs for N <= 128 */
8294b394a23SGary R Hook BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
8304b394a23SGary R Hook cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
831114e2ccdSGary R Hook cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,
8324b394a23SGary R Hook &cmd_q->qbase_dma,
8334b394a23SGary R Hook GFP_KERNEL);
8344b394a23SGary R Hook if (!cmd_q->qbase) {
8354b394a23SGary R Hook dev_err(dev, "unable to allocate command queue\n");
8364b394a23SGary R Hook ret = -ENOMEM;
8374b394a23SGary R Hook goto e_pool;
8384b394a23SGary R Hook }
8394b394a23SGary R Hook
8404b394a23SGary R Hook cmd_q->qidx = 0;
8414b394a23SGary R Hook /* Preset some register values and masks that are queue
8424b394a23SGary R Hook * number dependent
8434b394a23SGary R Hook */
8444b394a23SGary R Hook cmd_q->reg_control = ccp->io_regs +
8454b394a23SGary R Hook CMD5_Q_STATUS_INCR * (i + 1);
8464b394a23SGary R Hook cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
8474b394a23SGary R Hook cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
8484b394a23SGary R Hook cmd_q->reg_int_enable = cmd_q->reg_control +
8494b394a23SGary R Hook CMD5_Q_INT_ENABLE_BASE;
8504b394a23SGary R Hook cmd_q->reg_interrupt_status = cmd_q->reg_control +
8514b394a23SGary R Hook CMD5_Q_INTERRUPT_STATUS_BASE;
8524b394a23SGary R Hook cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
8534b394a23SGary R Hook cmd_q->reg_int_status = cmd_q->reg_control +
8544b394a23SGary R Hook CMD5_Q_INT_STATUS_BASE;
8554b394a23SGary R Hook cmd_q->reg_dma_status = cmd_q->reg_control +
8564b394a23SGary R Hook CMD5_Q_DMA_STATUS_BASE;
8574b394a23SGary R Hook cmd_q->reg_dma_read_status = cmd_q->reg_control +
8584b394a23SGary R Hook CMD5_Q_DMA_READ_STATUS_BASE;
8594b394a23SGary R Hook cmd_q->reg_dma_write_status = cmd_q->reg_control +
8604b394a23SGary R Hook CMD5_Q_DMA_WRITE_STATUS_BASE;
8614b394a23SGary R Hook
8624b394a23SGary R Hook init_waitqueue_head(&cmd_q->int_queue);
8634b394a23SGary R Hook
8644b394a23SGary R Hook dev_dbg(dev, "queue #%u available\n", i);
8654b394a23SGary R Hook }
866990672d4SGary R Hook
8674b394a23SGary R Hook if (ccp->cmd_q_count == 0) {
8684b394a23SGary R Hook dev_notice(dev, "no command queues available\n");
8690d3c6781SHook, Gary ret = 1;
8704b394a23SGary R Hook goto e_pool;
8714b394a23SGary R Hook }
8724b394a23SGary R Hook
8734b394a23SGary R Hook /* Turn off the queues and disable interrupts until ready */
8746263b51eSGary R Hook ccp5_disable_queue_interrupts(ccp);
8754b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
8764b394a23SGary R Hook cmd_q = &ccp->cmd_q[i];
8774b394a23SGary R Hook
8784b394a23SGary R Hook cmd_q->qcontrol = 0; /* Start with nothing */
8794b394a23SGary R Hook iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
8804b394a23SGary R Hook
8814b394a23SGary R Hook ioread32(cmd_q->reg_int_status);
8824b394a23SGary R Hook ioread32(cmd_q->reg_status);
8834b394a23SGary R Hook
8846263b51eSGary R Hook /* Clear the interrupt status */
88556467cb1SGary R Hook iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
8864b394a23SGary R Hook }
8874b394a23SGary R Hook
8884b394a23SGary R Hook dev_dbg(dev, "Requesting an IRQ...\n");
8894b394a23SGary R Hook /* Request an irq */
890f4d18d65SBrijesh Singh ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
8914b394a23SGary R Hook if (ret) {
8924b394a23SGary R Hook dev_err(dev, "unable to allocate an IRQ\n");
8934b394a23SGary R Hook goto e_pool;
8944b394a23SGary R Hook }
8956263b51eSGary R Hook /* Initialize the ISR tasklet */
8966263b51eSGary R Hook if (ccp->use_tasklet)
8976263b51eSGary R Hook tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
8986263b51eSGary R Hook (unsigned long)ccp);
8994b394a23SGary R Hook
9004b394a23SGary R Hook dev_dbg(dev, "Loading LSB map...\n");
9014b394a23SGary R Hook /* Copy the private LSB mask to the public registers */
9024b394a23SGary R Hook status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
9034b394a23SGary R Hook status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
9044b394a23SGary R Hook iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
9054b394a23SGary R Hook iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
9064b394a23SGary R Hook status = ((u64)status_hi<<30) | (u64)status_lo;
9074b394a23SGary R Hook
9084b394a23SGary R Hook dev_dbg(dev, "Configuring virtual queues...\n");
9094b394a23SGary R Hook /* Configure size of each virtual queue accessible to host */
9104b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
9114b394a23SGary R Hook u32 dma_addr_lo;
9124b394a23SGary R Hook u32 dma_addr_hi;
9134b394a23SGary R Hook
9144b394a23SGary R Hook cmd_q = &ccp->cmd_q[i];
9154b394a23SGary R Hook
9164b394a23SGary R Hook cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
9174b394a23SGary R Hook cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
9184b394a23SGary R Hook
9194b394a23SGary R Hook cmd_q->qdma_tail = cmd_q->qbase_dma;
9204b394a23SGary R Hook dma_addr_lo = low_address(cmd_q->qdma_tail);
9214b394a23SGary R Hook iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
9224b394a23SGary R Hook iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
9234b394a23SGary R Hook
9244b394a23SGary R Hook dma_addr_hi = high_address(cmd_q->qdma_tail);
9254b394a23SGary R Hook cmd_q->qcontrol |= (dma_addr_hi << 16);
9264b394a23SGary R Hook iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
9274b394a23SGary R Hook
9284b394a23SGary R Hook /* Find the LSB regions accessible to the queue */
9294b394a23SGary R Hook ccp_find_lsb_regions(cmd_q, status);
9304b394a23SGary R Hook cmd_q->lsb = -1; /* Unassigned value */
9314b394a23SGary R Hook }
9324b394a23SGary R Hook
9334b394a23SGary R Hook dev_dbg(dev, "Assigning LSBs...\n");
9344b394a23SGary R Hook ret = ccp_assign_lsbs(ccp);
9354b394a23SGary R Hook if (ret) {
9364b394a23SGary R Hook dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
9374b394a23SGary R Hook goto e_irq;
9384b394a23SGary R Hook }
9394b394a23SGary R Hook
9404b394a23SGary R Hook /* Optimization: pre-allocate LSB slots for each queue */
9414b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
9424b394a23SGary R Hook ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
9434b394a23SGary R Hook ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
9444b394a23SGary R Hook }
9454b394a23SGary R Hook
9464b394a23SGary R Hook dev_dbg(dev, "Starting threads...\n");
9474b394a23SGary R Hook /* Create a kthread for each queue */
9484b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
9494b394a23SGary R Hook struct task_struct *kthread;
9504b394a23SGary R Hook
9514b394a23SGary R Hook cmd_q = &ccp->cmd_q[i];
9524b394a23SGary R Hook
953*83bff109SCai Huoqing kthread = kthread_run(ccp_cmd_queue_thread, cmd_q,
9544b394a23SGary R Hook "%s-q%u", ccp->name, cmd_q->id);
9554b394a23SGary R Hook if (IS_ERR(kthread)) {
9564b394a23SGary R Hook dev_err(dev, "error creating queue thread (%ld)\n",
9574b394a23SGary R Hook PTR_ERR(kthread));
9584b394a23SGary R Hook ret = PTR_ERR(kthread);
9594b394a23SGary R Hook goto e_kthread;
9604b394a23SGary R Hook }
9614b394a23SGary R Hook
9624b394a23SGary R Hook cmd_q->kthread = kthread;
9634b394a23SGary R Hook }
9644b394a23SGary R Hook
9654b394a23SGary R Hook dev_dbg(dev, "Enabling interrupts...\n");
9666263b51eSGary R Hook ccp5_enable_queue_interrupts(ccp);
9674b394a23SGary R Hook
9684b394a23SGary R Hook dev_dbg(dev, "Registering device...\n");
9694b394a23SGary R Hook /* Put this on the unit list to make it available */
9704b394a23SGary R Hook ccp_add_device(ccp);
9714b394a23SGary R Hook
972084935b2SGary R Hook ret = ccp_register_rng(ccp);
973084935b2SGary R Hook if (ret)
974084935b2SGary R Hook goto e_kthread;
975084935b2SGary R Hook
97699d90b2eSGary R Hook /* Register the DMA engine support */
97799d90b2eSGary R Hook ret = ccp_dmaengine_register(ccp);
97899d90b2eSGary R Hook if (ret)
9799ddb9dc6SGary R Hook goto e_hwrng;
98099d90b2eSGary R Hook
98193308bafSHook, Gary #ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
9823cdbe346SGary R Hook /* Set up debugfs entries */
9833cdbe346SGary R Hook ccp5_debugfs_setup(ccp);
98493308bafSHook, Gary #endif
9853cdbe346SGary R Hook
9864b394a23SGary R Hook return 0;
9874b394a23SGary R Hook
9889ddb9dc6SGary R Hook e_hwrng:
9899ddb9dc6SGary R Hook ccp_unregister_rng(ccp);
9909ddb9dc6SGary R Hook
9914b394a23SGary R Hook e_kthread:
9924b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
9934b394a23SGary R Hook if (ccp->cmd_q[i].kthread)
9944b394a23SGary R Hook kthread_stop(ccp->cmd_q[i].kthread);
9954b394a23SGary R Hook
9964b394a23SGary R Hook e_irq:
997f4d18d65SBrijesh Singh sp_free_ccp_irq(ccp->sp, ccp);
9984b394a23SGary R Hook
9994b394a23SGary R Hook e_pool:
10004b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
10014b394a23SGary R Hook dma_pool_destroy(ccp->cmd_q[i].dma_pool);
10024b394a23SGary R Hook
10034b394a23SGary R Hook return ret;
10044b394a23SGary R Hook }
10054b394a23SGary R Hook
ccp5_destroy(struct ccp_device * ccp)10064b394a23SGary R Hook static void ccp5_destroy(struct ccp_device *ccp)
10074b394a23SGary R Hook {
10084b394a23SGary R Hook struct ccp_cmd_queue *cmd_q;
10094b394a23SGary R Hook struct ccp_cmd *cmd;
10104b394a23SGary R Hook unsigned int i;
10114b394a23SGary R Hook
101299d90b2eSGary R Hook /* Unregister the DMA engine */
101399d90b2eSGary R Hook ccp_dmaengine_unregister(ccp);
101499d90b2eSGary R Hook
1015084935b2SGary R Hook /* Unregister the RNG */
1016084935b2SGary R Hook ccp_unregister_rng(ccp);
1017084935b2SGary R Hook
10184b394a23SGary R Hook /* Remove this device from the list of available units first */
10194b394a23SGary R Hook ccp_del_device(ccp);
10204b394a23SGary R Hook
102193308bafSHook, Gary #ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
10223cdbe346SGary R Hook /* We're in the process of tearing down the entire driver;
10233cdbe346SGary R Hook * when all the devices are gone clean up debugfs
10243cdbe346SGary R Hook */
10253cdbe346SGary R Hook if (ccp_present())
10263cdbe346SGary R Hook ccp5_debugfs_destroy();
102793308bafSHook, Gary #endif
10283cdbe346SGary R Hook
10294b394a23SGary R Hook /* Disable and clear interrupts */
10306263b51eSGary R Hook ccp5_disable_queue_interrupts(ccp);
10314b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++) {
10324b394a23SGary R Hook cmd_q = &ccp->cmd_q[i];
10334b394a23SGary R Hook
10344b394a23SGary R Hook /* Turn off the run bit */
10354b394a23SGary R Hook iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
10364b394a23SGary R Hook
10374b394a23SGary R Hook /* Clear the interrupt status */
1038116591feSGary R Hook iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
10394b394a23SGary R Hook ioread32(cmd_q->reg_int_status);
10404b394a23SGary R Hook ioread32(cmd_q->reg_status);
10414b394a23SGary R Hook }
10424b394a23SGary R Hook
10434b394a23SGary R Hook /* Stop the queue kthreads */
10444b394a23SGary R Hook for (i = 0; i < ccp->cmd_q_count; i++)
10454b394a23SGary R Hook if (ccp->cmd_q[i].kthread)
10464b394a23SGary R Hook kthread_stop(ccp->cmd_q[i].kthread);
10474b394a23SGary R Hook
1048f4d18d65SBrijesh Singh sp_free_ccp_irq(ccp->sp, ccp);
10494b394a23SGary R Hook
10504b394a23SGary R Hook /* Flush the cmd and backlog queue */
10514b394a23SGary R Hook while (!list_empty(&ccp->cmd)) {
10524b394a23SGary R Hook /* Invoke the callback directly with an error code */
10534b394a23SGary R Hook cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
10544b394a23SGary R Hook list_del(&cmd->entry);
10554b394a23SGary R Hook cmd->callback(cmd->data, -ENODEV);
10564b394a23SGary R Hook }
10574b394a23SGary R Hook while (!list_empty(&ccp->backlog)) {
10584b394a23SGary R Hook /* Invoke the callback directly with an error code */
10594b394a23SGary R Hook cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
10604b394a23SGary R Hook list_del(&cmd->entry);
10614b394a23SGary R Hook cmd->callback(cmd->data, -ENODEV);
10624b394a23SGary R Hook }
10634b394a23SGary R Hook }
10644b394a23SGary R Hook
ccp5_config(struct ccp_device * ccp)10654b394a23SGary R Hook static void ccp5_config(struct ccp_device *ccp)
10664b394a23SGary R Hook {
10674b394a23SGary R Hook /* Public side */
1068500c0106SGary R Hook iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
10694b394a23SGary R Hook }
10704b394a23SGary R Hook
ccp5other_config(struct ccp_device * ccp)1071e14e7d12SGary R Hook static void ccp5other_config(struct ccp_device *ccp)
1072e14e7d12SGary R Hook {
1073e14e7d12SGary R Hook int i;
1074e14e7d12SGary R Hook u32 rnd;
1075e14e7d12SGary R Hook
1076e14e7d12SGary R Hook /* We own all of the queues on the NTB CCP */
1077e14e7d12SGary R Hook
1078e14e7d12SGary R Hook iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
1079e14e7d12SGary R Hook iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
1080e14e7d12SGary R Hook for (i = 0; i < 12; i++) {
1081e14e7d12SGary R Hook rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
1082e14e7d12SGary R Hook iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
1083e14e7d12SGary R Hook }
1084e14e7d12SGary R Hook
1085e14e7d12SGary R Hook iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
1086e14e7d12SGary R Hook iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
1087e14e7d12SGary R Hook iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
1088e14e7d12SGary R Hook
1089e14e7d12SGary R Hook iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
1090e14e7d12SGary R Hook iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
1091e14e7d12SGary R Hook
1092e14e7d12SGary R Hook iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
1093e14e7d12SGary R Hook
1094e14e7d12SGary R Hook ccp5_config(ccp);
1095e14e7d12SGary R Hook }
1096e14e7d12SGary R Hook
1097e14e7d12SGary R Hook /* Version 5 adds some function, but is essentially the same as v5 */
10984b394a23SGary R Hook static const struct ccp_actions ccp5_actions = {
10994b394a23SGary R Hook .aes = ccp5_perform_aes,
11004b394a23SGary R Hook .xts_aes = ccp5_perform_xts_aes,
11014b394a23SGary R Hook .sha = ccp5_perform_sha,
1102990672d4SGary R Hook .des3 = ccp5_perform_des3,
11034b394a23SGary R Hook .rsa = ccp5_perform_rsa,
11044b394a23SGary R Hook .passthru = ccp5_perform_passthru,
11054b394a23SGary R Hook .ecc = ccp5_perform_ecc,
11064b394a23SGary R Hook .sballoc = ccp_lsb_alloc,
11074b394a23SGary R Hook .sbfree = ccp_lsb_free,
11084b394a23SGary R Hook .init = ccp5_init,
11094b394a23SGary R Hook .destroy = ccp5_destroy,
11104b394a23SGary R Hook .get_free_slots = ccp5_get_free_slots,
11114b394a23SGary R Hook };
11124b394a23SGary R Hook
11139ddb9dc6SGary R Hook const struct ccp_vdata ccpv5a = {
11144b394a23SGary R Hook .version = CCP_VERSION(5, 0),
11154b394a23SGary R Hook .setup = ccp5_config,
11164b394a23SGary R Hook .perform = &ccp5_actions,
11174b394a23SGary R Hook .offset = 0x0,
1118e28c190dSGary R Hook .rsamax = CCP5_RSA_MAX_WIDTH,
11194b394a23SGary R Hook };
1120e14e7d12SGary R Hook
11219ddb9dc6SGary R Hook const struct ccp_vdata ccpv5b = {
1122e14e7d12SGary R Hook .version = CCP_VERSION(5, 0),
1123efc989fcSGary R Hook .dma_chan_attr = DMA_PRIVATE,
1124e14e7d12SGary R Hook .setup = ccp5other_config,
1125e14e7d12SGary R Hook .perform = &ccp5_actions,
1126e14e7d12SGary R Hook .offset = 0x0,
1127e28c190dSGary R Hook .rsamax = CCP5_RSA_MAX_WIDTH,
1128e14e7d12SGary R Hook };
1129