163b94509STom Lendacky /* 263b94509STom Lendacky * AMD Cryptographic Coprocessor (CCP) driver 363b94509STom Lendacky * 4553d2374SGary R Hook * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. 563b94509STom Lendacky * 663b94509STom Lendacky * Author: Tom Lendacky <thomas.lendacky@amd.com> 7fba8855cSGary R Hook * Author: Gary R Hook <gary.hook@amd.com> 863b94509STom Lendacky * 963b94509STom Lendacky * This program is free software; you can redistribute it and/or modify 1063b94509STom Lendacky * it under the terms of the GNU General Public License version 2 as 1163b94509STom Lendacky * published by the Free Software Foundation. 1263b94509STom Lendacky */ 1363b94509STom Lendacky 1463b94509STom Lendacky #ifndef __CCP_DEV_H__ 1563b94509STom Lendacky #define __CCP_DEV_H__ 1663b94509STom Lendacky 1763b94509STom Lendacky #include <linux/device.h> 1863b94509STom Lendacky #include <linux/pci.h> 1963b94509STom Lendacky #include <linux/spinlock.h> 2063b94509STom Lendacky #include <linux/mutex.h> 2163b94509STom Lendacky #include <linux/list.h> 2263b94509STom Lendacky #include <linux/wait.h> 2363b94509STom Lendacky #include <linux/dmapool.h> 2463b94509STom Lendacky #include <linux/hw_random.h> 258db88467STom Lendacky #include <linux/bitops.h> 2658ea8abfSGary R Hook #include <linux/interrupt.h> 2758ea8abfSGary R Hook #include <linux/irqreturn.h> 2858ea8abfSGary R Hook #include <linux/dmaengine.h> 2963b94509STom Lendacky 30553d2374SGary R Hook #define MAX_CCP_NAME_LEN 16 3163b94509STom Lendacky #define MAX_DMAPOOL_NAME_LEN 32 3263b94509STom Lendacky 3363b94509STom Lendacky #define MAX_HW_QUEUES 5 3463b94509STom Lendacky #define MAX_CMD_QLEN 100 3563b94509STom Lendacky 3663b94509STom Lendacky #define TRNG_RETRIES 10 3763b94509STom Lendacky 38126ae9adSTom Lendacky #define CACHE_NONE 0x00 39c4f4b325STom Lendacky #define CACHE_WB_NO_ALLOC 0xb7 40c4f4b325STom Lendacky 4163b94509STom Lendacky /****** Register Mappings ******/ 4263b94509STom Lendacky #define Q_MASK_REG 0x000 4363b94509STom Lendacky #define TRNG_OUT_REG 0x00c 4463b94509STom Lendacky #define IRQ_MASK_REG 0x040 4563b94509STom Lendacky #define IRQ_STATUS_REG 0x200 4663b94509STom Lendacky 4763b94509STom Lendacky #define DEL_CMD_Q_JOB 0x124 4863b94509STom Lendacky #define DEL_Q_ACTIVE 0x00000200 4963b94509STom Lendacky #define DEL_Q_ID_SHIFT 6 5063b94509STom Lendacky 5163b94509STom Lendacky #define CMD_REQ0 0x180 5263b94509STom Lendacky #define CMD_REQ_INCR 0x04 5363b94509STom Lendacky 5463b94509STom Lendacky #define CMD_Q_STATUS_BASE 0x210 5563b94509STom Lendacky #define CMD_Q_INT_STATUS_BASE 0x214 5663b94509STom Lendacky #define CMD_Q_STATUS_INCR 0x20 5763b94509STom Lendacky 58c4f4b325STom Lendacky #define CMD_Q_CACHE_BASE 0x228 5963b94509STom Lendacky #define CMD_Q_CACHE_INC 0x20 6063b94509STom Lendacky 618db88467STom Lendacky #define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) 628db88467STom Lendacky #define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f) 6363b94509STom Lendacky 644b394a23SGary R Hook /* ------------------------ CCP Version 5 Specifics ------------------------ */ 654b394a23SGary R Hook #define CMD5_QUEUE_MASK_OFFSET 0x00 66e14e7d12SGary R Hook #define CMD5_QUEUE_PRIO_OFFSET 0x04 674b394a23SGary R Hook #define CMD5_REQID_CONFIG_OFFSET 0x08 68e14e7d12SGary R Hook #define CMD5_CMD_TIMEOUT_OFFSET 0x10 694b394a23SGary R Hook #define LSB_PUBLIC_MASK_LO_OFFSET 0x18 704b394a23SGary R Hook #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C 714b394a23SGary R Hook #define LSB_PRIVATE_MASK_LO_OFFSET 0x20 724b394a23SGary R Hook #define LSB_PRIVATE_MASK_HI_OFFSET 0x24 733cdbe346SGary R Hook #define CMD5_PSP_CCP_VERSION 0x100 744b394a23SGary R Hook 754b394a23SGary R Hook #define CMD5_Q_CONTROL_BASE 0x0000 764b394a23SGary R Hook #define CMD5_Q_TAIL_LO_BASE 0x0004 774b394a23SGary R Hook #define CMD5_Q_HEAD_LO_BASE 0x0008 784b394a23SGary R Hook #define CMD5_Q_INT_ENABLE_BASE 0x000C 794b394a23SGary R Hook #define CMD5_Q_INTERRUPT_STATUS_BASE 0x0010 804b394a23SGary R Hook 814b394a23SGary R Hook #define CMD5_Q_STATUS_BASE 0x0100 824b394a23SGary R Hook #define CMD5_Q_INT_STATUS_BASE 0x0104 834b394a23SGary R Hook #define CMD5_Q_DMA_STATUS_BASE 0x0108 844b394a23SGary R Hook #define CMD5_Q_DMA_READ_STATUS_BASE 0x010C 854b394a23SGary R Hook #define CMD5_Q_DMA_WRITE_STATUS_BASE 0x0110 864b394a23SGary R Hook #define CMD5_Q_ABORT_BASE 0x0114 874b394a23SGary R Hook #define CMD5_Q_AX_CACHE_BASE 0x0118 884b394a23SGary R Hook 89e14e7d12SGary R Hook #define CMD5_CONFIG_0_OFFSET 0x6000 90e14e7d12SGary R Hook #define CMD5_TRNG_CTL_OFFSET 0x6008 91e14e7d12SGary R Hook #define CMD5_AES_MASK_OFFSET 0x6010 92e14e7d12SGary R Hook #define CMD5_CLK_GATE_CTL_OFFSET 0x603C 93e14e7d12SGary R Hook 944b394a23SGary R Hook /* Address offset between two virtual queue registers */ 954b394a23SGary R Hook #define CMD5_Q_STATUS_INCR 0x1000 964b394a23SGary R Hook 974b394a23SGary R Hook /* Bit masks */ 984b394a23SGary R Hook #define CMD5_Q_RUN 0x1 994b394a23SGary R Hook #define CMD5_Q_HALT 0x2 1004b394a23SGary R Hook #define CMD5_Q_MEM_LOCATION 0x4 1014b394a23SGary R Hook #define CMD5_Q_SIZE 0x1F 1024b394a23SGary R Hook #define CMD5_Q_SHIFT 3 1034b394a23SGary R Hook #define COMMANDS_PER_QUEUE 16 1044b394a23SGary R Hook #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \ 1054b394a23SGary R Hook CMD5_Q_SIZE) 1064b394a23SGary R Hook #define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) 1074b394a23SGary R Hook #define Q_DESC_SIZE sizeof(struct ccp5_desc) 1084b394a23SGary R Hook #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n)) 1094b394a23SGary R Hook 1104b394a23SGary R Hook #define INT_COMPLETION 0x1 1114b394a23SGary R Hook #define INT_ERROR 0x2 1124b394a23SGary R Hook #define INT_QUEUE_STOPPED 0x4 11356467cb1SGary R Hook #define INT_EMPTY_QUEUE 0x8 11456467cb1SGary R Hook #define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) 1154b394a23SGary R Hook 1164b394a23SGary R Hook #define LSB_REGION_WIDTH 5 1174b394a23SGary R Hook #define MAX_LSB_CNT 8 1184b394a23SGary R Hook 1194b394a23SGary R Hook #define LSB_SIZE 16 1204b394a23SGary R Hook #define LSB_ITEM_SIZE 32 1214b394a23SGary R Hook #define PLSB_MAP_SIZE (LSB_SIZE) 1224b394a23SGary R Hook #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE) 1234b394a23SGary R Hook 1244b394a23SGary R Hook #define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE) 1254b394a23SGary R Hook 1264b394a23SGary R Hook /* ------------------------ CCP Version 3 Specifics ------------------------ */ 12763b94509STom Lendacky #define REQ0_WAIT_FOR_WRITE 0x00000004 12863b94509STom Lendacky #define REQ0_INT_ON_COMPLETE 0x00000002 12963b94509STom Lendacky #define REQ0_STOP_ON_COMPLETE 0x00000001 13063b94509STom Lendacky 13163b94509STom Lendacky #define REQ0_CMD_Q_SHIFT 9 13263b94509STom Lendacky #define REQ0_JOBID_SHIFT 3 13363b94509STom Lendacky 13463b94509STom Lendacky /****** REQ1 Related Values ******/ 13563b94509STom Lendacky #define REQ1_PROTECT_SHIFT 27 13663b94509STom Lendacky #define REQ1_ENGINE_SHIFT 23 13763b94509STom Lendacky #define REQ1_KEY_KSB_SHIFT 2 13863b94509STom Lendacky 13963b94509STom Lendacky #define REQ1_EOM 0x00000002 14063b94509STom Lendacky #define REQ1_INIT 0x00000001 14163b94509STom Lendacky 14263b94509STom Lendacky /* AES Related Values */ 14363b94509STom Lendacky #define REQ1_AES_TYPE_SHIFT 21 14463b94509STom Lendacky #define REQ1_AES_MODE_SHIFT 18 14563b94509STom Lendacky #define REQ1_AES_ACTION_SHIFT 17 14663b94509STom Lendacky #define REQ1_AES_CFB_SIZE_SHIFT 10 14763b94509STom Lendacky 14863b94509STom Lendacky /* XTS-AES Related Values */ 14963b94509STom Lendacky #define REQ1_XTS_AES_SIZE_SHIFT 10 15063b94509STom Lendacky 15163b94509STom Lendacky /* SHA Related Values */ 15263b94509STom Lendacky #define REQ1_SHA_TYPE_SHIFT 21 15363b94509STom Lendacky 15463b94509STom Lendacky /* RSA Related Values */ 15563b94509STom Lendacky #define REQ1_RSA_MOD_SIZE_SHIFT 10 15663b94509STom Lendacky 15763b94509STom Lendacky /* Pass-Through Related Values */ 15863b94509STom Lendacky #define REQ1_PT_BW_SHIFT 12 15963b94509STom Lendacky #define REQ1_PT_BS_SHIFT 10 16063b94509STom Lendacky 16163b94509STom Lendacky /* ECC Related Values */ 16263b94509STom Lendacky #define REQ1_ECC_AFFINE_CONVERT 0x00200000 16363b94509STom Lendacky #define REQ1_ECC_FUNCTION_SHIFT 18 16463b94509STom Lendacky 16563b94509STom Lendacky /****** REQ4 Related Values ******/ 16663b94509STom Lendacky #define REQ4_KSB_SHIFT 18 16763b94509STom Lendacky #define REQ4_MEMTYPE_SHIFT 16 16863b94509STom Lendacky 16963b94509STom Lendacky /****** REQ6 Related Values ******/ 17063b94509STom Lendacky #define REQ6_MEMTYPE_SHIFT 16 17163b94509STom Lendacky 17263b94509STom Lendacky /****** Key Storage Block ******/ 17363b94509STom Lendacky #define KSB_START 77 17463b94509STom Lendacky #define KSB_END 127 17563b94509STom Lendacky #define KSB_COUNT (KSB_END - KSB_START + 1) 176956ee21aSGary R Hook #define CCP_SB_BITS 256 17763b94509STom Lendacky 17863b94509STom Lendacky #define CCP_JOBID_MASK 0x0000003f 17963b94509STom Lendacky 1804b394a23SGary R Hook /* ------------------------ General CCP Defines ------------------------ */ 1814b394a23SGary R Hook 182efc989fcSGary R Hook #define CCP_DMA_DFLT 0x0 183efc989fcSGary R Hook #define CCP_DMA_PRIV 0x1 184efc989fcSGary R Hook #define CCP_DMA_PUB 0x2 185efc989fcSGary R Hook 18663b94509STom Lendacky #define CCP_DMAPOOL_MAX_SIZE 64 1878db88467STom Lendacky #define CCP_DMAPOOL_ALIGN BIT(5) 18863b94509STom Lendacky 18963b94509STom Lendacky #define CCP_REVERSE_BUF_SIZE 64 19063b94509STom Lendacky 191956ee21aSGary R Hook #define CCP_AES_KEY_SB_COUNT 1 192956ee21aSGary R Hook #define CCP_AES_CTX_SB_COUNT 1 19363b94509STom Lendacky 194956ee21aSGary R Hook #define CCP_XTS_AES_KEY_SB_COUNT 1 195956ee21aSGary R Hook #define CCP_XTS_AES_CTX_SB_COUNT 1 19663b94509STom Lendacky 197990672d4SGary R Hook #define CCP_DES3_KEY_SB_COUNT 1 198990672d4SGary R Hook #define CCP_DES3_CTX_SB_COUNT 1 199990672d4SGary R Hook 200956ee21aSGary R Hook #define CCP_SHA_SB_COUNT 1 20163b94509STom Lendacky 20263b94509STom Lendacky #define CCP_RSA_MAX_WIDTH 4096 20363b94509STom Lendacky 20463b94509STom Lendacky #define CCP_PASSTHRU_BLOCKSIZE 256 20563b94509STom Lendacky #define CCP_PASSTHRU_MASKSIZE 32 206956ee21aSGary R Hook #define CCP_PASSTHRU_SB_COUNT 1 20763b94509STom Lendacky 20863b94509STom Lendacky #define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */ 20963b94509STom Lendacky #define CCP_ECC_MAX_OPERANDS 6 21063b94509STom Lendacky #define CCP_ECC_MAX_OUTPUTS 3 21163b94509STom Lendacky #define CCP_ECC_SRC_BUF_SIZE 448 21263b94509STom Lendacky #define CCP_ECC_DST_BUF_SIZE 192 21363b94509STom Lendacky #define CCP_ECC_OPERAND_SIZE 64 21463b94509STom Lendacky #define CCP_ECC_OUTPUT_SIZE 64 21563b94509STom Lendacky #define CCP_ECC_RESULT_OFFSET 60 21663b94509STom Lendacky #define CCP_ECC_RESULT_SUCCESS 0x0001 21763b94509STom Lendacky 218956ee21aSGary R Hook #define CCP_SB_BYTES 32 219956ee21aSGary R Hook 220ea0375afSGary R Hook struct ccp_op; 22163b94509STom Lendacky struct ccp_device; 22263b94509STom Lendacky struct ccp_cmd; 2234b394a23SGary R Hook struct ccp_fns; 22463b94509STom Lendacky 22558ea8abfSGary R Hook struct ccp_dma_cmd { 22658ea8abfSGary R Hook struct list_head entry; 22758ea8abfSGary R Hook 22858ea8abfSGary R Hook struct ccp_cmd ccp_cmd; 22958ea8abfSGary R Hook }; 23058ea8abfSGary R Hook 23158ea8abfSGary R Hook struct ccp_dma_desc { 23258ea8abfSGary R Hook struct list_head entry; 23358ea8abfSGary R Hook 23458ea8abfSGary R Hook struct ccp_device *ccp; 23558ea8abfSGary R Hook 23658ea8abfSGary R Hook struct list_head pending; 23758ea8abfSGary R Hook struct list_head active; 23858ea8abfSGary R Hook 23958ea8abfSGary R Hook enum dma_status status; 24058ea8abfSGary R Hook struct dma_async_tx_descriptor tx_desc; 24158ea8abfSGary R Hook size_t len; 24258ea8abfSGary R Hook }; 24358ea8abfSGary R Hook 24458ea8abfSGary R Hook struct ccp_dma_chan { 24558ea8abfSGary R Hook struct ccp_device *ccp; 24658ea8abfSGary R Hook 24758ea8abfSGary R Hook spinlock_t lock; 248e5da5c56SGary R Hook struct list_head created; 24958ea8abfSGary R Hook struct list_head pending; 25058ea8abfSGary R Hook struct list_head active; 25158ea8abfSGary R Hook struct list_head complete; 25258ea8abfSGary R Hook 25358ea8abfSGary R Hook struct tasklet_struct cleanup_tasklet; 25458ea8abfSGary R Hook 25558ea8abfSGary R Hook enum dma_status status; 25658ea8abfSGary R Hook struct dma_chan dma_chan; 25758ea8abfSGary R Hook }; 25858ea8abfSGary R Hook 25963b94509STom Lendacky struct ccp_cmd_queue { 26063b94509STom Lendacky struct ccp_device *ccp; 26163b94509STom Lendacky 26263b94509STom Lendacky /* Queue identifier */ 26363b94509STom Lendacky u32 id; 26463b94509STom Lendacky 26563b94509STom Lendacky /* Queue dma pool */ 26663b94509STom Lendacky struct dma_pool *dma_pool; 26763b94509STom Lendacky 2684b394a23SGary R Hook /* Queue base address (not neccessarily aligned)*/ 2694b394a23SGary R Hook struct ccp5_desc *qbase; 2704b394a23SGary R Hook 2714b394a23SGary R Hook /* Aligned queue start address (per requirement) */ 2724b394a23SGary R Hook struct mutex q_mutex ____cacheline_aligned; 2734b394a23SGary R Hook unsigned int qidx; 2744b394a23SGary R Hook 2754b394a23SGary R Hook /* Version 5 has different requirements for queue memory */ 2764b394a23SGary R Hook unsigned int qsize; 2774b394a23SGary R Hook dma_addr_t qbase_dma; 2784b394a23SGary R Hook dma_addr_t qdma_tail; 2794b394a23SGary R Hook 280956ee21aSGary R Hook /* Per-queue reserved storage block(s) */ 281956ee21aSGary R Hook u32 sb_key; 282956ee21aSGary R Hook u32 sb_ctx; 28363b94509STom Lendacky 2844b394a23SGary R Hook /* Bitmap of LSBs that can be accessed by this queue */ 2854b394a23SGary R Hook DECLARE_BITMAP(lsbmask, MAX_LSB_CNT); 2864b394a23SGary R Hook /* Private LSB that is assigned to this queue, or -1 if none. 2874b394a23SGary R Hook * Bitmap for my private LSB, unused otherwise 2884b394a23SGary R Hook */ 2893cf79968SGary R Hook int lsb; 2904b394a23SGary R Hook DECLARE_BITMAP(lsbmap, PLSB_MAP_SIZE); 2914b394a23SGary R Hook 29263b94509STom Lendacky /* Queue processing thread */ 29363b94509STom Lendacky struct task_struct *kthread; 29463b94509STom Lendacky unsigned int active; 29563b94509STom Lendacky unsigned int suspended; 29663b94509STom Lendacky 29763b94509STom Lendacky /* Number of free command slots available */ 29863b94509STom Lendacky unsigned int free_slots; 29963b94509STom Lendacky 30063b94509STom Lendacky /* Interrupt masks */ 30163b94509STom Lendacky u32 int_ok; 30263b94509STom Lendacky u32 int_err; 30363b94509STom Lendacky 30463b94509STom Lendacky /* Register addresses for queue */ 3054b394a23SGary R Hook void __iomem *reg_control; 3064b394a23SGary R Hook void __iomem *reg_tail_lo; 3074b394a23SGary R Hook void __iomem *reg_head_lo; 3084b394a23SGary R Hook void __iomem *reg_int_enable; 3094b394a23SGary R Hook void __iomem *reg_interrupt_status; 31063b94509STom Lendacky void __iomem *reg_status; 31163b94509STom Lendacky void __iomem *reg_int_status; 3124b394a23SGary R Hook void __iomem *reg_dma_status; 3134b394a23SGary R Hook void __iomem *reg_dma_read_status; 3144b394a23SGary R Hook void __iomem *reg_dma_write_status; 3154b394a23SGary R Hook u32 qcontrol; /* Cached control register */ 31663b94509STom Lendacky 31763b94509STom Lendacky /* Status values from job */ 31863b94509STom Lendacky u32 int_status; 31963b94509STom Lendacky u32 q_status; 32063b94509STom Lendacky u32 q_int_status; 32163b94509STom Lendacky u32 cmd_error; 32263b94509STom Lendacky 32363b94509STom Lendacky /* Interrupt wait queue */ 32463b94509STom Lendacky wait_queue_head_t int_queue; 32563b94509STom Lendacky unsigned int int_rcvd; 3263cdbe346SGary R Hook 3273cdbe346SGary R Hook /* Per-queue Statistics */ 3283cdbe346SGary R Hook unsigned long total_ops; 3293cdbe346SGary R Hook unsigned long total_aes_ops; 3303cdbe346SGary R Hook unsigned long total_xts_aes_ops; 3313cdbe346SGary R Hook unsigned long total_3des_ops; 3323cdbe346SGary R Hook unsigned long total_sha_ops; 3333cdbe346SGary R Hook unsigned long total_rsa_ops; 3343cdbe346SGary R Hook unsigned long total_pt_ops; 3353cdbe346SGary R Hook unsigned long total_ecc_ops; 33663b94509STom Lendacky } ____cacheline_aligned; 33763b94509STom Lendacky 33863b94509STom Lendacky struct ccp_device { 339553d2374SGary R Hook struct list_head entry; 340553d2374SGary R Hook 341c7019c4dSGary R Hook struct ccp_vdata *vdata; 342553d2374SGary R Hook unsigned int ord; 343553d2374SGary R Hook char name[MAX_CCP_NAME_LEN]; 344553d2374SGary R Hook char rngname[MAX_CCP_NAME_LEN]; 345553d2374SGary R Hook 34663b94509STom Lendacky struct device *dev; 34763b94509STom Lendacky 348fa242e80SGary R Hook /* Bus specific device information 34963b94509STom Lendacky */ 35063b94509STom Lendacky void *dev_specific; 35163b94509STom Lendacky int (*get_irq)(struct ccp_device *ccp); 35263b94509STom Lendacky void (*free_irq)(struct ccp_device *ccp); 3537b537b24SGary R Hook unsigned int qim; 3543d77565bSTom Lendacky unsigned int irq; 3557b537b24SGary R Hook bool use_tasklet; 3567b537b24SGary R Hook struct tasklet_struct irq_tasklet; 35763b94509STom Lendacky 358fa242e80SGary R Hook /* I/O area used for device communication. The register mapping 35963b94509STom Lendacky * starts at an offset into the mapped bar. 36063b94509STom Lendacky * The CMD_REQx registers and the Delete_Cmd_Queue_Job register 36163b94509STom Lendacky * need to be protected while a command queue thread is accessing 36263b94509STom Lendacky * them. 36363b94509STom Lendacky */ 36463b94509STom Lendacky struct mutex req_mutex ____cacheline_aligned; 36563b94509STom Lendacky void __iomem *io_map; 36663b94509STom Lendacky void __iomem *io_regs; 36763b94509STom Lendacky 368fa242e80SGary R Hook /* Master lists that all cmds are queued on. Because there can be 36963b94509STom Lendacky * more than one CCP command queue that can process a cmd a separate 37063b94509STom Lendacky * backlog list is neeeded so that the backlog completion call 37163b94509STom Lendacky * completes before the cmd is available for execution. 37263b94509STom Lendacky */ 37363b94509STom Lendacky spinlock_t cmd_lock ____cacheline_aligned; 37463b94509STom Lendacky unsigned int cmd_count; 37563b94509STom Lendacky struct list_head cmd; 37663b94509STom Lendacky struct list_head backlog; 37763b94509STom Lendacky 378fa242e80SGary R Hook /* The command queues. These represent the queues available on the 37963b94509STom Lendacky * CCP that are available for processing cmds 38063b94509STom Lendacky */ 38163b94509STom Lendacky struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES]; 38263b94509STom Lendacky unsigned int cmd_q_count; 38363b94509STom Lendacky 384fa242e80SGary R Hook /* Support for the CCP True RNG 38563b94509STom Lendacky */ 38663b94509STom Lendacky struct hwrng hwrng; 38763b94509STom Lendacky unsigned int hwrng_retries; 38863b94509STom Lendacky 389fa242e80SGary R Hook /* Support for the CCP DMA capabilities 39058ea8abfSGary R Hook */ 39158ea8abfSGary R Hook struct dma_device dma_dev; 39258ea8abfSGary R Hook struct ccp_dma_chan *ccp_dma_chan; 39358ea8abfSGary R Hook struct kmem_cache *dma_cmd_cache; 39458ea8abfSGary R Hook struct kmem_cache *dma_desc_cache; 39558ea8abfSGary R Hook 396fa242e80SGary R Hook /* A counter used to generate job-ids for cmds submitted to the CCP 39763b94509STom Lendacky */ 39863b94509STom Lendacky atomic_t current_id ____cacheline_aligned; 39963b94509STom Lendacky 40058a690b7SGary R Hook /* The v3 CCP uses key storage blocks (SB) to maintain context for 40158a690b7SGary R Hook * certain operations. To prevent multiple cmds from using the same 40258a690b7SGary R Hook * SB range a command queue reserves an SB range for the duration of 40358a690b7SGary R Hook * the cmd. Each queue, will however, reserve 2 SB blocks for 40458a690b7SGary R Hook * operations that only require single SB entries (eg. AES context/iv 40558a690b7SGary R Hook * and key) in order to avoid allocation contention. This will reserve 40658a690b7SGary R Hook * at most 10 SB entries, leaving 40 SB entries available for dynamic 40758a690b7SGary R Hook * allocation. 40858a690b7SGary R Hook * 40958a690b7SGary R Hook * The v5 CCP Local Storage Block (LSB) is broken up into 8 41058a690b7SGary R Hook * memrory ranges, each of which can be enabled for access by one 41158a690b7SGary R Hook * or more queues. Device initialization takes this into account, 41258a690b7SGary R Hook * and attempts to assign one region for exclusive use by each 41358a690b7SGary R Hook * available queue; the rest are then aggregated as "public" use. 41458a690b7SGary R Hook * If there are fewer regions than queues, all regions are shared 41558a690b7SGary R Hook * amongst all queues. 41663b94509STom Lendacky */ 417956ee21aSGary R Hook struct mutex sb_mutex ____cacheline_aligned; 418956ee21aSGary R Hook DECLARE_BITMAP(sb, KSB_COUNT); 419956ee21aSGary R Hook wait_queue_head_t sb_queue; 420956ee21aSGary R Hook unsigned int sb_avail; 421956ee21aSGary R Hook unsigned int sb_count; 422956ee21aSGary R Hook u32 sb_start; 42363b94509STom Lendacky 4244b394a23SGary R Hook /* Bitmap of shared LSBs, if any */ 4254b394a23SGary R Hook DECLARE_BITMAP(lsbmap, SLSB_MAP_SIZE); 4264b394a23SGary R Hook 42763b94509STom Lendacky /* Suspend support */ 42863b94509STom Lendacky unsigned int suspending; 42963b94509STom Lendacky wait_queue_head_t suspend_queue; 430126ae9adSTom Lendacky 431126ae9adSTom Lendacky /* DMA caching attribute support */ 432126ae9adSTom Lendacky unsigned int axcache; 4333cdbe346SGary R Hook 4343cdbe346SGary R Hook /* Device Statistics */ 4353cdbe346SGary R Hook unsigned long total_interrupts; 4363cdbe346SGary R Hook 4373cdbe346SGary R Hook /* DebugFS info */ 4383cdbe346SGary R Hook struct dentry *debugfs_instance; 43963b94509STom Lendacky }; 44063b94509STom Lendacky 441ea0375afSGary R Hook enum ccp_memtype { 442ea0375afSGary R Hook CCP_MEMTYPE_SYSTEM = 0, 443956ee21aSGary R Hook CCP_MEMTYPE_SB, 444ea0375afSGary R Hook CCP_MEMTYPE_LOCAL, 445ea0375afSGary R Hook CCP_MEMTYPE__LAST, 446ea0375afSGary R Hook }; 4474b394a23SGary R Hook #define CCP_MEMTYPE_LSB CCP_MEMTYPE_KSB 448ea0375afSGary R Hook 4492d158391SGary R Hook 450ea0375afSGary R Hook struct ccp_dma_info { 451ea0375afSGary R Hook dma_addr_t address; 452ea0375afSGary R Hook unsigned int offset; 453ea0375afSGary R Hook unsigned int length; 454ea0375afSGary R Hook enum dma_data_direction dir; 4552d158391SGary R Hook } __packed __aligned(4); 456ea0375afSGary R Hook 457ea0375afSGary R Hook struct ccp_dm_workarea { 458ea0375afSGary R Hook struct device *dev; 459ea0375afSGary R Hook struct dma_pool *dma_pool; 460ea0375afSGary R Hook 461ea0375afSGary R Hook u8 *address; 462ea0375afSGary R Hook struct ccp_dma_info dma; 4632d158391SGary R Hook unsigned int length; 464ea0375afSGary R Hook }; 465ea0375afSGary R Hook 466ea0375afSGary R Hook struct ccp_sg_workarea { 467ea0375afSGary R Hook struct scatterlist *sg; 468ea0375afSGary R Hook int nents; 4692d158391SGary R Hook unsigned int sg_used; 470ea0375afSGary R Hook 471ea0375afSGary R Hook struct scatterlist *dma_sg; 472ea0375afSGary R Hook struct device *dma_dev; 473ea0375afSGary R Hook unsigned int dma_count; 474ea0375afSGary R Hook enum dma_data_direction dma_dir; 475ea0375afSGary R Hook 476ea0375afSGary R Hook u64 bytes_left; 477ea0375afSGary R Hook }; 478ea0375afSGary R Hook 479ea0375afSGary R Hook struct ccp_data { 480ea0375afSGary R Hook struct ccp_sg_workarea sg_wa; 481ea0375afSGary R Hook struct ccp_dm_workarea dm_wa; 482ea0375afSGary R Hook }; 483ea0375afSGary R Hook 484ea0375afSGary R Hook struct ccp_mem { 485ea0375afSGary R Hook enum ccp_memtype type; 486ea0375afSGary R Hook union { 487ea0375afSGary R Hook struct ccp_dma_info dma; 488956ee21aSGary R Hook u32 sb; 489ea0375afSGary R Hook } u; 490ea0375afSGary R Hook }; 491ea0375afSGary R Hook 492ea0375afSGary R Hook struct ccp_aes_op { 493ea0375afSGary R Hook enum ccp_aes_type type; 494ea0375afSGary R Hook enum ccp_aes_mode mode; 495ea0375afSGary R Hook enum ccp_aes_action action; 496f7cc02b3SGary R Hook unsigned int size; 497ea0375afSGary R Hook }; 498ea0375afSGary R Hook 499ea0375afSGary R Hook struct ccp_xts_aes_op { 500ea0375afSGary R Hook enum ccp_aes_action action; 501ea0375afSGary R Hook enum ccp_xts_aes_unit_size unit_size; 502ea0375afSGary R Hook }; 503ea0375afSGary R Hook 504990672d4SGary R Hook struct ccp_des3_op { 505990672d4SGary R Hook enum ccp_des3_type type; 506990672d4SGary R Hook enum ccp_des3_mode mode; 507990672d4SGary R Hook enum ccp_des3_action action; 508990672d4SGary R Hook }; 509990672d4SGary R Hook 510ea0375afSGary R Hook struct ccp_sha_op { 511ea0375afSGary R Hook enum ccp_sha_type type; 512ea0375afSGary R Hook u64 msg_bits; 513ea0375afSGary R Hook }; 514ea0375afSGary R Hook 515ea0375afSGary R Hook struct ccp_rsa_op { 516ea0375afSGary R Hook u32 mod_size; 517ea0375afSGary R Hook u32 input_len; 518ea0375afSGary R Hook }; 519ea0375afSGary R Hook 520ea0375afSGary R Hook struct ccp_passthru_op { 521ea0375afSGary R Hook enum ccp_passthru_bitwise bit_mod; 522ea0375afSGary R Hook enum ccp_passthru_byteswap byte_swap; 523ea0375afSGary R Hook }; 524ea0375afSGary R Hook 525ea0375afSGary R Hook struct ccp_ecc_op { 526ea0375afSGary R Hook enum ccp_ecc_function function; 527ea0375afSGary R Hook }; 528ea0375afSGary R Hook 529ea0375afSGary R Hook struct ccp_op { 530ea0375afSGary R Hook struct ccp_cmd_queue *cmd_q; 531ea0375afSGary R Hook 532ea0375afSGary R Hook u32 jobid; 533ea0375afSGary R Hook u32 ioc; 534ea0375afSGary R Hook u32 soc; 535956ee21aSGary R Hook u32 sb_key; 536956ee21aSGary R Hook u32 sb_ctx; 537ea0375afSGary R Hook u32 init; 538ea0375afSGary R Hook u32 eom; 539ea0375afSGary R Hook 540ea0375afSGary R Hook struct ccp_mem src; 541ea0375afSGary R Hook struct ccp_mem dst; 5424b394a23SGary R Hook struct ccp_mem exp; 543ea0375afSGary R Hook 544ea0375afSGary R Hook union { 545ea0375afSGary R Hook struct ccp_aes_op aes; 546ea0375afSGary R Hook struct ccp_xts_aes_op xts; 547990672d4SGary R Hook struct ccp_des3_op des3; 548ea0375afSGary R Hook struct ccp_sha_op sha; 549ea0375afSGary R Hook struct ccp_rsa_op rsa; 550ea0375afSGary R Hook struct ccp_passthru_op passthru; 551ea0375afSGary R Hook struct ccp_ecc_op ecc; 552ea0375afSGary R Hook } u; 553ea0375afSGary R Hook }; 554ea0375afSGary R Hook 555ea0375afSGary R Hook static inline u32 ccp_addr_lo(struct ccp_dma_info *info) 556ea0375afSGary R Hook { 557ea0375afSGary R Hook return lower_32_bits(info->address + info->offset); 558ea0375afSGary R Hook } 559ea0375afSGary R Hook 560ea0375afSGary R Hook static inline u32 ccp_addr_hi(struct ccp_dma_info *info) 561ea0375afSGary R Hook { 562ea0375afSGary R Hook return upper_32_bits(info->address + info->offset) & 0x0000ffff; 563ea0375afSGary R Hook } 564ea0375afSGary R Hook 5654b394a23SGary R Hook /** 5664b394a23SGary R Hook * descriptor for version 5 CPP commands 5674b394a23SGary R Hook * 8 32-bit words: 5684b394a23SGary R Hook * word 0: function; engine; control bits 5694b394a23SGary R Hook * word 1: length of source data 5704b394a23SGary R Hook * word 2: low 32 bits of source pointer 5714b394a23SGary R Hook * word 3: upper 16 bits of source pointer; source memory type 5724b394a23SGary R Hook * word 4: low 32 bits of destination pointer 5734b394a23SGary R Hook * word 5: upper 16 bits of destination pointer; destination memory type 5744b394a23SGary R Hook * word 6: low 32 bits of key pointer 5754b394a23SGary R Hook * word 7: upper 16 bits of key pointer; key memory type 5764b394a23SGary R Hook */ 5774b394a23SGary R Hook struct dword0 { 578fdd2cf9dSGary R Hook unsigned int soc:1; 579fdd2cf9dSGary R Hook unsigned int ioc:1; 580fdd2cf9dSGary R Hook unsigned int rsvd1:1; 581fdd2cf9dSGary R Hook unsigned int init:1; 582fdd2cf9dSGary R Hook unsigned int eom:1; /* AES/SHA only */ 583fdd2cf9dSGary R Hook unsigned int function:15; 584fdd2cf9dSGary R Hook unsigned int engine:4; 585fdd2cf9dSGary R Hook unsigned int prot:1; 586fdd2cf9dSGary R Hook unsigned int rsvd2:7; 5874b394a23SGary R Hook }; 5884b394a23SGary R Hook 5894b394a23SGary R Hook struct dword3 { 590fdd2cf9dSGary R Hook unsigned int src_hi:16; 591fdd2cf9dSGary R Hook unsigned int src_mem:2; 592fdd2cf9dSGary R Hook unsigned int lsb_cxt_id:8; 593fdd2cf9dSGary R Hook unsigned int rsvd1:5; 594fdd2cf9dSGary R Hook unsigned int fixed:1; 5954b394a23SGary R Hook }; 5964b394a23SGary R Hook 5974b394a23SGary R Hook union dword4 { 5984b394a23SGary R Hook __le32 dst_lo; /* NON-SHA */ 5994b394a23SGary R Hook __le32 sha_len_lo; /* SHA */ 6004b394a23SGary R Hook }; 6014b394a23SGary R Hook 6024b394a23SGary R Hook union dword5 { 6034b394a23SGary R Hook struct { 604fdd2cf9dSGary R Hook unsigned int dst_hi:16; 605fdd2cf9dSGary R Hook unsigned int dst_mem:2; 606fdd2cf9dSGary R Hook unsigned int rsvd1:13; 607fdd2cf9dSGary R Hook unsigned int fixed:1; 6084b394a23SGary R Hook } fields; 6094b394a23SGary R Hook __le32 sha_len_hi; 6104b394a23SGary R Hook }; 6114b394a23SGary R Hook 6124b394a23SGary R Hook struct dword7 { 613fdd2cf9dSGary R Hook unsigned int key_hi:16; 614fdd2cf9dSGary R Hook unsigned int key_mem:2; 615fdd2cf9dSGary R Hook unsigned int rsvd1:14; 6164b394a23SGary R Hook }; 6174b394a23SGary R Hook 6184b394a23SGary R Hook struct ccp5_desc { 6194b394a23SGary R Hook struct dword0 dw0; 6204b394a23SGary R Hook __le32 length; 6214b394a23SGary R Hook __le32 src_lo; 6224b394a23SGary R Hook struct dword3 dw3; 6234b394a23SGary R Hook union dword4 dw4; 6244b394a23SGary R Hook union dword5 dw5; 6254b394a23SGary R Hook __le32 key_lo; 6264b394a23SGary R Hook struct dword7 dw7; 6274b394a23SGary R Hook }; 6284b394a23SGary R Hook 62963b94509STom Lendacky int ccp_pci_init(void); 63063b94509STom Lendacky void ccp_pci_exit(void); 63163b94509STom Lendacky 632c4f4b325STom Lendacky int ccp_platform_init(void); 633c4f4b325STom Lendacky void ccp_platform_exit(void); 634c4f4b325STom Lendacky 635ea0375afSGary R Hook void ccp_add_device(struct ccp_device *ccp); 636ea0375afSGary R Hook void ccp_del_device(struct ccp_device *ccp); 63763b94509STom Lendacky 63881422badSGary R Hook extern void ccp_log_error(struct ccp_device *, int); 63981422badSGary R Hook 640ea0375afSGary R Hook struct ccp_device *ccp_alloc_struct(struct device *dev); 641ea0375afSGary R Hook bool ccp_queues_suspended(struct ccp_device *ccp); 642ea0375afSGary R Hook int ccp_cmd_queue_thread(void *data); 6438256e683SGary R Hook int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait); 64463b94509STom Lendacky 64563b94509STom Lendacky int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd); 64663b94509STom Lendacky 647084935b2SGary R Hook int ccp_register_rng(struct ccp_device *ccp); 648084935b2SGary R Hook void ccp_unregister_rng(struct ccp_device *ccp); 64958ea8abfSGary R Hook int ccp_dmaengine_register(struct ccp_device *ccp); 65058ea8abfSGary R Hook void ccp_dmaengine_unregister(struct ccp_device *ccp); 65158ea8abfSGary R Hook 6523cdbe346SGary R Hook void ccp5_debugfs_setup(struct ccp_device *ccp); 6533cdbe346SGary R Hook void ccp5_debugfs_destroy(void); 6543cdbe346SGary R Hook 655*970e8303SBrijesh Singh int ccp_dev_init(struct ccp_device *ccp); 656*970e8303SBrijesh Singh void ccp_dev_destroy(struct ccp_device *ccp); 657*970e8303SBrijesh Singh int ccp_dev_suspend(struct ccp_device *ccp, pm_message_t state); 658*970e8303SBrijesh Singh int ccp_dev_resume(struct ccp_device *ccp); 659*970e8303SBrijesh Singh 66058a690b7SGary R Hook /* Structure for computation functions that are device-specific */ 66158a690b7SGary R Hook struct ccp_actions { 66258a690b7SGary R Hook int (*aes)(struct ccp_op *); 66358a690b7SGary R Hook int (*xts_aes)(struct ccp_op *); 664990672d4SGary R Hook int (*des3)(struct ccp_op *); 66558a690b7SGary R Hook int (*sha)(struct ccp_op *); 66658a690b7SGary R Hook int (*rsa)(struct ccp_op *); 66758a690b7SGary R Hook int (*passthru)(struct ccp_op *); 66858a690b7SGary R Hook int (*ecc)(struct ccp_op *); 66958a690b7SGary R Hook u32 (*sballoc)(struct ccp_cmd_queue *, unsigned int); 670990672d4SGary R Hook void (*sbfree)(struct ccp_cmd_queue *, unsigned int, unsigned int); 671bb4e89b3SGary R Hook unsigned int (*get_free_slots)(struct ccp_cmd_queue *); 67258a690b7SGary R Hook int (*init)(struct ccp_device *); 67358a690b7SGary R Hook void (*destroy)(struct ccp_device *); 67458a690b7SGary R Hook irqreturn_t (*irqhandler)(int, void *); 67558a690b7SGary R Hook }; 67658a690b7SGary R Hook 67758a690b7SGary R Hook /* Structure to hold CCP version-specific values */ 67858a690b7SGary R Hook struct ccp_vdata { 6794b394a23SGary R Hook const unsigned int version; 680efc989fcSGary R Hook const unsigned int dma_chan_attr; 6814b394a23SGary R Hook void (*setup)(struct ccp_device *); 68258a690b7SGary R Hook const struct ccp_actions *perform; 68358a690b7SGary R Hook const unsigned int bar; 68458a690b7SGary R Hook const unsigned int offset; 68558a690b7SGary R Hook }; 68658a690b7SGary R Hook 687*970e8303SBrijesh Singh extern const struct ccp_vdata ccpv3_platform; 6889ddb9dc6SGary R Hook extern const struct ccp_vdata ccpv3; 6899ddb9dc6SGary R Hook extern const struct ccp_vdata ccpv5a; 6909ddb9dc6SGary R Hook extern const struct ccp_vdata ccpv5b; 69158a690b7SGary R Hook 69263b94509STom Lendacky #endif 693