15c8d850cSKrzysztof Kozlowski // SPDX-License-Identifier: GPL-2.0 25c8d850cSKrzysztof Kozlowski // 35c8d850cSKrzysztof Kozlowski // Cryptographic API. 45c8d850cSKrzysztof Kozlowski // 55c8d850cSKrzysztof Kozlowski // Support for Samsung S5PV210 and Exynos HW acceleration. 65c8d850cSKrzysztof Kozlowski // 75c8d850cSKrzysztof Kozlowski // Copyright (C) 2011 NetUP Inc. All rights reserved. 85c8d850cSKrzysztof Kozlowski // Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. 95c8d850cSKrzysztof Kozlowski // 105c8d850cSKrzysztof Kozlowski // Hash part based on omap-sham.c driver. 11a49e490cSVladimir Zapolskiy 12a49e490cSVladimir Zapolskiy #include <linux/clk.h> 133cf9d84eSKrzysztof Kozlowski #include <linux/crypto.h> 143cf9d84eSKrzysztof Kozlowski #include <linux/dma-mapping.h> 153cf9d84eSKrzysztof Kozlowski #include <linux/err.h> 163cf9d84eSKrzysztof Kozlowski #include <linux/errno.h> 173cf9d84eSKrzysztof Kozlowski #include <linux/init.h> 183cf9d84eSKrzysztof Kozlowski #include <linux/interrupt.h> 193cf9d84eSKrzysztof Kozlowski #include <linux/io.h> 203cf9d84eSKrzysztof Kozlowski #include <linux/kernel.h> 213cf9d84eSKrzysztof Kozlowski #include <linux/module.h> 223cf9d84eSKrzysztof Kozlowski #include <linux/of.h> 236b238db7SKrzysztof Kozlowski #include <linux/of_device.h> 24a49e490cSVladimir Zapolskiy #include <linux/platform_device.h> 25a49e490cSVladimir Zapolskiy #include <linux/scatterlist.h> 26a49e490cSVladimir Zapolskiy 27a49e490cSVladimir Zapolskiy #include <crypto/ctr.h> 283cf9d84eSKrzysztof Kozlowski #include <crypto/aes.h> 293cf9d84eSKrzysztof Kozlowski #include <crypto/algapi.h> 309e4a1100SKrzysztof Kozlowski #include <crypto/scatterwalk.h> 31a49e490cSVladimir Zapolskiy 32c2afad6cSKamil Konieczny #include <crypto/hash.h> 33c2afad6cSKamil Konieczny #include <crypto/md5.h> 34a24d22b2SEric Biggers #include <crypto/sha1.h> 35a24d22b2SEric Biggers #include <crypto/sha2.h> 36c2afad6cSKamil Konieczny #include <crypto/internal/hash.h> 37c2afad6cSKamil Konieczny 38a49e490cSVladimir Zapolskiy #define _SBF(s, v) ((v) << (s)) 39a49e490cSVladimir Zapolskiy 40a49e490cSVladimir Zapolskiy /* Feed control registers */ 41a49e490cSVladimir Zapolskiy #define SSS_REG_FCINTSTAT 0x0000 42c2afad6cSKamil Konieczny #define SSS_FCINTSTAT_HPARTINT BIT(7) 43c2afad6cSKamil Konieczny #define SSS_FCINTSTAT_HDONEINT BIT(5) 445e00c604SKrzysztof Kozlowski #define SSS_FCINTSTAT_BRDMAINT BIT(3) 455e00c604SKrzysztof Kozlowski #define SSS_FCINTSTAT_BTDMAINT BIT(2) 465e00c604SKrzysztof Kozlowski #define SSS_FCINTSTAT_HRDMAINT BIT(1) 475e00c604SKrzysztof Kozlowski #define SSS_FCINTSTAT_PKDMAINT BIT(0) 48a49e490cSVladimir Zapolskiy 49a49e490cSVladimir Zapolskiy #define SSS_REG_FCINTENSET 0x0004 50c2afad6cSKamil Konieczny #define SSS_FCINTENSET_HPARTINTENSET BIT(7) 51c2afad6cSKamil Konieczny #define SSS_FCINTENSET_HDONEINTENSET BIT(5) 525e00c604SKrzysztof Kozlowski #define SSS_FCINTENSET_BRDMAINTENSET BIT(3) 535e00c604SKrzysztof Kozlowski #define SSS_FCINTENSET_BTDMAINTENSET BIT(2) 545e00c604SKrzysztof Kozlowski #define SSS_FCINTENSET_HRDMAINTENSET BIT(1) 555e00c604SKrzysztof Kozlowski #define SSS_FCINTENSET_PKDMAINTENSET BIT(0) 56a49e490cSVladimir Zapolskiy 57a49e490cSVladimir Zapolskiy #define SSS_REG_FCINTENCLR 0x0008 58c2afad6cSKamil Konieczny #define SSS_FCINTENCLR_HPARTINTENCLR BIT(7) 59c2afad6cSKamil Konieczny #define SSS_FCINTENCLR_HDONEINTENCLR BIT(5) 605e00c604SKrzysztof Kozlowski #define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) 615e00c604SKrzysztof Kozlowski #define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) 625e00c604SKrzysztof Kozlowski #define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) 635e00c604SKrzysztof Kozlowski #define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) 64a49e490cSVladimir Zapolskiy 65a49e490cSVladimir Zapolskiy #define SSS_REG_FCINTPEND 0x000C 66c2afad6cSKamil Konieczny #define SSS_FCINTPEND_HPARTINTP BIT(7) 67c2afad6cSKamil Konieczny #define SSS_FCINTPEND_HDONEINTP BIT(5) 685e00c604SKrzysztof Kozlowski #define SSS_FCINTPEND_BRDMAINTP BIT(3) 695e00c604SKrzysztof Kozlowski #define SSS_FCINTPEND_BTDMAINTP BIT(2) 705e00c604SKrzysztof Kozlowski #define SSS_FCINTPEND_HRDMAINTP BIT(1) 715e00c604SKrzysztof Kozlowski #define SSS_FCINTPEND_PKDMAINTP BIT(0) 72a49e490cSVladimir Zapolskiy 73a49e490cSVladimir Zapolskiy #define SSS_REG_FCFIFOSTAT 0x0010 745e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7) 755e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6) 765e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5) 775e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4) 785e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3) 795e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2) 805e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1) 815e00c604SKrzysztof Kozlowski #define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0) 82a49e490cSVladimir Zapolskiy 83a49e490cSVladimir Zapolskiy #define SSS_REG_FCFIFOCTRL 0x0014 845e00c604SKrzysztof Kozlowski #define SSS_FCFIFOCTRL_DESSEL BIT(2) 85a49e490cSVladimir Zapolskiy #define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) 86a49e490cSVladimir Zapolskiy #define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) 87a49e490cSVladimir Zapolskiy #define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) 88c2afad6cSKamil Konieczny #define SSS_HASHIN_MASK _SBF(0, 0x03) 89a49e490cSVladimir Zapolskiy 90a49e490cSVladimir Zapolskiy #define SSS_REG_FCBRDMAS 0x0020 91a49e490cSVladimir Zapolskiy #define SSS_REG_FCBRDMAL 0x0024 92a49e490cSVladimir Zapolskiy #define SSS_REG_FCBRDMAC 0x0028 935e00c604SKrzysztof Kozlowski #define SSS_FCBRDMAC_BYTESWAP BIT(1) 945e00c604SKrzysztof Kozlowski #define SSS_FCBRDMAC_FLUSH BIT(0) 95a49e490cSVladimir Zapolskiy 96a49e490cSVladimir Zapolskiy #define SSS_REG_FCBTDMAS 0x0030 97a49e490cSVladimir Zapolskiy #define SSS_REG_FCBTDMAL 0x0034 98a49e490cSVladimir Zapolskiy #define SSS_REG_FCBTDMAC 0x0038 995e00c604SKrzysztof Kozlowski #define SSS_FCBTDMAC_BYTESWAP BIT(1) 1005e00c604SKrzysztof Kozlowski #define SSS_FCBTDMAC_FLUSH BIT(0) 101a49e490cSVladimir Zapolskiy 102a49e490cSVladimir Zapolskiy #define SSS_REG_FCHRDMAS 0x0040 103a49e490cSVladimir Zapolskiy #define SSS_REG_FCHRDMAL 0x0044 104a49e490cSVladimir Zapolskiy #define SSS_REG_FCHRDMAC 0x0048 1055e00c604SKrzysztof Kozlowski #define SSS_FCHRDMAC_BYTESWAP BIT(1) 1065e00c604SKrzysztof Kozlowski #define SSS_FCHRDMAC_FLUSH BIT(0) 107a49e490cSVladimir Zapolskiy 108a49e490cSVladimir Zapolskiy #define SSS_REG_FCPKDMAS 0x0050 109a49e490cSVladimir Zapolskiy #define SSS_REG_FCPKDMAL 0x0054 110a49e490cSVladimir Zapolskiy #define SSS_REG_FCPKDMAC 0x0058 1115e00c604SKrzysztof Kozlowski #define SSS_FCPKDMAC_BYTESWAP BIT(3) 1125e00c604SKrzysztof Kozlowski #define SSS_FCPKDMAC_DESCEND BIT(2) 1135e00c604SKrzysztof Kozlowski #define SSS_FCPKDMAC_TRANSMIT BIT(1) 1145e00c604SKrzysztof Kozlowski #define SSS_FCPKDMAC_FLUSH BIT(0) 115a49e490cSVladimir Zapolskiy 116a49e490cSVladimir Zapolskiy #define SSS_REG_FCPKDMAO 0x005C 117a49e490cSVladimir Zapolskiy 118a49e490cSVladimir Zapolskiy /* AES registers */ 11989245107SNaveen Krishna Chatradhi #define SSS_REG_AES_CONTROL 0x00 1205e00c604SKrzysztof Kozlowski #define SSS_AES_BYTESWAP_DI BIT(11) 1215e00c604SKrzysztof Kozlowski #define SSS_AES_BYTESWAP_DO BIT(10) 1225e00c604SKrzysztof Kozlowski #define SSS_AES_BYTESWAP_IV BIT(9) 1235e00c604SKrzysztof Kozlowski #define SSS_AES_BYTESWAP_CNT BIT(8) 1245e00c604SKrzysztof Kozlowski #define SSS_AES_BYTESWAP_KEY BIT(7) 1255e00c604SKrzysztof Kozlowski #define SSS_AES_KEY_CHANGE_MODE BIT(6) 126a49e490cSVladimir Zapolskiy #define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00) 127a49e490cSVladimir Zapolskiy #define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01) 128a49e490cSVladimir Zapolskiy #define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02) 1295e00c604SKrzysztof Kozlowski #define SSS_AES_FIFO_MODE BIT(3) 130a49e490cSVladimir Zapolskiy #define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00) 131a49e490cSVladimir Zapolskiy #define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01) 132a49e490cSVladimir Zapolskiy #define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) 1335e00c604SKrzysztof Kozlowski #define SSS_AES_MODE_DECRYPT BIT(0) 134a49e490cSVladimir Zapolskiy 13589245107SNaveen Krishna Chatradhi #define SSS_REG_AES_STATUS 0x04 1365e00c604SKrzysztof Kozlowski #define SSS_AES_BUSY BIT(2) 1375e00c604SKrzysztof Kozlowski #define SSS_AES_INPUT_READY BIT(1) 1385e00c604SKrzysztof Kozlowski #define SSS_AES_OUTPUT_READY BIT(0) 139a49e490cSVladimir Zapolskiy 14089245107SNaveen Krishna Chatradhi #define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2)) 14189245107SNaveen Krishna Chatradhi #define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2)) 14289245107SNaveen Krishna Chatradhi #define SSS_REG_AES_IV_DATA(s) (0x30 + (s << 2)) 14389245107SNaveen Krishna Chatradhi #define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2)) 14489245107SNaveen Krishna Chatradhi #define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2)) 145a49e490cSVladimir Zapolskiy 146a49e490cSVladimir Zapolskiy #define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) 147a49e490cSVladimir Zapolskiy #define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) 148a49e490cSVladimir Zapolskiy #define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) 149a49e490cSVladimir Zapolskiy 15089245107SNaveen Krishna Chatradhi #define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) 15189245107SNaveen Krishna Chatradhi #define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \ 15289245107SNaveen Krishna Chatradhi SSS_AES_REG(dev, reg)) 15389245107SNaveen Krishna Chatradhi 154a49e490cSVladimir Zapolskiy /* HW engine modes */ 1555e00c604SKrzysztof Kozlowski #define FLAGS_AES_DECRYPT BIT(0) 156a49e490cSVladimir Zapolskiy #define FLAGS_AES_MODE_MASK _SBF(1, 0x03) 157a49e490cSVladimir Zapolskiy #define FLAGS_AES_CBC _SBF(1, 0x01) 158a49e490cSVladimir Zapolskiy #define FLAGS_AES_CTR _SBF(1, 0x02) 159a49e490cSVladimir Zapolskiy 160a49e490cSVladimir Zapolskiy #define AES_KEY_LEN 16 161a49e490cSVladimir Zapolskiy #define CRYPTO_QUEUE_LEN 1 162a49e490cSVladimir Zapolskiy 163c2afad6cSKamil Konieczny /* HASH registers */ 164c2afad6cSKamil Konieczny #define SSS_REG_HASH_CTRL 0x00 165c2afad6cSKamil Konieczny 166c2afad6cSKamil Konieczny #define SSS_HASH_USER_IV_EN BIT(5) 167c2afad6cSKamil Konieczny #define SSS_HASH_INIT_BIT BIT(4) 168c2afad6cSKamil Konieczny #define SSS_HASH_ENGINE_SHA1 _SBF(1, 0x00) 169c2afad6cSKamil Konieczny #define SSS_HASH_ENGINE_MD5 _SBF(1, 0x01) 170c2afad6cSKamil Konieczny #define SSS_HASH_ENGINE_SHA256 _SBF(1, 0x02) 171c2afad6cSKamil Konieczny 172c2afad6cSKamil Konieczny #define SSS_HASH_ENGINE_MASK _SBF(1, 0x03) 173c2afad6cSKamil Konieczny 174c2afad6cSKamil Konieczny #define SSS_REG_HASH_CTRL_PAUSE 0x04 175c2afad6cSKamil Konieczny 176c2afad6cSKamil Konieczny #define SSS_HASH_PAUSE BIT(0) 177c2afad6cSKamil Konieczny 178c2afad6cSKamil Konieczny #define SSS_REG_HASH_CTRL_FIFO 0x08 179c2afad6cSKamil Konieczny 180c2afad6cSKamil Konieczny #define SSS_HASH_FIFO_MODE_DMA BIT(0) 181c2afad6cSKamil Konieczny #define SSS_HASH_FIFO_MODE_CPU 0 182c2afad6cSKamil Konieczny 183c2afad6cSKamil Konieczny #define SSS_REG_HASH_CTRL_SWAP 0x0C 184c2afad6cSKamil Konieczny 185c2afad6cSKamil Konieczny #define SSS_HASH_BYTESWAP_DI BIT(3) 186c2afad6cSKamil Konieczny #define SSS_HASH_BYTESWAP_DO BIT(2) 187c2afad6cSKamil Konieczny #define SSS_HASH_BYTESWAP_IV BIT(1) 188c2afad6cSKamil Konieczny #define SSS_HASH_BYTESWAP_KEY BIT(0) 189c2afad6cSKamil Konieczny 190c2afad6cSKamil Konieczny #define SSS_REG_HASH_STATUS 0x10 191c2afad6cSKamil Konieczny 192c2afad6cSKamil Konieczny #define SSS_HASH_STATUS_MSG_DONE BIT(6) 193c2afad6cSKamil Konieczny #define SSS_HASH_STATUS_PARTIAL_DONE BIT(4) 194c2afad6cSKamil Konieczny #define SSS_HASH_STATUS_BUFFER_READY BIT(0) 195c2afad6cSKamil Konieczny 196c2afad6cSKamil Konieczny #define SSS_REG_HASH_MSG_SIZE_LOW 0x20 197c2afad6cSKamil Konieczny #define SSS_REG_HASH_MSG_SIZE_HIGH 0x24 198c2afad6cSKamil Konieczny 199c2afad6cSKamil Konieczny #define SSS_REG_HASH_PRE_MSG_SIZE_LOW 0x28 200c2afad6cSKamil Konieczny #define SSS_REG_HASH_PRE_MSG_SIZE_HIGH 0x2C 201c2afad6cSKamil Konieczny 202c2afad6cSKamil Konieczny #define SSS_REG_HASH_IV(s) (0xB0 + ((s) << 2)) 203c2afad6cSKamil Konieczny #define SSS_REG_HASH_OUT(s) (0x100 + ((s) << 2)) 204c2afad6cSKamil Konieczny 205c2afad6cSKamil Konieczny #define HASH_BLOCK_SIZE 64 206c2afad6cSKamil Konieczny #define HASH_REG_SIZEOF 4 207c2afad6cSKamil Konieczny #define HASH_MD5_MAX_REG (MD5_DIGEST_SIZE / HASH_REG_SIZEOF) 208c2afad6cSKamil Konieczny #define HASH_SHA1_MAX_REG (SHA1_DIGEST_SIZE / HASH_REG_SIZEOF) 209c2afad6cSKamil Konieczny #define HASH_SHA256_MAX_REG (SHA256_DIGEST_SIZE / HASH_REG_SIZEOF) 210c2afad6cSKamil Konieczny 211c2afad6cSKamil Konieczny /* 212c2afad6cSKamil Konieczny * HASH bit numbers, used by device, setting in dev->hash_flags with 213c2afad6cSKamil Konieczny * functions set_bit(), clear_bit() or tested with test_bit() or BIT(), 214c2afad6cSKamil Konieczny * to keep HASH state BUSY or FREE, or to signal state from irq_handler 215c2afad6cSKamil Konieczny * to hash_tasklet. SGS keep track of allocated memory for scatterlist 216c2afad6cSKamil Konieczny */ 217c2afad6cSKamil Konieczny #define HASH_FLAGS_BUSY 0 218c2afad6cSKamil Konieczny #define HASH_FLAGS_FINAL 1 219c2afad6cSKamil Konieczny #define HASH_FLAGS_DMA_ACTIVE 2 220c2afad6cSKamil Konieczny #define HASH_FLAGS_OUTPUT_READY 3 221c2afad6cSKamil Konieczny #define HASH_FLAGS_DMA_READY 4 222c2afad6cSKamil Konieczny #define HASH_FLAGS_SGS_COPIED 5 223c2afad6cSKamil Konieczny #define HASH_FLAGS_SGS_ALLOCED 6 224c2afad6cSKamil Konieczny 225c2afad6cSKamil Konieczny /* HASH HW constants */ 226c2afad6cSKamil Konieczny #define BUFLEN HASH_BLOCK_SIZE 227c2afad6cSKamil Konieczny 228c2afad6cSKamil Konieczny #define SSS_HASH_DMA_LEN_ALIGN 8 229c2afad6cSKamil Konieczny #define SSS_HASH_DMA_ALIGN_MASK (SSS_HASH_DMA_LEN_ALIGN - 1) 230c2afad6cSKamil Konieczny 231c2afad6cSKamil Konieczny #define SSS_HASH_QUEUE_LENGTH 10 232c2afad6cSKamil Konieczny 23389245107SNaveen Krishna Chatradhi /** 23489245107SNaveen Krishna Chatradhi * struct samsung_aes_variant - platform specific SSS driver data 23589245107SNaveen Krishna Chatradhi * @aes_offset: AES register offset from SSS module's base. 236c2afad6cSKamil Konieczny * @hash_offset: HASH register offset from SSS module's base. 2370918f18cSKamil Konieczny * @clk_names: names of clocks needed to run SSS IP 23889245107SNaveen Krishna Chatradhi * 23989245107SNaveen Krishna Chatradhi * Specifies platform specific configuration of SSS module. 24089245107SNaveen Krishna Chatradhi * Note: A structure for driver specific platform data is used for future 24189245107SNaveen Krishna Chatradhi * expansion of its usage. 24289245107SNaveen Krishna Chatradhi */ 24389245107SNaveen Krishna Chatradhi struct samsung_aes_variant { 24489245107SNaveen Krishna Chatradhi unsigned int aes_offset; 245c2afad6cSKamil Konieczny unsigned int hash_offset; 246aa1abbe0SKamil Konieczny const char *clk_names[2]; 24789245107SNaveen Krishna Chatradhi }; 24889245107SNaveen Krishna Chatradhi 249a49e490cSVladimir Zapolskiy struct s5p_aes_reqctx { 250a49e490cSVladimir Zapolskiy unsigned long mode; 251a49e490cSVladimir Zapolskiy }; 252a49e490cSVladimir Zapolskiy 253a49e490cSVladimir Zapolskiy struct s5p_aes_ctx { 254a49e490cSVladimir Zapolskiy struct s5p_aes_dev *dev; 255a49e490cSVladimir Zapolskiy 256b1b4416fSChristoph Manszewski u8 aes_key[AES_MAX_KEY_SIZE]; 257b1b4416fSChristoph Manszewski u8 nonce[CTR_RFC3686_NONCE_SIZE]; 258a49e490cSVladimir Zapolskiy int keylen; 259a49e490cSVladimir Zapolskiy }; 260a49e490cSVladimir Zapolskiy 261106d7334SKrzysztof Kozlowski /** 262106d7334SKrzysztof Kozlowski * struct s5p_aes_dev - Crypto device state container 263106d7334SKrzysztof Kozlowski * @dev: Associated device 264106d7334SKrzysztof Kozlowski * @clk: Clock for accessing hardware 26598a6bc10SKrzysztof Kozlowski * @pclk: APB bus clock necessary to access the hardware 266106d7334SKrzysztof Kozlowski * @ioaddr: Mapped IO memory region 267106d7334SKrzysztof Kozlowski * @aes_ioaddr: Per-varian offset for AES block IO memory 268106d7334SKrzysztof Kozlowski * @irq_fc: Feed control interrupt line 269106d7334SKrzysztof Kozlowski * @req: Crypto request currently handled by the device 270106d7334SKrzysztof Kozlowski * @ctx: Configuration for currently handled crypto request 271106d7334SKrzysztof Kozlowski * @sg_src: Scatter list with source data for currently handled block 272106d7334SKrzysztof Kozlowski * in device. This is DMA-mapped into device. 273106d7334SKrzysztof Kozlowski * @sg_dst: Scatter list with destination data for currently handled block 274106d7334SKrzysztof Kozlowski * in device. This is DMA-mapped into device. 275106d7334SKrzysztof Kozlowski * @sg_src_cpy: In case of unaligned access, copied scatter list 276106d7334SKrzysztof Kozlowski * with source data. 277106d7334SKrzysztof Kozlowski * @sg_dst_cpy: In case of unaligned access, copied scatter list 278106d7334SKrzysztof Kozlowski * with destination data. 279106d7334SKrzysztof Kozlowski * @tasklet: New request scheduling jib 280106d7334SKrzysztof Kozlowski * @queue: Crypto queue 281106d7334SKrzysztof Kozlowski * @busy: Indicates whether the device is currently handling some request 282106d7334SKrzysztof Kozlowski * thus it uses some of the fields from this state, like: 283106d7334SKrzysztof Kozlowski * req, ctx, sg_src/dst (and copies). This essentially 284106d7334SKrzysztof Kozlowski * protects against concurrent access to these fields. 285106d7334SKrzysztof Kozlowski * @lock: Lock for protecting both access to device hardware registers 286106d7334SKrzysztof Kozlowski * and fields related to current request (including the busy field). 287c2afad6cSKamil Konieczny * @res: Resources for hash. 288c2afad6cSKamil Konieczny * @io_hash_base: Per-variant offset for HASH block IO memory. 289c2afad6cSKamil Konieczny * @hash_lock: Lock for protecting hash_req, hash_queue and hash_flags 290c2afad6cSKamil Konieczny * variable. 291c2afad6cSKamil Konieczny * @hash_flags: Flags for current HASH op. 292c2afad6cSKamil Konieczny * @hash_queue: Async hash queue. 293c2afad6cSKamil Konieczny * @hash_tasklet: New HASH request scheduling job. 294c2afad6cSKamil Konieczny * @xmit_buf: Buffer for current HASH request transfer into SSS block. 295c2afad6cSKamil Konieczny * @hash_req: Current request sending to SSS HASH block. 296c2afad6cSKamil Konieczny * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block. 297c2afad6cSKamil Konieczny * @hash_sg_cnt: Counter for hash_sg_iter. 298c2afad6cSKamil Konieczny * 299c2afad6cSKamil Konieczny * @use_hash: true if HASH algs enabled 300106d7334SKrzysztof Kozlowski */ 301a49e490cSVladimir Zapolskiy struct s5p_aes_dev { 302a49e490cSVladimir Zapolskiy struct device *dev; 303a49e490cSVladimir Zapolskiy struct clk *clk; 3040918f18cSKamil Konieczny struct clk *pclk; 305a49e490cSVladimir Zapolskiy void __iomem *ioaddr; 30689245107SNaveen Krishna Chatradhi void __iomem *aes_ioaddr; 307a49e490cSVladimir Zapolskiy int irq_fc; 308a49e490cSVladimir Zapolskiy 309e6b98ce6SArd Biesheuvel struct skcipher_request *req; 310a49e490cSVladimir Zapolskiy struct s5p_aes_ctx *ctx; 311a49e490cSVladimir Zapolskiy struct scatterlist *sg_src; 312a49e490cSVladimir Zapolskiy struct scatterlist *sg_dst; 313a49e490cSVladimir Zapolskiy 3149e4a1100SKrzysztof Kozlowski struct scatterlist *sg_src_cpy; 3159e4a1100SKrzysztof Kozlowski struct scatterlist *sg_dst_cpy; 3169e4a1100SKrzysztof Kozlowski 317a49e490cSVladimir Zapolskiy struct tasklet_struct tasklet; 318a49e490cSVladimir Zapolskiy struct crypto_queue queue; 319a49e490cSVladimir Zapolskiy bool busy; 320a49e490cSVladimir Zapolskiy spinlock_t lock; 321c2afad6cSKamil Konieczny 322c2afad6cSKamil Konieczny struct resource *res; 323c2afad6cSKamil Konieczny void __iomem *io_hash_base; 324c2afad6cSKamil Konieczny 325c2afad6cSKamil Konieczny spinlock_t hash_lock; /* protect hash_ vars */ 326c2afad6cSKamil Konieczny unsigned long hash_flags; 327c2afad6cSKamil Konieczny struct crypto_queue hash_queue; 328c2afad6cSKamil Konieczny struct tasklet_struct hash_tasklet; 329c2afad6cSKamil Konieczny 330c2afad6cSKamil Konieczny u8 xmit_buf[BUFLEN]; 331c2afad6cSKamil Konieczny struct ahash_request *hash_req; 332c2afad6cSKamil Konieczny struct scatterlist *hash_sg_iter; 333c2afad6cSKamil Konieczny unsigned int hash_sg_cnt; 334c2afad6cSKamil Konieczny 335c2afad6cSKamil Konieczny bool use_hash; 336a49e490cSVladimir Zapolskiy }; 337a49e490cSVladimir Zapolskiy 338c2afad6cSKamil Konieczny /** 339c2afad6cSKamil Konieczny * struct s5p_hash_reqctx - HASH request context 340c2afad6cSKamil Konieczny * @dd: Associated device 341c2afad6cSKamil Konieczny * @op_update: Current request operation (OP_UPDATE or OP_FINAL) 342c2afad6cSKamil Konieczny * @digcnt: Number of bytes processed by HW (without buffer[] ones) 343c2afad6cSKamil Konieczny * @digest: Digest message or IV for partial result 344c2afad6cSKamil Konieczny * @nregs: Number of HW registers for digest or IV read/write 345c2afad6cSKamil Konieczny * @engine: Bits for selecting type of HASH in SSS block 346c2afad6cSKamil Konieczny * @sg: sg for DMA transfer 347c2afad6cSKamil Konieczny * @sg_len: Length of sg for DMA transfer 34898a6bc10SKrzysztof Kozlowski * @sgl: sg for joining buffer and req->src scatterlist 349c2afad6cSKamil Konieczny * @skip: Skip offset in req->src for current op 350c2afad6cSKamil Konieczny * @total: Total number of bytes for current request 351c2afad6cSKamil Konieczny * @finup: Keep state for finup or final. 352c2afad6cSKamil Konieczny * @error: Keep track of error. 353c2afad6cSKamil Konieczny * @bufcnt: Number of bytes holded in buffer[] 35498a6bc10SKrzysztof Kozlowski * @buffer: For byte(s) from end of req->src in UPDATE op 355c2afad6cSKamil Konieczny */ 356c2afad6cSKamil Konieczny struct s5p_hash_reqctx { 357c2afad6cSKamil Konieczny struct s5p_aes_dev *dd; 358c2afad6cSKamil Konieczny bool op_update; 359c2afad6cSKamil Konieczny 360c2afad6cSKamil Konieczny u64 digcnt; 361c2afad6cSKamil Konieczny u8 digest[SHA256_DIGEST_SIZE]; 362c2afad6cSKamil Konieczny 363c2afad6cSKamil Konieczny unsigned int nregs; /* digest_size / sizeof(reg) */ 364c2afad6cSKamil Konieczny u32 engine; 365c2afad6cSKamil Konieczny 366c2afad6cSKamil Konieczny struct scatterlist *sg; 367c2afad6cSKamil Konieczny unsigned int sg_len; 368c2afad6cSKamil Konieczny struct scatterlist sgl[2]; 369c2afad6cSKamil Konieczny unsigned int skip; 370c2afad6cSKamil Konieczny unsigned int total; 371c2afad6cSKamil Konieczny bool finup; 372c2afad6cSKamil Konieczny bool error; 373c2afad6cSKamil Konieczny 374c2afad6cSKamil Konieczny u32 bufcnt; 375a4a70fa9SGustavo A. R. Silva u8 buffer[]; 376c2afad6cSKamil Konieczny }; 377c2afad6cSKamil Konieczny 378c2afad6cSKamil Konieczny /** 379c2afad6cSKamil Konieczny * struct s5p_hash_ctx - HASH transformation context 380c2afad6cSKamil Konieczny * @dd: Associated device 381c2afad6cSKamil Konieczny * @flags: Bits for algorithm HASH. 382c2afad6cSKamil Konieczny * @fallback: Software transformation for zero message or size < BUFLEN. 383c2afad6cSKamil Konieczny */ 384c2afad6cSKamil Konieczny struct s5p_hash_ctx { 385c2afad6cSKamil Konieczny struct s5p_aes_dev *dd; 386c2afad6cSKamil Konieczny unsigned long flags; 387c2afad6cSKamil Konieczny struct crypto_shash *fallback; 388c2afad6cSKamil Konieczny }; 389a49e490cSVladimir Zapolskiy 39089245107SNaveen Krishna Chatradhi static const struct samsung_aes_variant s5p_aes_data = { 39189245107SNaveen Krishna Chatradhi .aes_offset = 0x4000, 392c2afad6cSKamil Konieczny .hash_offset = 0x6000, 3930918f18cSKamil Konieczny .clk_names = { "secss", }, 39489245107SNaveen Krishna Chatradhi }; 39589245107SNaveen Krishna Chatradhi 39689245107SNaveen Krishna Chatradhi static const struct samsung_aes_variant exynos_aes_data = { 39789245107SNaveen Krishna Chatradhi .aes_offset = 0x200, 398c2afad6cSKamil Konieczny .hash_offset = 0x400, 3990918f18cSKamil Konieczny .clk_names = { "secss", }, 4000918f18cSKamil Konieczny }; 4010918f18cSKamil Konieczny 4020918f18cSKamil Konieczny static const struct samsung_aes_variant exynos5433_slim_aes_data = { 4030918f18cSKamil Konieczny .aes_offset = 0x400, 4040918f18cSKamil Konieczny .hash_offset = 0x800, 405664b0f41SKrzysztof Kozlowski .clk_names = { "aclk", "pclk", }, 40689245107SNaveen Krishna Chatradhi }; 40789245107SNaveen Krishna Chatradhi 4086b9f16e6SNaveen Krishna Chatradhi static const struct of_device_id s5p_sss_dt_match[] = { 40989245107SNaveen Krishna Chatradhi { 41089245107SNaveen Krishna Chatradhi .compatible = "samsung,s5pv210-secss", 41189245107SNaveen Krishna Chatradhi .data = &s5p_aes_data, 41289245107SNaveen Krishna Chatradhi }, 41389245107SNaveen Krishna Chatradhi { 41489245107SNaveen Krishna Chatradhi .compatible = "samsung,exynos4210-secss", 41589245107SNaveen Krishna Chatradhi .data = &exynos_aes_data, 41689245107SNaveen Krishna Chatradhi }, 4170918f18cSKamil Konieczny { 4180918f18cSKamil Konieczny .compatible = "samsung,exynos5433-slim-sss", 4190918f18cSKamil Konieczny .data = &exynos5433_slim_aes_data, 4200918f18cSKamil Konieczny }, 4216b9f16e6SNaveen Krishna Chatradhi { }, 4226b9f16e6SNaveen Krishna Chatradhi }; 4236b9f16e6SNaveen Krishna Chatradhi MODULE_DEVICE_TABLE(of, s5p_sss_dt_match); 4246b9f16e6SNaveen Krishna Chatradhi 4256584eacbSKrzysztof Kozlowski static inline const struct samsung_aes_variant *find_s5p_sss_version 4266584eacbSKrzysztof Kozlowski (const struct platform_device *pdev) 42789245107SNaveen Krishna Chatradhi { 4286b238db7SKrzysztof Kozlowski if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) 4296b238db7SKrzysztof Kozlowski return of_device_get_match_data(&pdev->dev); 430313becd1SKrzysztof Koz?owski 4316584eacbSKrzysztof Kozlowski return (const struct samsung_aes_variant *) 43289245107SNaveen Krishna Chatradhi platform_get_device_id(pdev)->driver_data; 43389245107SNaveen Krishna Chatradhi } 43489245107SNaveen Krishna Chatradhi 435c2afad6cSKamil Konieczny static struct s5p_aes_dev *s5p_dev; 436c2afad6cSKamil Konieczny 4376584eacbSKrzysztof Kozlowski static void s5p_set_dma_indata(struct s5p_aes_dev *dev, 4386584eacbSKrzysztof Kozlowski const struct scatterlist *sg) 439a49e490cSVladimir Zapolskiy { 440a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg)); 441a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg)); 442a49e490cSVladimir Zapolskiy } 443a49e490cSVladimir Zapolskiy 4446584eacbSKrzysztof Kozlowski static void s5p_set_dma_outdata(struct s5p_aes_dev *dev, 4456584eacbSKrzysztof Kozlowski const struct scatterlist *sg) 446a49e490cSVladimir Zapolskiy { 447a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg)); 448a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg)); 449a49e490cSVladimir Zapolskiy } 450a49e490cSVladimir Zapolskiy 4519e4a1100SKrzysztof Kozlowski static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg) 4529e4a1100SKrzysztof Kozlowski { 4539e4a1100SKrzysztof Kozlowski int len; 4549e4a1100SKrzysztof Kozlowski 4559e4a1100SKrzysztof Kozlowski if (!*sg) 4569e4a1100SKrzysztof Kozlowski return; 4579e4a1100SKrzysztof Kozlowski 458e6b98ce6SArd Biesheuvel len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE); 4599e4a1100SKrzysztof Kozlowski free_pages((unsigned long)sg_virt(*sg), get_order(len)); 4609e4a1100SKrzysztof Kozlowski 4619e4a1100SKrzysztof Kozlowski kfree(*sg); 4629e4a1100SKrzysztof Kozlowski *sg = NULL; 4639e4a1100SKrzysztof Kozlowski } 4649e4a1100SKrzysztof Kozlowski 4659e4a1100SKrzysztof Kozlowski static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg, 4669e4a1100SKrzysztof Kozlowski unsigned int nbytes, int out) 4679e4a1100SKrzysztof Kozlowski { 4689e4a1100SKrzysztof Kozlowski struct scatter_walk walk; 4699e4a1100SKrzysztof Kozlowski 4709e4a1100SKrzysztof Kozlowski if (!nbytes) 4719e4a1100SKrzysztof Kozlowski return; 4729e4a1100SKrzysztof Kozlowski 4739e4a1100SKrzysztof Kozlowski scatterwalk_start(&walk, sg); 4749e4a1100SKrzysztof Kozlowski scatterwalk_copychunks(buf, &walk, nbytes, out); 4759e4a1100SKrzysztof Kozlowski scatterwalk_done(&walk, out, 0); 4769e4a1100SKrzysztof Kozlowski } 4779e4a1100SKrzysztof Kozlowski 47828b62b14SKrzysztof Kozlowski static void s5p_sg_done(struct s5p_aes_dev *dev) 479a49e490cSVladimir Zapolskiy { 480e6b98ce6SArd Biesheuvel struct skcipher_request *req = dev->req; 481e6b98ce6SArd Biesheuvel struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req); 482e8e3c1caSKamil Konieczny 4839e4a1100SKrzysztof Kozlowski if (dev->sg_dst_cpy) { 4849e4a1100SKrzysztof Kozlowski dev_dbg(dev->dev, 4859e4a1100SKrzysztof Kozlowski "Copying %d bytes of output data back to original place\n", 486e6b98ce6SArd Biesheuvel dev->req->cryptlen); 4879e4a1100SKrzysztof Kozlowski s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst, 488e6b98ce6SArd Biesheuvel dev->req->cryptlen, 1); 4899e4a1100SKrzysztof Kozlowski } 4909e4a1100SKrzysztof Kozlowski s5p_free_sg_cpy(dev, &dev->sg_src_cpy); 4919e4a1100SKrzysztof Kozlowski s5p_free_sg_cpy(dev, &dev->sg_dst_cpy); 492e8e3c1caSKamil Konieczny if (reqctx->mode & FLAGS_AES_CBC) 493e6b98ce6SArd Biesheuvel memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), AES_BLOCK_SIZE); 494e8e3c1caSKamil Konieczny 495e8e3c1caSKamil Konieczny else if (reqctx->mode & FLAGS_AES_CTR) 496e6b98ce6SArd Biesheuvel memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), AES_BLOCK_SIZE); 49728b62b14SKrzysztof Kozlowski } 4989e4a1100SKrzysztof Kozlowski 49928b62b14SKrzysztof Kozlowski /* Calls the completion. Cannot be called with dev->lock hold. */ 500e6b98ce6SArd Biesheuvel static void s5p_aes_complete(struct skcipher_request *req, int err) 50128b62b14SKrzysztof Kozlowski { 5025842cd44SChristoph Manszewski req->base.complete(&req->base, err); 503a49e490cSVladimir Zapolskiy } 504a49e490cSVladimir Zapolskiy 505a49e490cSVladimir Zapolskiy static void s5p_unset_outdata(struct s5p_aes_dev *dev) 506a49e490cSVladimir Zapolskiy { 507a49e490cSVladimir Zapolskiy dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE); 508a49e490cSVladimir Zapolskiy } 509a49e490cSVladimir Zapolskiy 510a49e490cSVladimir Zapolskiy static void s5p_unset_indata(struct s5p_aes_dev *dev) 511a49e490cSVladimir Zapolskiy { 512a49e490cSVladimir Zapolskiy dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE); 513a49e490cSVladimir Zapolskiy } 514a49e490cSVladimir Zapolskiy 5159e4a1100SKrzysztof Kozlowski static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src, 5169e4a1100SKrzysztof Kozlowski struct scatterlist **dst) 5179e4a1100SKrzysztof Kozlowski { 5189e4a1100SKrzysztof Kozlowski void *pages; 5199e4a1100SKrzysztof Kozlowski int len; 5209e4a1100SKrzysztof Kozlowski 5219e4a1100SKrzysztof Kozlowski *dst = kmalloc(sizeof(**dst), GFP_ATOMIC); 5229e4a1100SKrzysztof Kozlowski if (!*dst) 5239e4a1100SKrzysztof Kozlowski return -ENOMEM; 5249e4a1100SKrzysztof Kozlowski 525e6b98ce6SArd Biesheuvel len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE); 5269e4a1100SKrzysztof Kozlowski pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len)); 5279e4a1100SKrzysztof Kozlowski if (!pages) { 5289e4a1100SKrzysztof Kozlowski kfree(*dst); 5299e4a1100SKrzysztof Kozlowski *dst = NULL; 5309e4a1100SKrzysztof Kozlowski return -ENOMEM; 5319e4a1100SKrzysztof Kozlowski } 5329e4a1100SKrzysztof Kozlowski 533e6b98ce6SArd Biesheuvel s5p_sg_copy_buf(pages, src, dev->req->cryptlen, 0); 5349e4a1100SKrzysztof Kozlowski 5359e4a1100SKrzysztof Kozlowski sg_init_table(*dst, 1); 5369e4a1100SKrzysztof Kozlowski sg_set_buf(*dst, pages, len); 5379e4a1100SKrzysztof Kozlowski 5389e4a1100SKrzysztof Kozlowski return 0; 5399e4a1100SKrzysztof Kozlowski } 5409e4a1100SKrzysztof Kozlowski 541a49e490cSVladimir Zapolskiy static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg) 542a49e490cSVladimir Zapolskiy { 543b1b4416fSChristoph Manszewski if (!sg->length) 544b1b4416fSChristoph Manszewski return -EINVAL; 545a49e490cSVladimir Zapolskiy 546b1b4416fSChristoph Manszewski if (!dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE)) 547b1b4416fSChristoph Manszewski return -ENOMEM; 548a49e490cSVladimir Zapolskiy 549a49e490cSVladimir Zapolskiy dev->sg_dst = sg; 550a49e490cSVladimir Zapolskiy 551b1b4416fSChristoph Manszewski return 0; 552a49e490cSVladimir Zapolskiy } 553a49e490cSVladimir Zapolskiy 554a49e490cSVladimir Zapolskiy static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg) 555a49e490cSVladimir Zapolskiy { 556b1b4416fSChristoph Manszewski if (!sg->length) 557b1b4416fSChristoph Manszewski return -EINVAL; 558a49e490cSVladimir Zapolskiy 559b1b4416fSChristoph Manszewski if (!dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE)) 560b1b4416fSChristoph Manszewski return -ENOMEM; 561a49e490cSVladimir Zapolskiy 562a49e490cSVladimir Zapolskiy dev->sg_src = sg; 563a49e490cSVladimir Zapolskiy 564b1b4416fSChristoph Manszewski return 0; 565a49e490cSVladimir Zapolskiy } 566a49e490cSVladimir Zapolskiy 56779152e8dSKrzysztof Kozlowski /* 56828b62b14SKrzysztof Kozlowski * Returns -ERRNO on error (mapping of new data failed). 56928b62b14SKrzysztof Kozlowski * On success returns: 57028b62b14SKrzysztof Kozlowski * - 0 if there is no more data, 57128b62b14SKrzysztof Kozlowski * - 1 if new transmitting (output) data is ready and its address+length 57228b62b14SKrzysztof Kozlowski * have to be written to device (by calling s5p_set_dma_outdata()). 57379152e8dSKrzysztof Kozlowski */ 57428b62b14SKrzysztof Kozlowski static int s5p_aes_tx(struct s5p_aes_dev *dev) 575a49e490cSVladimir Zapolskiy { 57628b62b14SKrzysztof Kozlowski int ret = 0; 577a49e490cSVladimir Zapolskiy 578a49e490cSVladimir Zapolskiy s5p_unset_outdata(dev); 579a49e490cSVladimir Zapolskiy 580a49e490cSVladimir Zapolskiy if (!sg_is_last(dev->sg_dst)) { 58128b62b14SKrzysztof Kozlowski ret = s5p_set_outdata(dev, sg_next(dev->sg_dst)); 58228b62b14SKrzysztof Kozlowski if (!ret) 58328b62b14SKrzysztof Kozlowski ret = 1; 584dc5e3f19SNaveen Krishna Chatradhi } 58579152e8dSKrzysztof Kozlowski 58679152e8dSKrzysztof Kozlowski return ret; 587a49e490cSVladimir Zapolskiy } 588a49e490cSVladimir Zapolskiy 58979152e8dSKrzysztof Kozlowski /* 59028b62b14SKrzysztof Kozlowski * Returns -ERRNO on error (mapping of new data failed). 59128b62b14SKrzysztof Kozlowski * On success returns: 59228b62b14SKrzysztof Kozlowski * - 0 if there is no more data, 59328b62b14SKrzysztof Kozlowski * - 1 if new receiving (input) data is ready and its address+length 59428b62b14SKrzysztof Kozlowski * have to be written to device (by calling s5p_set_dma_indata()). 59579152e8dSKrzysztof Kozlowski */ 59628b62b14SKrzysztof Kozlowski static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/) 597a49e490cSVladimir Zapolskiy { 59828b62b14SKrzysztof Kozlowski int ret = 0; 599a49e490cSVladimir Zapolskiy 600a49e490cSVladimir Zapolskiy s5p_unset_indata(dev); 601a49e490cSVladimir Zapolskiy 602a49e490cSVladimir Zapolskiy if (!sg_is_last(dev->sg_src)) { 60328b62b14SKrzysztof Kozlowski ret = s5p_set_indata(dev, sg_next(dev->sg_src)); 60428b62b14SKrzysztof Kozlowski if (!ret) 60528b62b14SKrzysztof Kozlowski ret = 1; 606a49e490cSVladimir Zapolskiy } 607a49e490cSVladimir Zapolskiy 60879152e8dSKrzysztof Kozlowski return ret; 609a49e490cSVladimir Zapolskiy } 610a49e490cSVladimir Zapolskiy 611c2afad6cSKamil Konieczny static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset) 612c2afad6cSKamil Konieczny { 613c2afad6cSKamil Konieczny return __raw_readl(dd->io_hash_base + offset); 614c2afad6cSKamil Konieczny } 615c2afad6cSKamil Konieczny 616c2afad6cSKamil Konieczny static inline void s5p_hash_write(struct s5p_aes_dev *dd, 617c2afad6cSKamil Konieczny u32 offset, u32 value) 618c2afad6cSKamil Konieczny { 619c2afad6cSKamil Konieczny __raw_writel(value, dd->io_hash_base + offset); 620c2afad6cSKamil Konieczny } 621c2afad6cSKamil Konieczny 622c2afad6cSKamil Konieczny /** 623c2afad6cSKamil Konieczny * s5p_set_dma_hashdata() - start DMA with sg 624c2afad6cSKamil Konieczny * @dev: device 625c2afad6cSKamil Konieczny * @sg: scatterlist ready to DMA transmit 626c2afad6cSKamil Konieczny */ 627c2afad6cSKamil Konieczny static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev, 6286584eacbSKrzysztof Kozlowski const struct scatterlist *sg) 629c2afad6cSKamil Konieczny { 630c2afad6cSKamil Konieczny dev->hash_sg_cnt--; 631c2afad6cSKamil Konieczny SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg)); 632c2afad6cSKamil Konieczny SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */ 633c2afad6cSKamil Konieczny } 634c2afad6cSKamil Konieczny 635c2afad6cSKamil Konieczny /** 636c2afad6cSKamil Konieczny * s5p_hash_rx() - get next hash_sg_iter 637c2afad6cSKamil Konieczny * @dev: device 638c2afad6cSKamil Konieczny * 639c2afad6cSKamil Konieczny * Return: 640c2afad6cSKamil Konieczny * 2 if there is no more data and it is UPDATE op 641c2afad6cSKamil Konieczny * 1 if new receiving (input) data is ready and can be written to device 642c2afad6cSKamil Konieczny * 0 if there is no more data and it is FINAL op 643c2afad6cSKamil Konieczny */ 644c2afad6cSKamil Konieczny static int s5p_hash_rx(struct s5p_aes_dev *dev) 645c2afad6cSKamil Konieczny { 646c2afad6cSKamil Konieczny if (dev->hash_sg_cnt > 0) { 647c2afad6cSKamil Konieczny dev->hash_sg_iter = sg_next(dev->hash_sg_iter); 648c2afad6cSKamil Konieczny return 1; 649c2afad6cSKamil Konieczny } 650c2afad6cSKamil Konieczny 651c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags); 652c2afad6cSKamil Konieczny if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags)) 653c2afad6cSKamil Konieczny return 0; 654c2afad6cSKamil Konieczny 655c2afad6cSKamil Konieczny return 2; 656c2afad6cSKamil Konieczny } 657c2afad6cSKamil Konieczny 658a49e490cSVladimir Zapolskiy static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) 659a49e490cSVladimir Zapolskiy { 660a49e490cSVladimir Zapolskiy struct platform_device *pdev = dev_id; 661a49e490cSVladimir Zapolskiy struct s5p_aes_dev *dev = platform_get_drvdata(pdev); 662e6b98ce6SArd Biesheuvel struct skcipher_request *req; 66328b62b14SKrzysztof Kozlowski int err_dma_tx = 0; 66428b62b14SKrzysztof Kozlowski int err_dma_rx = 0; 665c2afad6cSKamil Konieczny int err_dma_hx = 0; 66628b62b14SKrzysztof Kozlowski bool tx_end = false; 667c2afad6cSKamil Konieczny bool hx_end = false; 6685318c53dSKrzysztof Kozlowski unsigned long flags; 669b1b4416fSChristoph Manszewski u32 status, st_bits; 67028b62b14SKrzysztof Kozlowski int err; 671a49e490cSVladimir Zapolskiy 672a49e490cSVladimir Zapolskiy spin_lock_irqsave(&dev->lock, flags); 673a49e490cSVladimir Zapolskiy 67428b62b14SKrzysztof Kozlowski /* 67528b62b14SKrzysztof Kozlowski * Handle rx or tx interrupt. If there is still data (scatterlist did not 67628b62b14SKrzysztof Kozlowski * reach end), then map next scatterlist entry. 67728b62b14SKrzysztof Kozlowski * In case of such mapping error, s5p_aes_complete() should be called. 67828b62b14SKrzysztof Kozlowski * 67928b62b14SKrzysztof Kozlowski * If there is no more data in tx scatter list, call s5p_aes_complete() 68028b62b14SKrzysztof Kozlowski * and schedule new tasklet. 681c2afad6cSKamil Konieczny * 682c2afad6cSKamil Konieczny * Handle hx interrupt. If there is still data map next entry. 68328b62b14SKrzysztof Kozlowski */ 684a49e490cSVladimir Zapolskiy status = SSS_READ(dev, FCINTSTAT); 685a49e490cSVladimir Zapolskiy if (status & SSS_FCINTSTAT_BRDMAINT) 68628b62b14SKrzysztof Kozlowski err_dma_rx = s5p_aes_rx(dev); 68728b62b14SKrzysztof Kozlowski 68828b62b14SKrzysztof Kozlowski if (status & SSS_FCINTSTAT_BTDMAINT) { 68928b62b14SKrzysztof Kozlowski if (sg_is_last(dev->sg_dst)) 69028b62b14SKrzysztof Kozlowski tx_end = true; 69128b62b14SKrzysztof Kozlowski err_dma_tx = s5p_aes_tx(dev); 69228b62b14SKrzysztof Kozlowski } 693a49e490cSVladimir Zapolskiy 694c2afad6cSKamil Konieczny if (status & SSS_FCINTSTAT_HRDMAINT) 695c2afad6cSKamil Konieczny err_dma_hx = s5p_hash_rx(dev); 696c2afad6cSKamil Konieczny 697c2afad6cSKamil Konieczny st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT | 698c2afad6cSKamil Konieczny SSS_FCINTSTAT_HRDMAINT); 699c2afad6cSKamil Konieczny /* clear DMA bits */ 700c2afad6cSKamil Konieczny SSS_WRITE(dev, FCINTPEND, st_bits); 701c2afad6cSKamil Konieczny 702c2afad6cSKamil Konieczny /* clear HASH irq bits */ 703c2afad6cSKamil Konieczny if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) { 704c2afad6cSKamil Konieczny /* cannot have both HPART and HDONE */ 705c2afad6cSKamil Konieczny if (status & SSS_FCINTSTAT_HPARTINT) 706c2afad6cSKamil Konieczny st_bits = SSS_HASH_STATUS_PARTIAL_DONE; 707c2afad6cSKamil Konieczny 708c2afad6cSKamil Konieczny if (status & SSS_FCINTSTAT_HDONEINT) 709c2afad6cSKamil Konieczny st_bits = SSS_HASH_STATUS_MSG_DONE; 710c2afad6cSKamil Konieczny 711c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags); 712c2afad6cSKamil Konieczny s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits); 713c2afad6cSKamil Konieczny hx_end = true; 714c2afad6cSKamil Konieczny /* when DONE or PART, do not handle HASH DMA */ 715c2afad6cSKamil Konieczny err_dma_hx = 0; 716c2afad6cSKamil Konieczny } 717a49e490cSVladimir Zapolskiy 71828b62b14SKrzysztof Kozlowski if (err_dma_rx < 0) { 71928b62b14SKrzysztof Kozlowski err = err_dma_rx; 72028b62b14SKrzysztof Kozlowski goto error; 72128b62b14SKrzysztof Kozlowski } 72228b62b14SKrzysztof Kozlowski if (err_dma_tx < 0) { 72328b62b14SKrzysztof Kozlowski err = err_dma_tx; 72428b62b14SKrzysztof Kozlowski goto error; 72528b62b14SKrzysztof Kozlowski } 72628b62b14SKrzysztof Kozlowski 72728b62b14SKrzysztof Kozlowski if (tx_end) { 72828b62b14SKrzysztof Kozlowski s5p_sg_done(dev); 729c2afad6cSKamil Konieczny if (err_dma_hx == 1) 730c2afad6cSKamil Konieczny s5p_set_dma_hashdata(dev, dev->hash_sg_iter); 73128b62b14SKrzysztof Kozlowski 73228b62b14SKrzysztof Kozlowski spin_unlock_irqrestore(&dev->lock, flags); 73328b62b14SKrzysztof Kozlowski 7345842cd44SChristoph Manszewski s5p_aes_complete(dev->req, 0); 73542d5c176SKrzysztof Kozlowski /* Device is still busy */ 73628b62b14SKrzysztof Kozlowski tasklet_schedule(&dev->tasklet); 73728b62b14SKrzysztof Kozlowski } else { 73879152e8dSKrzysztof Kozlowski /* 73928b62b14SKrzysztof Kozlowski * Writing length of DMA block (either receiving or 74028b62b14SKrzysztof Kozlowski * transmitting) will start the operation immediately, so this 74128b62b14SKrzysztof Kozlowski * should be done at the end (even after clearing pending 74228b62b14SKrzysztof Kozlowski * interrupts to not miss the interrupt). 74379152e8dSKrzysztof Kozlowski */ 74428b62b14SKrzysztof Kozlowski if (err_dma_tx == 1) 74579152e8dSKrzysztof Kozlowski s5p_set_dma_outdata(dev, dev->sg_dst); 74628b62b14SKrzysztof Kozlowski if (err_dma_rx == 1) 74779152e8dSKrzysztof Kozlowski s5p_set_dma_indata(dev, dev->sg_src); 748c2afad6cSKamil Konieczny if (err_dma_hx == 1) 749c2afad6cSKamil Konieczny s5p_set_dma_hashdata(dev, dev->hash_sg_iter); 75079152e8dSKrzysztof Kozlowski 751a49e490cSVladimir Zapolskiy spin_unlock_irqrestore(&dev->lock, flags); 75228b62b14SKrzysztof Kozlowski } 75328b62b14SKrzysztof Kozlowski 754c2afad6cSKamil Konieczny goto hash_irq_end; 75528b62b14SKrzysztof Kozlowski 75628b62b14SKrzysztof Kozlowski error: 75728b62b14SKrzysztof Kozlowski s5p_sg_done(dev); 75842d5c176SKrzysztof Kozlowski dev->busy = false; 7595842cd44SChristoph Manszewski req = dev->req; 760c2afad6cSKamil Konieczny if (err_dma_hx == 1) 761c2afad6cSKamil Konieczny s5p_set_dma_hashdata(dev, dev->hash_sg_iter); 762c2afad6cSKamil Konieczny 76328b62b14SKrzysztof Kozlowski spin_unlock_irqrestore(&dev->lock, flags); 7645842cd44SChristoph Manszewski s5p_aes_complete(req, err); 765a49e490cSVladimir Zapolskiy 766c2afad6cSKamil Konieczny hash_irq_end: 767c2afad6cSKamil Konieczny /* 768c2afad6cSKamil Konieczny * Note about else if: 769c2afad6cSKamil Konieczny * when hash_sg_iter reaches end and its UPDATE op, 770c2afad6cSKamil Konieczny * issue SSS_HASH_PAUSE and wait for HPART irq 771c2afad6cSKamil Konieczny */ 772c2afad6cSKamil Konieczny if (hx_end) 773c2afad6cSKamil Konieczny tasklet_schedule(&dev->hash_tasklet); 774c2afad6cSKamil Konieczny else if (err_dma_hx == 2) 775c2afad6cSKamil Konieczny s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE, 776c2afad6cSKamil Konieczny SSS_HASH_PAUSE); 777c2afad6cSKamil Konieczny 778a49e490cSVladimir Zapolskiy return IRQ_HANDLED; 779a49e490cSVladimir Zapolskiy } 780a49e490cSVladimir Zapolskiy 781c2afad6cSKamil Konieczny /** 782c2afad6cSKamil Konieczny * s5p_hash_read_msg() - read message or IV from HW 783c2afad6cSKamil Konieczny * @req: AHASH request 784c2afad6cSKamil Konieczny */ 785c2afad6cSKamil Konieczny static void s5p_hash_read_msg(struct ahash_request *req) 786c2afad6cSKamil Konieczny { 787c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 788c2afad6cSKamil Konieczny struct s5p_aes_dev *dd = ctx->dd; 789c2afad6cSKamil Konieczny u32 *hash = (u32 *)ctx->digest; 790c2afad6cSKamil Konieczny unsigned int i; 791c2afad6cSKamil Konieczny 792c2afad6cSKamil Konieczny for (i = 0; i < ctx->nregs; i++) 793c2afad6cSKamil Konieczny hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i)); 794c2afad6cSKamil Konieczny } 795c2afad6cSKamil Konieczny 796c2afad6cSKamil Konieczny /** 797c2afad6cSKamil Konieczny * s5p_hash_write_ctx_iv() - write IV for next partial/finup op. 798c2afad6cSKamil Konieczny * @dd: device 799c2afad6cSKamil Konieczny * @ctx: request context 800c2afad6cSKamil Konieczny */ 801c2afad6cSKamil Konieczny static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd, 8026584eacbSKrzysztof Kozlowski const struct s5p_hash_reqctx *ctx) 803c2afad6cSKamil Konieczny { 8046584eacbSKrzysztof Kozlowski const u32 *hash = (const u32 *)ctx->digest; 805c2afad6cSKamil Konieczny unsigned int i; 806c2afad6cSKamil Konieczny 807c2afad6cSKamil Konieczny for (i = 0; i < ctx->nregs; i++) 808c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]); 809c2afad6cSKamil Konieczny } 810c2afad6cSKamil Konieczny 811c2afad6cSKamil Konieczny /** 812c2afad6cSKamil Konieczny * s5p_hash_write_iv() - write IV for next partial/finup op. 813c2afad6cSKamil Konieczny * @req: AHASH request 814c2afad6cSKamil Konieczny */ 815c2afad6cSKamil Konieczny static void s5p_hash_write_iv(struct ahash_request *req) 816c2afad6cSKamil Konieczny { 817c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 818c2afad6cSKamil Konieczny 819c2afad6cSKamil Konieczny s5p_hash_write_ctx_iv(ctx->dd, ctx); 820c2afad6cSKamil Konieczny } 821c2afad6cSKamil Konieczny 822c2afad6cSKamil Konieczny /** 823c2afad6cSKamil Konieczny * s5p_hash_copy_result() - copy digest into req->result 824c2afad6cSKamil Konieczny * @req: AHASH request 825c2afad6cSKamil Konieczny */ 826c2afad6cSKamil Konieczny static void s5p_hash_copy_result(struct ahash_request *req) 827c2afad6cSKamil Konieczny { 8286584eacbSKrzysztof Kozlowski const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 829c2afad6cSKamil Konieczny 830c2afad6cSKamil Konieczny if (!req->result) 831c2afad6cSKamil Konieczny return; 832c2afad6cSKamil Konieczny 833c2afad6cSKamil Konieczny memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF); 834c2afad6cSKamil Konieczny } 835c2afad6cSKamil Konieczny 836c2afad6cSKamil Konieczny /** 837c2afad6cSKamil Konieczny * s5p_hash_dma_flush() - flush HASH DMA 838c2afad6cSKamil Konieczny * @dev: secss device 839c2afad6cSKamil Konieczny */ 840c2afad6cSKamil Konieczny static void s5p_hash_dma_flush(struct s5p_aes_dev *dev) 841c2afad6cSKamil Konieczny { 842c2afad6cSKamil Konieczny SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH); 843c2afad6cSKamil Konieczny } 844c2afad6cSKamil Konieczny 845c2afad6cSKamil Konieczny /** 846c2afad6cSKamil Konieczny * s5p_hash_dma_enable() - enable DMA mode for HASH 847c2afad6cSKamil Konieczny * @dev: secss device 848c2afad6cSKamil Konieczny * 849c2afad6cSKamil Konieczny * enable DMA mode for HASH 850c2afad6cSKamil Konieczny */ 851c2afad6cSKamil Konieczny static void s5p_hash_dma_enable(struct s5p_aes_dev *dev) 852c2afad6cSKamil Konieczny { 853c2afad6cSKamil Konieczny s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA); 854c2afad6cSKamil Konieczny } 855c2afad6cSKamil Konieczny 856c2afad6cSKamil Konieczny /** 857c2afad6cSKamil Konieczny * s5p_hash_irq_disable() - disable irq HASH signals 858c2afad6cSKamil Konieczny * @dev: secss device 859c2afad6cSKamil Konieczny * @flags: bitfield with irq's to be disabled 860c2afad6cSKamil Konieczny */ 861c2afad6cSKamil Konieczny static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags) 862c2afad6cSKamil Konieczny { 863c2afad6cSKamil Konieczny SSS_WRITE(dev, FCINTENCLR, flags); 864c2afad6cSKamil Konieczny } 865c2afad6cSKamil Konieczny 866c2afad6cSKamil Konieczny /** 867c2afad6cSKamil Konieczny * s5p_hash_irq_enable() - enable irq signals 868c2afad6cSKamil Konieczny * @dev: secss device 869c2afad6cSKamil Konieczny * @flags: bitfield with irq's to be enabled 870c2afad6cSKamil Konieczny */ 871c2afad6cSKamil Konieczny static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags) 872c2afad6cSKamil Konieczny { 873c2afad6cSKamil Konieczny SSS_WRITE(dev, FCINTENSET, flags); 874c2afad6cSKamil Konieczny } 875c2afad6cSKamil Konieczny 876c2afad6cSKamil Konieczny /** 877c2afad6cSKamil Konieczny * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH 878c2afad6cSKamil Konieczny * @dev: secss device 879c2afad6cSKamil Konieczny * @hashflow: HASH stream flow with/without crypto AES/DES 880c2afad6cSKamil Konieczny */ 881c2afad6cSKamil Konieczny static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow) 882c2afad6cSKamil Konieczny { 883c2afad6cSKamil Konieczny unsigned long flags; 884c2afad6cSKamil Konieczny u32 flow; 885c2afad6cSKamil Konieczny 886c2afad6cSKamil Konieczny spin_lock_irqsave(&dev->lock, flags); 887c2afad6cSKamil Konieczny 888c2afad6cSKamil Konieczny flow = SSS_READ(dev, FCFIFOCTRL); 889c2afad6cSKamil Konieczny flow &= ~SSS_HASHIN_MASK; 890c2afad6cSKamil Konieczny flow |= hashflow; 891c2afad6cSKamil Konieczny SSS_WRITE(dev, FCFIFOCTRL, flow); 892c2afad6cSKamil Konieczny 893c2afad6cSKamil Konieczny spin_unlock_irqrestore(&dev->lock, flags); 894c2afad6cSKamil Konieczny } 895c2afad6cSKamil Konieczny 896c2afad6cSKamil Konieczny /** 897c2afad6cSKamil Konieczny * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS 898c2afad6cSKamil Konieczny * @dev: secss device 899c2afad6cSKamil Konieczny * @hashflow: HASH stream flow with/without AES/DES 900c2afad6cSKamil Konieczny * 901c2afad6cSKamil Konieczny * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW, 902c2afad6cSKamil Konieczny * enable HASH irq's HRDMA, HDONE, HPART 903c2afad6cSKamil Konieczny */ 904c2afad6cSKamil Konieczny static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow) 905c2afad6cSKamil Konieczny { 906c2afad6cSKamil Konieczny s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR | 907c2afad6cSKamil Konieczny SSS_FCINTENCLR_HDONEINTENCLR | 908c2afad6cSKamil Konieczny SSS_FCINTENCLR_HPARTINTENCLR); 909c2afad6cSKamil Konieczny s5p_hash_dma_flush(dev); 910c2afad6cSKamil Konieczny 911c2afad6cSKamil Konieczny s5p_hash_dma_enable(dev); 912c2afad6cSKamil Konieczny s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK); 913c2afad6cSKamil Konieczny s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET | 914c2afad6cSKamil Konieczny SSS_FCINTENSET_HDONEINTENSET | 915c2afad6cSKamil Konieczny SSS_FCINTENSET_HPARTINTENSET); 916c2afad6cSKamil Konieczny } 917c2afad6cSKamil Konieczny 918c2afad6cSKamil Konieczny /** 919c2afad6cSKamil Konieczny * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing 920c2afad6cSKamil Konieczny * @dd: secss device 921c2afad6cSKamil Konieczny * @length: length for request 922c2afad6cSKamil Konieczny * @final: true if final op 923c2afad6cSKamil Konieczny * 924c2afad6cSKamil Konieczny * Prepare SSS HASH block for processing bytes in DMA mode. If it is called 925c2afad6cSKamil Konieczny * after previous updates, fill up IV words. For final, calculate and set 926c2afad6cSKamil Konieczny * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH 927c2afad6cSKamil Konieczny * length as 2^63 so it will be never reached and set to zero prelow and 928c2afad6cSKamil Konieczny * prehigh. 929c2afad6cSKamil Konieczny * 930c2afad6cSKamil Konieczny * This function does not start DMA transfer. 931c2afad6cSKamil Konieczny */ 932c2afad6cSKamil Konieczny static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length, 933c2afad6cSKamil Konieczny bool final) 934c2afad6cSKamil Konieczny { 935c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); 936c2afad6cSKamil Konieczny u32 prelow, prehigh, low, high; 937c2afad6cSKamil Konieczny u32 configflags, swapflags; 938c2afad6cSKamil Konieczny u64 tmplen; 939c2afad6cSKamil Konieczny 940c2afad6cSKamil Konieczny configflags = ctx->engine | SSS_HASH_INIT_BIT; 941c2afad6cSKamil Konieczny 942c2afad6cSKamil Konieczny if (likely(ctx->digcnt)) { 943c2afad6cSKamil Konieczny s5p_hash_write_ctx_iv(dd, ctx); 944c2afad6cSKamil Konieczny configflags |= SSS_HASH_USER_IV_EN; 945c2afad6cSKamil Konieczny } 946c2afad6cSKamil Konieczny 947c2afad6cSKamil Konieczny if (final) { 948c2afad6cSKamil Konieczny /* number of bytes for last part */ 949c2afad6cSKamil Konieczny low = length; 950c2afad6cSKamil Konieczny high = 0; 951c2afad6cSKamil Konieczny /* total number of bits prev hashed */ 952c2afad6cSKamil Konieczny tmplen = ctx->digcnt * 8; 953c2afad6cSKamil Konieczny prelow = (u32)tmplen; 954c2afad6cSKamil Konieczny prehigh = (u32)(tmplen >> 32); 955c2afad6cSKamil Konieczny } else { 956c2afad6cSKamil Konieczny prelow = 0; 957c2afad6cSKamil Konieczny prehigh = 0; 958c2afad6cSKamil Konieczny low = 0; 959c2afad6cSKamil Konieczny high = BIT(31); 960c2afad6cSKamil Konieczny } 961c2afad6cSKamil Konieczny 962c2afad6cSKamil Konieczny swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO | 963c2afad6cSKamil Konieczny SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY; 964c2afad6cSKamil Konieczny 965c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low); 966c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high); 967c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow); 968c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh); 969c2afad6cSKamil Konieczny 970c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags); 971c2afad6cSKamil Konieczny s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags); 972c2afad6cSKamil Konieczny } 973c2afad6cSKamil Konieczny 974c2afad6cSKamil Konieczny /** 975c2afad6cSKamil Konieczny * s5p_hash_xmit_dma() - start DMA hash processing 976c2afad6cSKamil Konieczny * @dd: secss device 977c2afad6cSKamil Konieczny * @length: length for request 978c2afad6cSKamil Konieczny * @final: true if final op 979c2afad6cSKamil Konieczny * 980c2afad6cSKamil Konieczny * Update digcnt here, as it is needed for finup/final op. 981c2afad6cSKamil Konieczny */ 982c2afad6cSKamil Konieczny static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length, 983c2afad6cSKamil Konieczny bool final) 984c2afad6cSKamil Konieczny { 985c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); 986c2afad6cSKamil Konieczny unsigned int cnt; 987c2afad6cSKamil Konieczny 988c2afad6cSKamil Konieczny cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); 989c2afad6cSKamil Konieczny if (!cnt) { 990c2afad6cSKamil Konieczny dev_err(dd->dev, "dma_map_sg error\n"); 991c2afad6cSKamil Konieczny ctx->error = true; 992c2afad6cSKamil Konieczny return -EINVAL; 993c2afad6cSKamil Konieczny } 994c2afad6cSKamil Konieczny 995c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); 996c2afad6cSKamil Konieczny dd->hash_sg_iter = ctx->sg; 997c2afad6cSKamil Konieczny dd->hash_sg_cnt = cnt; 998c2afad6cSKamil Konieczny s5p_hash_write_ctrl(dd, length, final); 999c2afad6cSKamil Konieczny ctx->digcnt += length; 1000c2afad6cSKamil Konieczny ctx->total -= length; 1001c2afad6cSKamil Konieczny 1002c2afad6cSKamil Konieczny /* catch last interrupt */ 1003c2afad6cSKamil Konieczny if (final) 1004c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_FINAL, &dd->hash_flags); 1005c2afad6cSKamil Konieczny 1006c2afad6cSKamil Konieczny s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */ 1007c2afad6cSKamil Konieczny 1008c2afad6cSKamil Konieczny return -EINPROGRESS; 1009c2afad6cSKamil Konieczny } 1010c2afad6cSKamil Konieczny 1011c2afad6cSKamil Konieczny /** 1012c2afad6cSKamil Konieczny * s5p_hash_copy_sgs() - copy request's bytes into new buffer 1013c2afad6cSKamil Konieczny * @ctx: request context 1014c2afad6cSKamil Konieczny * @sg: source scatterlist request 1015c2afad6cSKamil Konieczny * @new_len: number of bytes to process from sg 1016c2afad6cSKamil Konieczny * 1017c2afad6cSKamil Konieczny * Allocate new buffer, copy data for HASH into it. If there was xmit_buf 1018c2afad6cSKamil Konieczny * filled, copy it first, then copy data from sg into it. Prepare one sgl[0] 1019c2afad6cSKamil Konieczny * with allocated buffer. 1020c2afad6cSKamil Konieczny * 1021c2afad6cSKamil Konieczny * Set bit in dd->hash_flag so we can free it after irq ends processing. 1022c2afad6cSKamil Konieczny */ 1023c2afad6cSKamil Konieczny static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx, 1024c2afad6cSKamil Konieczny struct scatterlist *sg, unsigned int new_len) 1025c2afad6cSKamil Konieczny { 1026c2afad6cSKamil Konieczny unsigned int pages, len; 1027c2afad6cSKamil Konieczny void *buf; 1028c2afad6cSKamil Konieczny 1029c2afad6cSKamil Konieczny len = new_len + ctx->bufcnt; 1030c2afad6cSKamil Konieczny pages = get_order(len); 1031c2afad6cSKamil Konieczny 1032c2afad6cSKamil Konieczny buf = (void *)__get_free_pages(GFP_ATOMIC, pages); 1033c2afad6cSKamil Konieczny if (!buf) { 1034c2afad6cSKamil Konieczny dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n"); 1035c2afad6cSKamil Konieczny ctx->error = true; 1036c2afad6cSKamil Konieczny return -ENOMEM; 1037c2afad6cSKamil Konieczny } 1038c2afad6cSKamil Konieczny 1039c2afad6cSKamil Konieczny if (ctx->bufcnt) 1040c2afad6cSKamil Konieczny memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt); 1041c2afad6cSKamil Konieczny 1042c2afad6cSKamil Konieczny scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip, 1043c2afad6cSKamil Konieczny new_len, 0); 1044c2afad6cSKamil Konieczny sg_init_table(ctx->sgl, 1); 1045c2afad6cSKamil Konieczny sg_set_buf(ctx->sgl, buf, len); 1046c2afad6cSKamil Konieczny ctx->sg = ctx->sgl; 1047c2afad6cSKamil Konieczny ctx->sg_len = 1; 1048c2afad6cSKamil Konieczny ctx->bufcnt = 0; 1049c2afad6cSKamil Konieczny ctx->skip = 0; 1050c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags); 1051c2afad6cSKamil Konieczny 1052c2afad6cSKamil Konieczny return 0; 1053c2afad6cSKamil Konieczny } 1054c2afad6cSKamil Konieczny 1055c2afad6cSKamil Konieczny /** 1056c2afad6cSKamil Konieczny * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy 1057c2afad6cSKamil Konieczny * @ctx: request context 1058c2afad6cSKamil Konieczny * @sg: source scatterlist request 1059c2afad6cSKamil Konieczny * @new_len: number of bytes to process from sg 1060c2afad6cSKamil Konieczny * 1061c2afad6cSKamil Konieczny * Allocate new scatterlist table, copy data for HASH into it. If there was 1062c2afad6cSKamil Konieczny * xmit_buf filled, prepare it first, then copy page, length and offset from 1063c2afad6cSKamil Konieczny * source sg into it, adjusting begin and/or end for skip offset and 1064c2afad6cSKamil Konieczny * hash_later value. 1065c2afad6cSKamil Konieczny * 1066c2afad6cSKamil Konieczny * Resulting sg table will be assigned to ctx->sg. Set flag so we can free 1067c2afad6cSKamil Konieczny * it after irq ends processing. 1068c2afad6cSKamil Konieczny */ 1069c2afad6cSKamil Konieczny static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx, 1070c2afad6cSKamil Konieczny struct scatterlist *sg, unsigned int new_len) 1071c2afad6cSKamil Konieczny { 1072c2afad6cSKamil Konieczny unsigned int skip = ctx->skip, n = sg_nents(sg); 1073c2afad6cSKamil Konieczny struct scatterlist *tmp; 1074c2afad6cSKamil Konieczny unsigned int len; 1075c2afad6cSKamil Konieczny 1076c2afad6cSKamil Konieczny if (ctx->bufcnt) 1077c2afad6cSKamil Konieczny n++; 1078c2afad6cSKamil Konieczny 1079c2afad6cSKamil Konieczny ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); 1080c2afad6cSKamil Konieczny if (!ctx->sg) { 1081c2afad6cSKamil Konieczny ctx->error = true; 1082c2afad6cSKamil Konieczny return -ENOMEM; 1083c2afad6cSKamil Konieczny } 1084c2afad6cSKamil Konieczny 1085c2afad6cSKamil Konieczny sg_init_table(ctx->sg, n); 1086c2afad6cSKamil Konieczny 1087c2afad6cSKamil Konieczny tmp = ctx->sg; 1088c2afad6cSKamil Konieczny 1089c2afad6cSKamil Konieczny ctx->sg_len = 0; 1090c2afad6cSKamil Konieczny 1091c2afad6cSKamil Konieczny if (ctx->bufcnt) { 1092c2afad6cSKamil Konieczny sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt); 1093c2afad6cSKamil Konieczny tmp = sg_next(tmp); 1094c2afad6cSKamil Konieczny ctx->sg_len++; 1095c2afad6cSKamil Konieczny } 1096c2afad6cSKamil Konieczny 1097c2afad6cSKamil Konieczny while (sg && skip >= sg->length) { 1098c2afad6cSKamil Konieczny skip -= sg->length; 1099c2afad6cSKamil Konieczny sg = sg_next(sg); 1100c2afad6cSKamil Konieczny } 1101c2afad6cSKamil Konieczny 1102c2afad6cSKamil Konieczny while (sg && new_len) { 1103c2afad6cSKamil Konieczny len = sg->length - skip; 1104c2afad6cSKamil Konieczny if (new_len < len) 1105c2afad6cSKamil Konieczny len = new_len; 1106c2afad6cSKamil Konieczny 1107c2afad6cSKamil Konieczny new_len -= len; 1108c2afad6cSKamil Konieczny sg_set_page(tmp, sg_page(sg), len, sg->offset + skip); 1109c2afad6cSKamil Konieczny skip = 0; 1110c2afad6cSKamil Konieczny if (new_len <= 0) 1111c2afad6cSKamil Konieczny sg_mark_end(tmp); 1112c2afad6cSKamil Konieczny 1113c2afad6cSKamil Konieczny tmp = sg_next(tmp); 1114c2afad6cSKamil Konieczny ctx->sg_len++; 1115c2afad6cSKamil Konieczny sg = sg_next(sg); 1116c2afad6cSKamil Konieczny } 1117c2afad6cSKamil Konieczny 1118c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags); 1119c2afad6cSKamil Konieczny 1120c2afad6cSKamil Konieczny return 0; 1121c2afad6cSKamil Konieczny } 1122c2afad6cSKamil Konieczny 1123c2afad6cSKamil Konieczny /** 1124c2afad6cSKamil Konieczny * s5p_hash_prepare_sgs() - prepare sg for processing 1125c2afad6cSKamil Konieczny * @ctx: request context 1126c2afad6cSKamil Konieczny * @sg: source scatterlist request 112798a6bc10SKrzysztof Kozlowski * @new_len: number of bytes to process from sg 1128c2afad6cSKamil Konieczny * @final: final flag 1129c2afad6cSKamil Konieczny * 1130c2afad6cSKamil Konieczny * Check two conditions: (1) if buffers in sg have len aligned data, and (2) 1131c2afad6cSKamil Konieczny * sg table have good aligned elements (list_ok). If one of this checks fails, 1132c2afad6cSKamil Konieczny * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy 1133c2afad6cSKamil Konieczny * data into this buffer and prepare request in sgl, or (2) allocates new sg 1134c2afad6cSKamil Konieczny * table and prepare sg elements. 1135c2afad6cSKamil Konieczny * 1136c2afad6cSKamil Konieczny * For digest or finup all conditions can be good, and we may not need any 1137c2afad6cSKamil Konieczny * fixes. 1138c2afad6cSKamil Konieczny */ 1139c2afad6cSKamil Konieczny static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx, 1140c2afad6cSKamil Konieczny struct scatterlist *sg, 1141c2afad6cSKamil Konieczny unsigned int new_len, bool final) 1142c2afad6cSKamil Konieczny { 1143c2afad6cSKamil Konieczny unsigned int skip = ctx->skip, nbytes = new_len, n = 0; 1144c2afad6cSKamil Konieczny bool aligned = true, list_ok = true; 1145c2afad6cSKamil Konieczny struct scatterlist *sg_tmp = sg; 1146c2afad6cSKamil Konieczny 1147c2afad6cSKamil Konieczny if (!sg || !sg->length || !new_len) 1148c2afad6cSKamil Konieczny return 0; 1149c2afad6cSKamil Konieczny 1150c2afad6cSKamil Konieczny if (skip || !final) 1151c2afad6cSKamil Konieczny list_ok = false; 1152c2afad6cSKamil Konieczny 1153c2afad6cSKamil Konieczny while (nbytes > 0 && sg_tmp) { 1154c2afad6cSKamil Konieczny n++; 1155c2afad6cSKamil Konieczny if (skip >= sg_tmp->length) { 1156c2afad6cSKamil Konieczny skip -= sg_tmp->length; 1157c2afad6cSKamil Konieczny if (!sg_tmp->length) { 1158c2afad6cSKamil Konieczny aligned = false; 1159c2afad6cSKamil Konieczny break; 1160c2afad6cSKamil Konieczny } 1161c2afad6cSKamil Konieczny } else { 1162c2afad6cSKamil Konieczny if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) { 1163c2afad6cSKamil Konieczny aligned = false; 1164c2afad6cSKamil Konieczny break; 1165c2afad6cSKamil Konieczny } 1166c2afad6cSKamil Konieczny 1167c2afad6cSKamil Konieczny if (nbytes < sg_tmp->length - skip) { 1168c2afad6cSKamil Konieczny list_ok = false; 1169c2afad6cSKamil Konieczny break; 1170c2afad6cSKamil Konieczny } 1171c2afad6cSKamil Konieczny 1172c2afad6cSKamil Konieczny nbytes -= sg_tmp->length - skip; 1173c2afad6cSKamil Konieczny skip = 0; 1174c2afad6cSKamil Konieczny } 1175c2afad6cSKamil Konieczny 1176c2afad6cSKamil Konieczny sg_tmp = sg_next(sg_tmp); 1177c2afad6cSKamil Konieczny } 1178c2afad6cSKamil Konieczny 1179c2afad6cSKamil Konieczny if (!aligned) 1180c2afad6cSKamil Konieczny return s5p_hash_copy_sgs(ctx, sg, new_len); 1181c2afad6cSKamil Konieczny else if (!list_ok) 1182c2afad6cSKamil Konieczny return s5p_hash_copy_sg_lists(ctx, sg, new_len); 1183c2afad6cSKamil Konieczny 1184c2afad6cSKamil Konieczny /* 1185c2afad6cSKamil Konieczny * Have aligned data from previous operation and/or current 1186c2afad6cSKamil Konieczny * Note: will enter here only if (digest or finup) and aligned 1187c2afad6cSKamil Konieczny */ 1188c2afad6cSKamil Konieczny if (ctx->bufcnt) { 1189c2afad6cSKamil Konieczny ctx->sg_len = n; 1190c2afad6cSKamil Konieczny sg_init_table(ctx->sgl, 2); 1191c2afad6cSKamil Konieczny sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt); 1192c2afad6cSKamil Konieczny sg_chain(ctx->sgl, 2, sg); 1193c2afad6cSKamil Konieczny ctx->sg = ctx->sgl; 1194c2afad6cSKamil Konieczny ctx->sg_len++; 1195c2afad6cSKamil Konieczny } else { 1196c2afad6cSKamil Konieczny ctx->sg = sg; 1197c2afad6cSKamil Konieczny ctx->sg_len = n; 1198c2afad6cSKamil Konieczny } 1199c2afad6cSKamil Konieczny 1200c2afad6cSKamil Konieczny return 0; 1201c2afad6cSKamil Konieczny } 1202c2afad6cSKamil Konieczny 1203c2afad6cSKamil Konieczny /** 1204c2afad6cSKamil Konieczny * s5p_hash_prepare_request() - prepare request for processing 1205c2afad6cSKamil Konieczny * @req: AHASH request 1206c2afad6cSKamil Konieczny * @update: true if UPDATE op 1207c2afad6cSKamil Konieczny * 1208c2afad6cSKamil Konieczny * Note 1: we can have update flag _and_ final flag at the same time. 1209c2afad6cSKamil Konieczny * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or 1210c2afad6cSKamil Konieczny * either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or 1211c2afad6cSKamil Konieczny * we have final op 1212c2afad6cSKamil Konieczny */ 1213c2afad6cSKamil Konieczny static int s5p_hash_prepare_request(struct ahash_request *req, bool update) 1214c2afad6cSKamil Konieczny { 1215c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1216c2afad6cSKamil Konieczny bool final = ctx->finup; 1217c2afad6cSKamil Konieczny int xmit_len, hash_later, nbytes; 1218c2afad6cSKamil Konieczny int ret; 1219c2afad6cSKamil Konieczny 1220c2afad6cSKamil Konieczny if (update) 1221c2afad6cSKamil Konieczny nbytes = req->nbytes; 1222c2afad6cSKamil Konieczny else 1223c2afad6cSKamil Konieczny nbytes = 0; 1224c2afad6cSKamil Konieczny 1225c2afad6cSKamil Konieczny ctx->total = nbytes + ctx->bufcnt; 1226c2afad6cSKamil Konieczny if (!ctx->total) 1227c2afad6cSKamil Konieczny return 0; 1228c2afad6cSKamil Konieczny 1229c2afad6cSKamil Konieczny if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) { 1230c2afad6cSKamil Konieczny /* bytes left from previous request, so fill up to BUFLEN */ 1231c2afad6cSKamil Konieczny int len = BUFLEN - ctx->bufcnt % BUFLEN; 1232c2afad6cSKamil Konieczny 1233c2afad6cSKamil Konieczny if (len > nbytes) 1234c2afad6cSKamil Konieczny len = nbytes; 1235c2afad6cSKamil Konieczny 1236c2afad6cSKamil Konieczny scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, 1237c2afad6cSKamil Konieczny 0, len, 0); 1238c2afad6cSKamil Konieczny ctx->bufcnt += len; 1239c2afad6cSKamil Konieczny nbytes -= len; 1240c2afad6cSKamil Konieczny ctx->skip = len; 1241c2afad6cSKamil Konieczny } else { 1242c2afad6cSKamil Konieczny ctx->skip = 0; 1243c2afad6cSKamil Konieczny } 1244c2afad6cSKamil Konieczny 1245c2afad6cSKamil Konieczny if (ctx->bufcnt) 1246c2afad6cSKamil Konieczny memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt); 1247c2afad6cSKamil Konieczny 1248c2afad6cSKamil Konieczny xmit_len = ctx->total; 1249c2afad6cSKamil Konieczny if (final) { 1250c2afad6cSKamil Konieczny hash_later = 0; 1251c2afad6cSKamil Konieczny } else { 1252c2afad6cSKamil Konieczny if (IS_ALIGNED(xmit_len, BUFLEN)) 1253c2afad6cSKamil Konieczny xmit_len -= BUFLEN; 1254c2afad6cSKamil Konieczny else 1255c2afad6cSKamil Konieczny xmit_len -= xmit_len & (BUFLEN - 1); 1256c2afad6cSKamil Konieczny 1257c2afad6cSKamil Konieczny hash_later = ctx->total - xmit_len; 1258c2afad6cSKamil Konieczny /* copy hash_later bytes from end of req->src */ 1259c2afad6cSKamil Konieczny /* previous bytes are in xmit_buf, so no overwrite */ 1260c2afad6cSKamil Konieczny scatterwalk_map_and_copy(ctx->buffer, req->src, 1261c2afad6cSKamil Konieczny req->nbytes - hash_later, 1262c2afad6cSKamil Konieczny hash_later, 0); 1263c2afad6cSKamil Konieczny } 1264c2afad6cSKamil Konieczny 1265c2afad6cSKamil Konieczny if (xmit_len > BUFLEN) { 1266c2afad6cSKamil Konieczny ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later, 1267c2afad6cSKamil Konieczny final); 1268c2afad6cSKamil Konieczny if (ret) 1269c2afad6cSKamil Konieczny return ret; 1270c2afad6cSKamil Konieczny } else { 1271c2afad6cSKamil Konieczny /* have buffered data only */ 1272c2afad6cSKamil Konieczny if (unlikely(!ctx->bufcnt)) { 1273c2afad6cSKamil Konieczny /* first update didn't fill up buffer */ 1274c2afad6cSKamil Konieczny scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src, 1275c2afad6cSKamil Konieczny 0, xmit_len, 0); 1276c2afad6cSKamil Konieczny } 1277c2afad6cSKamil Konieczny 1278c2afad6cSKamil Konieczny sg_init_table(ctx->sgl, 1); 1279c2afad6cSKamil Konieczny sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len); 1280c2afad6cSKamil Konieczny 1281c2afad6cSKamil Konieczny ctx->sg = ctx->sgl; 1282c2afad6cSKamil Konieczny ctx->sg_len = 1; 1283c2afad6cSKamil Konieczny } 1284c2afad6cSKamil Konieczny 1285c2afad6cSKamil Konieczny ctx->bufcnt = hash_later; 1286c2afad6cSKamil Konieczny if (!final) 1287c2afad6cSKamil Konieczny ctx->total = xmit_len; 1288c2afad6cSKamil Konieczny 1289c2afad6cSKamil Konieczny return 0; 1290c2afad6cSKamil Konieczny } 1291c2afad6cSKamil Konieczny 1292c2afad6cSKamil Konieczny /** 1293c2afad6cSKamil Konieczny * s5p_hash_update_dma_stop() - unmap DMA 1294c2afad6cSKamil Konieczny * @dd: secss device 1295c2afad6cSKamil Konieczny * 1296c2afad6cSKamil Konieczny * Unmap scatterlist ctx->sg. 1297c2afad6cSKamil Konieczny */ 1298c2afad6cSKamil Konieczny static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd) 1299c2afad6cSKamil Konieczny { 13006584eacbSKrzysztof Kozlowski const struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); 1301c2afad6cSKamil Konieczny 1302c2afad6cSKamil Konieczny dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); 1303c2afad6cSKamil Konieczny clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); 1304c2afad6cSKamil Konieczny } 1305c2afad6cSKamil Konieczny 1306c2afad6cSKamil Konieczny /** 1307c2afad6cSKamil Konieczny * s5p_hash_finish() - copy calculated digest to crypto layer 1308c2afad6cSKamil Konieczny * @req: AHASH request 1309c2afad6cSKamil Konieczny */ 1310c2afad6cSKamil Konieczny static void s5p_hash_finish(struct ahash_request *req) 1311c2afad6cSKamil Konieczny { 1312c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1313c2afad6cSKamil Konieczny struct s5p_aes_dev *dd = ctx->dd; 1314c2afad6cSKamil Konieczny 1315c2afad6cSKamil Konieczny if (ctx->digcnt) 1316c2afad6cSKamil Konieczny s5p_hash_copy_result(req); 1317c2afad6cSKamil Konieczny 1318c2afad6cSKamil Konieczny dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt); 1319c2afad6cSKamil Konieczny } 1320c2afad6cSKamil Konieczny 1321c2afad6cSKamil Konieczny /** 1322c2afad6cSKamil Konieczny * s5p_hash_finish_req() - finish request 1323c2afad6cSKamil Konieczny * @req: AHASH request 1324c2afad6cSKamil Konieczny * @err: error 1325c2afad6cSKamil Konieczny */ 1326c2afad6cSKamil Konieczny static void s5p_hash_finish_req(struct ahash_request *req, int err) 1327c2afad6cSKamil Konieczny { 1328c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1329c2afad6cSKamil Konieczny struct s5p_aes_dev *dd = ctx->dd; 1330c2afad6cSKamil Konieczny unsigned long flags; 1331c2afad6cSKamil Konieczny 1332c2afad6cSKamil Konieczny if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags)) 1333c2afad6cSKamil Konieczny free_pages((unsigned long)sg_virt(ctx->sg), 1334c2afad6cSKamil Konieczny get_order(ctx->sg->length)); 1335c2afad6cSKamil Konieczny 1336c2afad6cSKamil Konieczny if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags)) 1337c2afad6cSKamil Konieczny kfree(ctx->sg); 1338c2afad6cSKamil Konieczny 1339c2afad6cSKamil Konieczny ctx->sg = NULL; 1340c2afad6cSKamil Konieczny dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) | 1341c2afad6cSKamil Konieczny BIT(HASH_FLAGS_SGS_COPIED)); 1342c2afad6cSKamil Konieczny 1343c2afad6cSKamil Konieczny if (!err && !ctx->error) { 1344c2afad6cSKamil Konieczny s5p_hash_read_msg(req); 1345c2afad6cSKamil Konieczny if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags)) 1346c2afad6cSKamil Konieczny s5p_hash_finish(req); 1347c2afad6cSKamil Konieczny } else { 1348c2afad6cSKamil Konieczny ctx->error = true; 1349c2afad6cSKamil Konieczny } 1350c2afad6cSKamil Konieczny 1351c2afad6cSKamil Konieczny spin_lock_irqsave(&dd->hash_lock, flags); 1352c2afad6cSKamil Konieczny dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) | 1353c2afad6cSKamil Konieczny BIT(HASH_FLAGS_DMA_READY) | 1354c2afad6cSKamil Konieczny BIT(HASH_FLAGS_OUTPUT_READY)); 1355c2afad6cSKamil Konieczny spin_unlock_irqrestore(&dd->hash_lock, flags); 1356c2afad6cSKamil Konieczny 1357c2afad6cSKamil Konieczny if (req->base.complete) 1358c2afad6cSKamil Konieczny req->base.complete(&req->base, err); 1359c2afad6cSKamil Konieczny } 1360c2afad6cSKamil Konieczny 1361c2afad6cSKamil Konieczny /** 1362c2afad6cSKamil Konieczny * s5p_hash_handle_queue() - handle hash queue 1363c2afad6cSKamil Konieczny * @dd: device s5p_aes_dev 1364c2afad6cSKamil Konieczny * @req: AHASH request 1365c2afad6cSKamil Konieczny * 1366c2afad6cSKamil Konieczny * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the 1367c2afad6cSKamil Konieczny * device then processes the first request from the dd->queue 1368c2afad6cSKamil Konieczny * 1369c2afad6cSKamil Konieczny * Returns: see s5p_hash_final below. 1370c2afad6cSKamil Konieczny */ 1371c2afad6cSKamil Konieczny static int s5p_hash_handle_queue(struct s5p_aes_dev *dd, 1372c2afad6cSKamil Konieczny struct ahash_request *req) 1373c2afad6cSKamil Konieczny { 1374c2afad6cSKamil Konieczny struct crypto_async_request *async_req, *backlog; 1375c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx; 1376c2afad6cSKamil Konieczny unsigned long flags; 1377c2afad6cSKamil Konieczny int err = 0, ret = 0; 1378c2afad6cSKamil Konieczny 1379c2afad6cSKamil Konieczny retry: 1380c2afad6cSKamil Konieczny spin_lock_irqsave(&dd->hash_lock, flags); 1381c2afad6cSKamil Konieczny if (req) 1382c2afad6cSKamil Konieczny ret = ahash_enqueue_request(&dd->hash_queue, req); 1383c2afad6cSKamil Konieczny 1384c2afad6cSKamil Konieczny if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { 1385c2afad6cSKamil Konieczny spin_unlock_irqrestore(&dd->hash_lock, flags); 1386c2afad6cSKamil Konieczny return ret; 1387c2afad6cSKamil Konieczny } 1388c2afad6cSKamil Konieczny 1389c2afad6cSKamil Konieczny backlog = crypto_get_backlog(&dd->hash_queue); 1390c2afad6cSKamil Konieczny async_req = crypto_dequeue_request(&dd->hash_queue); 1391c2afad6cSKamil Konieczny if (async_req) 1392c2afad6cSKamil Konieczny set_bit(HASH_FLAGS_BUSY, &dd->hash_flags); 1393c2afad6cSKamil Konieczny 1394c2afad6cSKamil Konieczny spin_unlock_irqrestore(&dd->hash_lock, flags); 1395c2afad6cSKamil Konieczny 1396c2afad6cSKamil Konieczny if (!async_req) 1397c2afad6cSKamil Konieczny return ret; 1398c2afad6cSKamil Konieczny 1399c2afad6cSKamil Konieczny if (backlog) 1400c2afad6cSKamil Konieczny backlog->complete(backlog, -EINPROGRESS); 1401c2afad6cSKamil Konieczny 1402c2afad6cSKamil Konieczny req = ahash_request_cast(async_req); 1403c2afad6cSKamil Konieczny dd->hash_req = req; 1404c2afad6cSKamil Konieczny ctx = ahash_request_ctx(req); 1405c2afad6cSKamil Konieczny 1406c2afad6cSKamil Konieczny err = s5p_hash_prepare_request(req, ctx->op_update); 1407c2afad6cSKamil Konieczny if (err || !ctx->total) 1408c2afad6cSKamil Konieczny goto out; 1409c2afad6cSKamil Konieczny 1410c2afad6cSKamil Konieczny dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n", 1411c2afad6cSKamil Konieczny ctx->op_update, req->nbytes); 1412c2afad6cSKamil Konieczny 1413c2afad6cSKamil Konieczny s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT); 1414c2afad6cSKamil Konieczny if (ctx->digcnt) 1415c2afad6cSKamil Konieczny s5p_hash_write_iv(req); /* restore hash IV */ 1416c2afad6cSKamil Konieczny 1417c2afad6cSKamil Konieczny if (ctx->op_update) { /* HASH_OP_UPDATE */ 1418c2afad6cSKamil Konieczny err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup); 1419c2afad6cSKamil Konieczny if (err != -EINPROGRESS && ctx->finup && !ctx->error) 1420c2afad6cSKamil Konieczny /* no final() after finup() */ 1421c2afad6cSKamil Konieczny err = s5p_hash_xmit_dma(dd, ctx->total, true); 1422c2afad6cSKamil Konieczny } else { /* HASH_OP_FINAL */ 1423c2afad6cSKamil Konieczny err = s5p_hash_xmit_dma(dd, ctx->total, true); 1424c2afad6cSKamil Konieczny } 1425c2afad6cSKamil Konieczny out: 1426c2afad6cSKamil Konieczny if (err != -EINPROGRESS) { 1427c2afad6cSKamil Konieczny /* hash_tasklet_cb will not finish it, so do it here */ 1428c2afad6cSKamil Konieczny s5p_hash_finish_req(req, err); 1429c2afad6cSKamil Konieczny req = NULL; 1430c2afad6cSKamil Konieczny 1431c2afad6cSKamil Konieczny /* 1432c2afad6cSKamil Konieczny * Execute next request immediately if there is anything 1433c2afad6cSKamil Konieczny * in queue. 1434c2afad6cSKamil Konieczny */ 1435c2afad6cSKamil Konieczny goto retry; 1436c2afad6cSKamil Konieczny } 1437c2afad6cSKamil Konieczny 1438c2afad6cSKamil Konieczny return ret; 1439c2afad6cSKamil Konieczny } 1440c2afad6cSKamil Konieczny 1441c2afad6cSKamil Konieczny /** 1442c2afad6cSKamil Konieczny * s5p_hash_tasklet_cb() - hash tasklet 1443c2afad6cSKamil Konieczny * @data: ptr to s5p_aes_dev 1444c2afad6cSKamil Konieczny */ 1445c2afad6cSKamil Konieczny static void s5p_hash_tasklet_cb(unsigned long data) 1446c2afad6cSKamil Konieczny { 1447c2afad6cSKamil Konieczny struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data; 1448c2afad6cSKamil Konieczny 1449c2afad6cSKamil Konieczny if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { 1450c2afad6cSKamil Konieczny s5p_hash_handle_queue(dd, NULL); 1451c2afad6cSKamil Konieczny return; 1452c2afad6cSKamil Konieczny } 1453c2afad6cSKamil Konieczny 1454c2afad6cSKamil Konieczny if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) { 1455c2afad6cSKamil Konieczny if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE, 1456c2afad6cSKamil Konieczny &dd->hash_flags)) { 1457c2afad6cSKamil Konieczny s5p_hash_update_dma_stop(dd); 1458c2afad6cSKamil Konieczny } 1459c2afad6cSKamil Konieczny 1460c2afad6cSKamil Konieczny if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY, 1461c2afad6cSKamil Konieczny &dd->hash_flags)) { 1462c2afad6cSKamil Konieczny /* hash or semi-hash ready */ 1463c2afad6cSKamil Konieczny clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags); 1464c2afad6cSKamil Konieczny goto finish; 1465c2afad6cSKamil Konieczny } 1466c2afad6cSKamil Konieczny } 1467c2afad6cSKamil Konieczny 1468c2afad6cSKamil Konieczny return; 1469c2afad6cSKamil Konieczny 1470c2afad6cSKamil Konieczny finish: 1471c2afad6cSKamil Konieczny /* finish curent request */ 1472c2afad6cSKamil Konieczny s5p_hash_finish_req(dd->hash_req, 0); 1473c2afad6cSKamil Konieczny 1474c2afad6cSKamil Konieczny /* If we are not busy, process next req */ 1475c2afad6cSKamil Konieczny if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) 1476c2afad6cSKamil Konieczny s5p_hash_handle_queue(dd, NULL); 1477c2afad6cSKamil Konieczny } 1478c2afad6cSKamil Konieczny 1479c2afad6cSKamil Konieczny /** 1480c2afad6cSKamil Konieczny * s5p_hash_enqueue() - enqueue request 1481c2afad6cSKamil Konieczny * @req: AHASH request 1482c2afad6cSKamil Konieczny * @op: operation UPDATE (true) or FINAL (false) 1483c2afad6cSKamil Konieczny * 1484c2afad6cSKamil Konieczny * Returns: see s5p_hash_final below. 1485c2afad6cSKamil Konieczny */ 1486c2afad6cSKamil Konieczny static int s5p_hash_enqueue(struct ahash_request *req, bool op) 1487c2afad6cSKamil Konieczny { 1488c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1489c2afad6cSKamil Konieczny struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 1490c2afad6cSKamil Konieczny 1491c2afad6cSKamil Konieczny ctx->op_update = op; 1492c2afad6cSKamil Konieczny 1493c2afad6cSKamil Konieczny return s5p_hash_handle_queue(tctx->dd, req); 1494c2afad6cSKamil Konieczny } 1495c2afad6cSKamil Konieczny 1496c2afad6cSKamil Konieczny /** 1497c2afad6cSKamil Konieczny * s5p_hash_update() - process the hash input data 1498c2afad6cSKamil Konieczny * @req: AHASH request 1499c2afad6cSKamil Konieczny * 1500c2afad6cSKamil Konieczny * If request will fit in buffer, copy it and return immediately 1501c2afad6cSKamil Konieczny * else enqueue it with OP_UPDATE. 1502c2afad6cSKamil Konieczny * 1503c2afad6cSKamil Konieczny * Returns: see s5p_hash_final below. 1504c2afad6cSKamil Konieczny */ 1505c2afad6cSKamil Konieczny static int s5p_hash_update(struct ahash_request *req) 1506c2afad6cSKamil Konieczny { 1507c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1508c2afad6cSKamil Konieczny 1509c2afad6cSKamil Konieczny if (!req->nbytes) 1510c2afad6cSKamil Konieczny return 0; 1511c2afad6cSKamil Konieczny 1512c2afad6cSKamil Konieczny if (ctx->bufcnt + req->nbytes <= BUFLEN) { 1513c2afad6cSKamil Konieczny scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, 1514c2afad6cSKamil Konieczny 0, req->nbytes, 0); 1515c2afad6cSKamil Konieczny ctx->bufcnt += req->nbytes; 1516c2afad6cSKamil Konieczny return 0; 1517c2afad6cSKamil Konieczny } 1518c2afad6cSKamil Konieczny 1519c2afad6cSKamil Konieczny return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */ 1520c2afad6cSKamil Konieczny } 1521c2afad6cSKamil Konieczny 1522c2afad6cSKamil Konieczny /** 1523c2afad6cSKamil Konieczny * s5p_hash_final() - close up hash and calculate digest 1524c2afad6cSKamil Konieczny * @req: AHASH request 1525c2afad6cSKamil Konieczny * 1526c2afad6cSKamil Konieczny * Note: in final req->src do not have any data, and req->nbytes can be 1527c2afad6cSKamil Konieczny * non-zero. 1528c2afad6cSKamil Konieczny * 1529c2afad6cSKamil Konieczny * If there were no input data processed yet and the buffered hash data is 1530c2afad6cSKamil Konieczny * less than BUFLEN (64) then calculate the final hash immediately by using 1531c2afad6cSKamil Konieczny * SW algorithm fallback. 1532c2afad6cSKamil Konieczny * 1533c2afad6cSKamil Konieczny * Otherwise enqueues the current AHASH request with OP_FINAL operation op 1534c2afad6cSKamil Konieczny * and finalize hash message in HW. Note that if digcnt!=0 then there were 1535c2afad6cSKamil Konieczny * previous update op, so there are always some buffered bytes in ctx->buffer, 1536c2afad6cSKamil Konieczny * which means that ctx->bufcnt!=0 1537c2afad6cSKamil Konieczny * 1538c2afad6cSKamil Konieczny * Returns: 1539c2afad6cSKamil Konieczny * 0 if the request has been processed immediately, 1540c2afad6cSKamil Konieczny * -EINPROGRESS if the operation has been queued for later execution or is set 1541c2afad6cSKamil Konieczny * to processing by HW, 1542c2afad6cSKamil Konieczny * -EBUSY if queue is full and request should be resubmitted later, 1543c2afad6cSKamil Konieczny * other negative values denotes an error. 1544c2afad6cSKamil Konieczny */ 1545c2afad6cSKamil Konieczny static int s5p_hash_final(struct ahash_request *req) 1546c2afad6cSKamil Konieczny { 1547c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1548c2afad6cSKamil Konieczny 1549c2afad6cSKamil Konieczny ctx->finup = true; 1550c2afad6cSKamil Konieczny if (ctx->error) 1551c2afad6cSKamil Konieczny return -EINVAL; /* uncompleted hash is not needed */ 1552c2afad6cSKamil Konieczny 1553ecca1ad6SEric Biggers if (!ctx->digcnt && ctx->bufcnt < BUFLEN) { 1554ecca1ad6SEric Biggers struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 1555ecca1ad6SEric Biggers 1556ecca1ad6SEric Biggers return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer, 1557ecca1ad6SEric Biggers ctx->bufcnt, req->result); 1558ecca1ad6SEric Biggers } 1559c2afad6cSKamil Konieczny 1560c2afad6cSKamil Konieczny return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */ 1561c2afad6cSKamil Konieczny } 1562c2afad6cSKamil Konieczny 1563c2afad6cSKamil Konieczny /** 1564c2afad6cSKamil Konieczny * s5p_hash_finup() - process last req->src and calculate digest 1565c2afad6cSKamil Konieczny * @req: AHASH request containing the last update data 1566c2afad6cSKamil Konieczny * 1567c2afad6cSKamil Konieczny * Return values: see s5p_hash_final above. 1568c2afad6cSKamil Konieczny */ 1569c2afad6cSKamil Konieczny static int s5p_hash_finup(struct ahash_request *req) 1570c2afad6cSKamil Konieczny { 1571c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1572c2afad6cSKamil Konieczny int err1, err2; 1573c2afad6cSKamil Konieczny 1574c2afad6cSKamil Konieczny ctx->finup = true; 1575c2afad6cSKamil Konieczny 1576c2afad6cSKamil Konieczny err1 = s5p_hash_update(req); 1577c2afad6cSKamil Konieczny if (err1 == -EINPROGRESS || err1 == -EBUSY) 1578c2afad6cSKamil Konieczny return err1; 1579c2afad6cSKamil Konieczny 1580c2afad6cSKamil Konieczny /* 1581c2afad6cSKamil Konieczny * final() has to be always called to cleanup resources even if 1582c2afad6cSKamil Konieczny * update() failed, except EINPROGRESS or calculate digest for small 1583c2afad6cSKamil Konieczny * size 1584c2afad6cSKamil Konieczny */ 1585c2afad6cSKamil Konieczny err2 = s5p_hash_final(req); 1586c2afad6cSKamil Konieczny 1587c2afad6cSKamil Konieczny return err1 ?: err2; 1588c2afad6cSKamil Konieczny } 1589c2afad6cSKamil Konieczny 1590c2afad6cSKamil Konieczny /** 1591c2afad6cSKamil Konieczny * s5p_hash_init() - initialize AHASH request contex 1592c2afad6cSKamil Konieczny * @req: AHASH request 1593c2afad6cSKamil Konieczny * 1594c2afad6cSKamil Konieczny * Init async hash request context. 1595c2afad6cSKamil Konieczny */ 1596c2afad6cSKamil Konieczny static int s5p_hash_init(struct ahash_request *req) 1597c2afad6cSKamil Konieczny { 1598c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1599c2afad6cSKamil Konieczny struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1600c2afad6cSKamil Konieczny struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); 1601c2afad6cSKamil Konieczny 1602c2afad6cSKamil Konieczny ctx->dd = tctx->dd; 1603c2afad6cSKamil Konieczny ctx->error = false; 1604c2afad6cSKamil Konieczny ctx->finup = false; 1605c2afad6cSKamil Konieczny ctx->bufcnt = 0; 1606c2afad6cSKamil Konieczny ctx->digcnt = 0; 1607c2afad6cSKamil Konieczny ctx->total = 0; 1608c2afad6cSKamil Konieczny ctx->skip = 0; 1609c2afad6cSKamil Konieczny 1610c2afad6cSKamil Konieczny dev_dbg(tctx->dd->dev, "init: digest size: %d\n", 1611c2afad6cSKamil Konieczny crypto_ahash_digestsize(tfm)); 1612c2afad6cSKamil Konieczny 1613c2afad6cSKamil Konieczny switch (crypto_ahash_digestsize(tfm)) { 1614c2afad6cSKamil Konieczny case MD5_DIGEST_SIZE: 1615c2afad6cSKamil Konieczny ctx->engine = SSS_HASH_ENGINE_MD5; 1616c2afad6cSKamil Konieczny ctx->nregs = HASH_MD5_MAX_REG; 1617c2afad6cSKamil Konieczny break; 1618c2afad6cSKamil Konieczny case SHA1_DIGEST_SIZE: 1619c2afad6cSKamil Konieczny ctx->engine = SSS_HASH_ENGINE_SHA1; 1620c2afad6cSKamil Konieczny ctx->nregs = HASH_SHA1_MAX_REG; 1621c2afad6cSKamil Konieczny break; 1622c2afad6cSKamil Konieczny case SHA256_DIGEST_SIZE: 1623c2afad6cSKamil Konieczny ctx->engine = SSS_HASH_ENGINE_SHA256; 1624c2afad6cSKamil Konieczny ctx->nregs = HASH_SHA256_MAX_REG; 1625c2afad6cSKamil Konieczny break; 1626c2afad6cSKamil Konieczny default: 1627c2afad6cSKamil Konieczny ctx->error = true; 1628c2afad6cSKamil Konieczny return -EINVAL; 1629c2afad6cSKamil Konieczny } 1630c2afad6cSKamil Konieczny 1631c2afad6cSKamil Konieczny return 0; 1632c2afad6cSKamil Konieczny } 1633c2afad6cSKamil Konieczny 1634c2afad6cSKamil Konieczny /** 1635c2afad6cSKamil Konieczny * s5p_hash_digest - calculate digest from req->src 1636c2afad6cSKamil Konieczny * @req: AHASH request 1637c2afad6cSKamil Konieczny * 1638c2afad6cSKamil Konieczny * Return values: see s5p_hash_final above. 1639c2afad6cSKamil Konieczny */ 1640c2afad6cSKamil Konieczny static int s5p_hash_digest(struct ahash_request *req) 1641c2afad6cSKamil Konieczny { 1642c2afad6cSKamil Konieczny return s5p_hash_init(req) ?: s5p_hash_finup(req); 1643c2afad6cSKamil Konieczny } 1644c2afad6cSKamil Konieczny 1645c2afad6cSKamil Konieczny /** 1646c2afad6cSKamil Konieczny * s5p_hash_cra_init_alg - init crypto alg transformation 1647c2afad6cSKamil Konieczny * @tfm: crypto transformation 1648c2afad6cSKamil Konieczny */ 1649c2afad6cSKamil Konieczny static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm) 1650c2afad6cSKamil Konieczny { 1651c2afad6cSKamil Konieczny struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); 1652c2afad6cSKamil Konieczny const char *alg_name = crypto_tfm_alg_name(tfm); 1653c2afad6cSKamil Konieczny 1654c2afad6cSKamil Konieczny tctx->dd = s5p_dev; 1655c2afad6cSKamil Konieczny /* Allocate a fallback and abort if it failed. */ 1656c2afad6cSKamil Konieczny tctx->fallback = crypto_alloc_shash(alg_name, 0, 1657c2afad6cSKamil Konieczny CRYPTO_ALG_NEED_FALLBACK); 1658c2afad6cSKamil Konieczny if (IS_ERR(tctx->fallback)) { 1659c2afad6cSKamil Konieczny pr_err("fallback alloc fails for '%s'\n", alg_name); 1660c2afad6cSKamil Konieczny return PTR_ERR(tctx->fallback); 1661c2afad6cSKamil Konieczny } 1662c2afad6cSKamil Konieczny 1663c2afad6cSKamil Konieczny crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1664c2afad6cSKamil Konieczny sizeof(struct s5p_hash_reqctx) + BUFLEN); 1665c2afad6cSKamil Konieczny 1666c2afad6cSKamil Konieczny return 0; 1667c2afad6cSKamil Konieczny } 1668c2afad6cSKamil Konieczny 1669c2afad6cSKamil Konieczny /** 1670c2afad6cSKamil Konieczny * s5p_hash_cra_init - init crypto tfm 1671c2afad6cSKamil Konieczny * @tfm: crypto transformation 1672c2afad6cSKamil Konieczny */ 1673c2afad6cSKamil Konieczny static int s5p_hash_cra_init(struct crypto_tfm *tfm) 1674c2afad6cSKamil Konieczny { 1675c2afad6cSKamil Konieczny return s5p_hash_cra_init_alg(tfm); 1676c2afad6cSKamil Konieczny } 1677c2afad6cSKamil Konieczny 1678c2afad6cSKamil Konieczny /** 1679c2afad6cSKamil Konieczny * s5p_hash_cra_exit - exit crypto tfm 1680c2afad6cSKamil Konieczny * @tfm: crypto transformation 1681c2afad6cSKamil Konieczny * 1682c2afad6cSKamil Konieczny * free allocated fallback 1683c2afad6cSKamil Konieczny */ 1684c2afad6cSKamil Konieczny static void s5p_hash_cra_exit(struct crypto_tfm *tfm) 1685c2afad6cSKamil Konieczny { 1686c2afad6cSKamil Konieczny struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); 1687c2afad6cSKamil Konieczny 1688c2afad6cSKamil Konieczny crypto_free_shash(tctx->fallback); 1689c2afad6cSKamil Konieczny tctx->fallback = NULL; 1690c2afad6cSKamil Konieczny } 1691c2afad6cSKamil Konieczny 1692c2afad6cSKamil Konieczny /** 1693c2afad6cSKamil Konieczny * s5p_hash_export - export hash state 1694c2afad6cSKamil Konieczny * @req: AHASH request 1695c2afad6cSKamil Konieczny * @out: buffer for exported state 1696c2afad6cSKamil Konieczny */ 1697c2afad6cSKamil Konieczny static int s5p_hash_export(struct ahash_request *req, void *out) 1698c2afad6cSKamil Konieczny { 16996584eacbSKrzysztof Kozlowski const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1700c2afad6cSKamil Konieczny 1701c2afad6cSKamil Konieczny memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt); 1702c2afad6cSKamil Konieczny 1703c2afad6cSKamil Konieczny return 0; 1704c2afad6cSKamil Konieczny } 1705c2afad6cSKamil Konieczny 1706c2afad6cSKamil Konieczny /** 1707c2afad6cSKamil Konieczny * s5p_hash_import - import hash state 1708c2afad6cSKamil Konieczny * @req: AHASH request 1709c2afad6cSKamil Konieczny * @in: buffer with state to be imported from 1710c2afad6cSKamil Konieczny */ 1711c2afad6cSKamil Konieczny static int s5p_hash_import(struct ahash_request *req, const void *in) 1712c2afad6cSKamil Konieczny { 1713c2afad6cSKamil Konieczny struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); 1714c2afad6cSKamil Konieczny struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1715c2afad6cSKamil Konieczny struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); 1716c2afad6cSKamil Konieczny const struct s5p_hash_reqctx *ctx_in = in; 1717c2afad6cSKamil Konieczny 1718c2afad6cSKamil Konieczny memcpy(ctx, in, sizeof(*ctx) + BUFLEN); 1719c2afad6cSKamil Konieczny if (ctx_in->bufcnt > BUFLEN) { 1720c2afad6cSKamil Konieczny ctx->error = true; 1721c2afad6cSKamil Konieczny return -EINVAL; 1722c2afad6cSKamil Konieczny } 1723c2afad6cSKamil Konieczny 1724c2afad6cSKamil Konieczny ctx->dd = tctx->dd; 1725c2afad6cSKamil Konieczny ctx->error = false; 1726c2afad6cSKamil Konieczny 1727c2afad6cSKamil Konieczny return 0; 1728c2afad6cSKamil Konieczny } 1729c2afad6cSKamil Konieczny 1730c2afad6cSKamil Konieczny static struct ahash_alg algs_sha1_md5_sha256[] = { 1731c2afad6cSKamil Konieczny { 1732c2afad6cSKamil Konieczny .init = s5p_hash_init, 1733c2afad6cSKamil Konieczny .update = s5p_hash_update, 1734c2afad6cSKamil Konieczny .final = s5p_hash_final, 1735c2afad6cSKamil Konieczny .finup = s5p_hash_finup, 1736c2afad6cSKamil Konieczny .digest = s5p_hash_digest, 1737c2afad6cSKamil Konieczny .export = s5p_hash_export, 1738c2afad6cSKamil Konieczny .import = s5p_hash_import, 1739c2afad6cSKamil Konieczny .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, 1740c2afad6cSKamil Konieczny .halg.digestsize = SHA1_DIGEST_SIZE, 1741c2afad6cSKamil Konieczny .halg.base = { 1742c2afad6cSKamil Konieczny .cra_name = "sha1", 1743c2afad6cSKamil Konieczny .cra_driver_name = "exynos-sha1", 1744c2afad6cSKamil Konieczny .cra_priority = 100, 17456a38f622SEric Biggers .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | 1746c2afad6cSKamil Konieczny CRYPTO_ALG_ASYNC | 1747c2afad6cSKamil Konieczny CRYPTO_ALG_NEED_FALLBACK, 1748c2afad6cSKamil Konieczny .cra_blocksize = HASH_BLOCK_SIZE, 1749c2afad6cSKamil Konieczny .cra_ctxsize = sizeof(struct s5p_hash_ctx), 1750c2afad6cSKamil Konieczny .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, 1751c2afad6cSKamil Konieczny .cra_module = THIS_MODULE, 1752c2afad6cSKamil Konieczny .cra_init = s5p_hash_cra_init, 1753c2afad6cSKamil Konieczny .cra_exit = s5p_hash_cra_exit, 1754c2afad6cSKamil Konieczny } 1755c2afad6cSKamil Konieczny }, 1756c2afad6cSKamil Konieczny { 1757c2afad6cSKamil Konieczny .init = s5p_hash_init, 1758c2afad6cSKamil Konieczny .update = s5p_hash_update, 1759c2afad6cSKamil Konieczny .final = s5p_hash_final, 1760c2afad6cSKamil Konieczny .finup = s5p_hash_finup, 1761c2afad6cSKamil Konieczny .digest = s5p_hash_digest, 1762c2afad6cSKamil Konieczny .export = s5p_hash_export, 1763c2afad6cSKamil Konieczny .import = s5p_hash_import, 1764c2afad6cSKamil Konieczny .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, 1765c2afad6cSKamil Konieczny .halg.digestsize = MD5_DIGEST_SIZE, 1766c2afad6cSKamil Konieczny .halg.base = { 1767c2afad6cSKamil Konieczny .cra_name = "md5", 1768c2afad6cSKamil Konieczny .cra_driver_name = "exynos-md5", 1769c2afad6cSKamil Konieczny .cra_priority = 100, 17706a38f622SEric Biggers .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | 1771c2afad6cSKamil Konieczny CRYPTO_ALG_ASYNC | 1772c2afad6cSKamil Konieczny CRYPTO_ALG_NEED_FALLBACK, 1773c2afad6cSKamil Konieczny .cra_blocksize = HASH_BLOCK_SIZE, 1774c2afad6cSKamil Konieczny .cra_ctxsize = sizeof(struct s5p_hash_ctx), 1775c2afad6cSKamil Konieczny .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, 1776c2afad6cSKamil Konieczny .cra_module = THIS_MODULE, 1777c2afad6cSKamil Konieczny .cra_init = s5p_hash_cra_init, 1778c2afad6cSKamil Konieczny .cra_exit = s5p_hash_cra_exit, 1779c2afad6cSKamil Konieczny } 1780c2afad6cSKamil Konieczny }, 1781c2afad6cSKamil Konieczny { 1782c2afad6cSKamil Konieczny .init = s5p_hash_init, 1783c2afad6cSKamil Konieczny .update = s5p_hash_update, 1784c2afad6cSKamil Konieczny .final = s5p_hash_final, 1785c2afad6cSKamil Konieczny .finup = s5p_hash_finup, 1786c2afad6cSKamil Konieczny .digest = s5p_hash_digest, 1787c2afad6cSKamil Konieczny .export = s5p_hash_export, 1788c2afad6cSKamil Konieczny .import = s5p_hash_import, 1789c2afad6cSKamil Konieczny .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, 1790c2afad6cSKamil Konieczny .halg.digestsize = SHA256_DIGEST_SIZE, 1791c2afad6cSKamil Konieczny .halg.base = { 1792c2afad6cSKamil Konieczny .cra_name = "sha256", 1793c2afad6cSKamil Konieczny .cra_driver_name = "exynos-sha256", 1794c2afad6cSKamil Konieczny .cra_priority = 100, 17956a38f622SEric Biggers .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | 1796c2afad6cSKamil Konieczny CRYPTO_ALG_ASYNC | 1797c2afad6cSKamil Konieczny CRYPTO_ALG_NEED_FALLBACK, 1798c2afad6cSKamil Konieczny .cra_blocksize = HASH_BLOCK_SIZE, 1799c2afad6cSKamil Konieczny .cra_ctxsize = sizeof(struct s5p_hash_ctx), 1800c2afad6cSKamil Konieczny .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, 1801c2afad6cSKamil Konieczny .cra_module = THIS_MODULE, 1802c2afad6cSKamil Konieczny .cra_init = s5p_hash_cra_init, 1803c2afad6cSKamil Konieczny .cra_exit = s5p_hash_cra_exit, 1804c2afad6cSKamil Konieczny } 1805c2afad6cSKamil Konieczny } 1806c2afad6cSKamil Konieczny 1807c2afad6cSKamil Konieczny }; 1808c2afad6cSKamil Konieczny 1809a49e490cSVladimir Zapolskiy static void s5p_set_aes(struct s5p_aes_dev *dev, 1810cdf640a6SChristoph Manszewski const u8 *key, const u8 *iv, const u8 *ctr, 18116584eacbSKrzysztof Kozlowski unsigned int keylen) 1812a49e490cSVladimir Zapolskiy { 1813a49e490cSVladimir Zapolskiy void __iomem *keystart; 1814a49e490cSVladimir Zapolskiy 18158f9702aaSNaveen Krishna Chatradhi if (iv) 1816ef5c73b3SKrzysztof Kozlowski memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 1817ef5c73b3SKrzysztof Kozlowski AES_BLOCK_SIZE); 1818a49e490cSVladimir Zapolskiy 1819cdf640a6SChristoph Manszewski if (ctr) 1820ef5c73b3SKrzysztof Kozlowski memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), ctr, 1821ef5c73b3SKrzysztof Kozlowski AES_BLOCK_SIZE); 1822cdf640a6SChristoph Manszewski 1823a49e490cSVladimir Zapolskiy if (keylen == AES_KEYSIZE_256) 182489245107SNaveen Krishna Chatradhi keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0); 1825a49e490cSVladimir Zapolskiy else if (keylen == AES_KEYSIZE_192) 182689245107SNaveen Krishna Chatradhi keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2); 1827a49e490cSVladimir Zapolskiy else 182889245107SNaveen Krishna Chatradhi keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4); 1829a49e490cSVladimir Zapolskiy 18301e3012d0SKrzysztof Koz?owski memcpy_toio(keystart, key, keylen); 1831a49e490cSVladimir Zapolskiy } 1832a49e490cSVladimir Zapolskiy 18339e4a1100SKrzysztof Kozlowski static bool s5p_is_sg_aligned(struct scatterlist *sg) 18349e4a1100SKrzysztof Kozlowski { 18359e4a1100SKrzysztof Kozlowski while (sg) { 1836d1497977SMarek Szyprowski if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) 18379e4a1100SKrzysztof Kozlowski return false; 18389e4a1100SKrzysztof Kozlowski sg = sg_next(sg); 18399e4a1100SKrzysztof Kozlowski } 18409e4a1100SKrzysztof Kozlowski 18419e4a1100SKrzysztof Kozlowski return true; 18429e4a1100SKrzysztof Kozlowski } 18439e4a1100SKrzysztof Kozlowski 18449e4a1100SKrzysztof Kozlowski static int s5p_set_indata_start(struct s5p_aes_dev *dev, 1845e6b98ce6SArd Biesheuvel struct skcipher_request *req) 18469e4a1100SKrzysztof Kozlowski { 18479e4a1100SKrzysztof Kozlowski struct scatterlist *sg; 18489e4a1100SKrzysztof Kozlowski int err; 18499e4a1100SKrzysztof Kozlowski 18509e4a1100SKrzysztof Kozlowski dev->sg_src_cpy = NULL; 18519e4a1100SKrzysztof Kozlowski sg = req->src; 18529e4a1100SKrzysztof Kozlowski if (!s5p_is_sg_aligned(sg)) { 18539e4a1100SKrzysztof Kozlowski dev_dbg(dev->dev, 18549e4a1100SKrzysztof Kozlowski "At least one unaligned source scatter list, making a copy\n"); 18559e4a1100SKrzysztof Kozlowski err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy); 18569e4a1100SKrzysztof Kozlowski if (err) 18579e4a1100SKrzysztof Kozlowski return err; 18589e4a1100SKrzysztof Kozlowski 18599e4a1100SKrzysztof Kozlowski sg = dev->sg_src_cpy; 18609e4a1100SKrzysztof Kozlowski } 18619e4a1100SKrzysztof Kozlowski 18629e4a1100SKrzysztof Kozlowski err = s5p_set_indata(dev, sg); 18639e4a1100SKrzysztof Kozlowski if (err) { 18649e4a1100SKrzysztof Kozlowski s5p_free_sg_cpy(dev, &dev->sg_src_cpy); 18659e4a1100SKrzysztof Kozlowski return err; 18669e4a1100SKrzysztof Kozlowski } 18679e4a1100SKrzysztof Kozlowski 18689e4a1100SKrzysztof Kozlowski return 0; 18699e4a1100SKrzysztof Kozlowski } 18709e4a1100SKrzysztof Kozlowski 18719e4a1100SKrzysztof Kozlowski static int s5p_set_outdata_start(struct s5p_aes_dev *dev, 1872e6b98ce6SArd Biesheuvel struct skcipher_request *req) 18739e4a1100SKrzysztof Kozlowski { 18749e4a1100SKrzysztof Kozlowski struct scatterlist *sg; 18759e4a1100SKrzysztof Kozlowski int err; 18769e4a1100SKrzysztof Kozlowski 18779e4a1100SKrzysztof Kozlowski dev->sg_dst_cpy = NULL; 18789e4a1100SKrzysztof Kozlowski sg = req->dst; 18799e4a1100SKrzysztof Kozlowski if (!s5p_is_sg_aligned(sg)) { 18809e4a1100SKrzysztof Kozlowski dev_dbg(dev->dev, 18819e4a1100SKrzysztof Kozlowski "At least one unaligned dest scatter list, making a copy\n"); 18829e4a1100SKrzysztof Kozlowski err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy); 18839e4a1100SKrzysztof Kozlowski if (err) 18849e4a1100SKrzysztof Kozlowski return err; 18859e4a1100SKrzysztof Kozlowski 18869e4a1100SKrzysztof Kozlowski sg = dev->sg_dst_cpy; 18879e4a1100SKrzysztof Kozlowski } 18889e4a1100SKrzysztof Kozlowski 18899e4a1100SKrzysztof Kozlowski err = s5p_set_outdata(dev, sg); 18909e4a1100SKrzysztof Kozlowski if (err) { 18919e4a1100SKrzysztof Kozlowski s5p_free_sg_cpy(dev, &dev->sg_dst_cpy); 18929e4a1100SKrzysztof Kozlowski return err; 18939e4a1100SKrzysztof Kozlowski } 18949e4a1100SKrzysztof Kozlowski 18959e4a1100SKrzysztof Kozlowski return 0; 18969e4a1100SKrzysztof Kozlowski } 18979e4a1100SKrzysztof Kozlowski 1898a49e490cSVladimir Zapolskiy static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode) 1899a49e490cSVladimir Zapolskiy { 1900e6b98ce6SArd Biesheuvel struct skcipher_request *req = dev->req; 1901b1b4416fSChristoph Manszewski u32 aes_control; 1902a49e490cSVladimir Zapolskiy unsigned long flags; 19035318c53dSKrzysztof Kozlowski int err; 1904cdf640a6SChristoph Manszewski u8 *iv, *ctr; 1905a49e490cSVladimir Zapolskiy 1906cdf640a6SChristoph Manszewski /* This sets bit [13:12] to 00, which selects 128-bit counter */ 1907a49e490cSVladimir Zapolskiy aes_control = SSS_AES_KEY_CHANGE_MODE; 1908a49e490cSVladimir Zapolskiy if (mode & FLAGS_AES_DECRYPT) 1909a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_MODE_DECRYPT; 1910a49e490cSVladimir Zapolskiy 1911c927b080SKamil Konieczny if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) { 1912a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_CHAIN_MODE_CBC; 1913e6b98ce6SArd Biesheuvel iv = req->iv; 1914cdf640a6SChristoph Manszewski ctr = NULL; 1915c927b080SKamil Konieczny } else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) { 1916a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_CHAIN_MODE_CTR; 1917cdf640a6SChristoph Manszewski iv = NULL; 1918e6b98ce6SArd Biesheuvel ctr = req->iv; 1919c927b080SKamil Konieczny } else { 1920c927b080SKamil Konieczny iv = NULL; /* AES_ECB */ 1921cdf640a6SChristoph Manszewski ctr = NULL; 1922c927b080SKamil Konieczny } 1923a49e490cSVladimir Zapolskiy 1924a49e490cSVladimir Zapolskiy if (dev->ctx->keylen == AES_KEYSIZE_192) 1925a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_KEY_SIZE_192; 1926a49e490cSVladimir Zapolskiy else if (dev->ctx->keylen == AES_KEYSIZE_256) 1927a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_KEY_SIZE_256; 1928a49e490cSVladimir Zapolskiy 1929a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_FIFO_MODE; 1930a49e490cSVladimir Zapolskiy 1931a49e490cSVladimir Zapolskiy /* as a variant it is possible to use byte swapping on DMA side */ 1932a49e490cSVladimir Zapolskiy aes_control |= SSS_AES_BYTESWAP_DI 1933a49e490cSVladimir Zapolskiy | SSS_AES_BYTESWAP_DO 1934a49e490cSVladimir Zapolskiy | SSS_AES_BYTESWAP_IV 1935a49e490cSVladimir Zapolskiy | SSS_AES_BYTESWAP_KEY 1936a49e490cSVladimir Zapolskiy | SSS_AES_BYTESWAP_CNT; 1937a49e490cSVladimir Zapolskiy 1938a49e490cSVladimir Zapolskiy spin_lock_irqsave(&dev->lock, flags); 1939a49e490cSVladimir Zapolskiy 1940a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCINTENCLR, 1941a49e490cSVladimir Zapolskiy SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR); 1942a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCFIFOCTRL, 0x00); 1943a49e490cSVladimir Zapolskiy 19449e4a1100SKrzysztof Kozlowski err = s5p_set_indata_start(dev, req); 1945a49e490cSVladimir Zapolskiy if (err) 1946a49e490cSVladimir Zapolskiy goto indata_error; 1947a49e490cSVladimir Zapolskiy 19489e4a1100SKrzysztof Kozlowski err = s5p_set_outdata_start(dev, req); 1949a49e490cSVladimir Zapolskiy if (err) 1950a49e490cSVladimir Zapolskiy goto outdata_error; 1951a49e490cSVladimir Zapolskiy 195289245107SNaveen Krishna Chatradhi SSS_AES_WRITE(dev, AES_CONTROL, aes_control); 1953cdf640a6SChristoph Manszewski s5p_set_aes(dev, dev->ctx->aes_key, iv, ctr, dev->ctx->keylen); 1954a49e490cSVladimir Zapolskiy 19559e4a1100SKrzysztof Kozlowski s5p_set_dma_indata(dev, dev->sg_src); 19569e4a1100SKrzysztof Kozlowski s5p_set_dma_outdata(dev, dev->sg_dst); 1957a49e490cSVladimir Zapolskiy 1958a49e490cSVladimir Zapolskiy SSS_WRITE(dev, FCINTENSET, 1959a49e490cSVladimir Zapolskiy SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET); 1960a49e490cSVladimir Zapolskiy 1961a49e490cSVladimir Zapolskiy spin_unlock_irqrestore(&dev->lock, flags); 1962a49e490cSVladimir Zapolskiy 1963a49e490cSVladimir Zapolskiy return; 1964a49e490cSVladimir Zapolskiy 1965a49e490cSVladimir Zapolskiy outdata_error: 1966a49e490cSVladimir Zapolskiy s5p_unset_indata(dev); 1967a49e490cSVladimir Zapolskiy 1968a49e490cSVladimir Zapolskiy indata_error: 196928b62b14SKrzysztof Kozlowski s5p_sg_done(dev); 197042d5c176SKrzysztof Kozlowski dev->busy = false; 1971a49e490cSVladimir Zapolskiy spin_unlock_irqrestore(&dev->lock, flags); 19725842cd44SChristoph Manszewski s5p_aes_complete(req, err); 1973a49e490cSVladimir Zapolskiy } 1974a49e490cSVladimir Zapolskiy 1975a49e490cSVladimir Zapolskiy static void s5p_tasklet_cb(unsigned long data) 1976a49e490cSVladimir Zapolskiy { 1977a49e490cSVladimir Zapolskiy struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data; 1978a49e490cSVladimir Zapolskiy struct crypto_async_request *async_req, *backlog; 1979a49e490cSVladimir Zapolskiy struct s5p_aes_reqctx *reqctx; 1980a49e490cSVladimir Zapolskiy unsigned long flags; 1981a49e490cSVladimir Zapolskiy 1982a49e490cSVladimir Zapolskiy spin_lock_irqsave(&dev->lock, flags); 1983a49e490cSVladimir Zapolskiy backlog = crypto_get_backlog(&dev->queue); 1984a49e490cSVladimir Zapolskiy async_req = crypto_dequeue_request(&dev->queue); 1985a49e490cSVladimir Zapolskiy 1986dc5e3f19SNaveen Krishna Chatradhi if (!async_req) { 1987dc5e3f19SNaveen Krishna Chatradhi dev->busy = false; 1988dc5e3f19SNaveen Krishna Chatradhi spin_unlock_irqrestore(&dev->lock, flags); 1989a49e490cSVladimir Zapolskiy return; 1990dc5e3f19SNaveen Krishna Chatradhi } 1991dc5e3f19SNaveen Krishna Chatradhi spin_unlock_irqrestore(&dev->lock, flags); 1992a49e490cSVladimir Zapolskiy 1993a49e490cSVladimir Zapolskiy if (backlog) 1994a49e490cSVladimir Zapolskiy backlog->complete(backlog, -EINPROGRESS); 1995a49e490cSVladimir Zapolskiy 1996e6b98ce6SArd Biesheuvel dev->req = skcipher_request_cast(async_req); 1997a49e490cSVladimir Zapolskiy dev->ctx = crypto_tfm_ctx(dev->req->base.tfm); 1998e6b98ce6SArd Biesheuvel reqctx = skcipher_request_ctx(dev->req); 1999a49e490cSVladimir Zapolskiy 2000a49e490cSVladimir Zapolskiy s5p_aes_crypt_start(dev, reqctx->mode); 2001a49e490cSVladimir Zapolskiy } 2002a49e490cSVladimir Zapolskiy 2003a49e490cSVladimir Zapolskiy static int s5p_aes_handle_req(struct s5p_aes_dev *dev, 2004e6b98ce6SArd Biesheuvel struct skcipher_request *req) 2005a49e490cSVladimir Zapolskiy { 2006a49e490cSVladimir Zapolskiy unsigned long flags; 2007a49e490cSVladimir Zapolskiy int err; 2008a49e490cSVladimir Zapolskiy 2009a49e490cSVladimir Zapolskiy spin_lock_irqsave(&dev->lock, flags); 2010e6b98ce6SArd Biesheuvel err = crypto_enqueue_request(&dev->queue, &req->base); 2011a49e490cSVladimir Zapolskiy if (dev->busy) { 2012a49e490cSVladimir Zapolskiy spin_unlock_irqrestore(&dev->lock, flags); 2013b1b4416fSChristoph Manszewski return err; 2014a49e490cSVladimir Zapolskiy } 2015a49e490cSVladimir Zapolskiy dev->busy = true; 2016a49e490cSVladimir Zapolskiy 2017a49e490cSVladimir Zapolskiy spin_unlock_irqrestore(&dev->lock, flags); 2018a49e490cSVladimir Zapolskiy 2019a49e490cSVladimir Zapolskiy tasklet_schedule(&dev->tasklet); 2020a49e490cSVladimir Zapolskiy 2021a49e490cSVladimir Zapolskiy return err; 2022a49e490cSVladimir Zapolskiy } 2023a49e490cSVladimir Zapolskiy 2024e6b98ce6SArd Biesheuvel static int s5p_aes_crypt(struct skcipher_request *req, unsigned long mode) 2025a49e490cSVladimir Zapolskiy { 2026e6b98ce6SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 2027e6b98ce6SArd Biesheuvel struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req); 2028e6b98ce6SArd Biesheuvel struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 2029a49e490cSVladimir Zapolskiy struct s5p_aes_dev *dev = ctx->dev; 2030a49e490cSVladimir Zapolskiy 2031e6b98ce6SArd Biesheuvel if (!req->cryptlen) 203284a0b00aSArd Biesheuvel return 0; 203384a0b00aSArd Biesheuvel 2034e6b98ce6SArd Biesheuvel if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE) && 2035cdf640a6SChristoph Manszewski ((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) { 203684a0b00aSArd Biesheuvel dev_dbg(dev->dev, "request size is not exact amount of AES blocks\n"); 2037a49e490cSVladimir Zapolskiy return -EINVAL; 2038a49e490cSVladimir Zapolskiy } 2039a49e490cSVladimir Zapolskiy 2040a49e490cSVladimir Zapolskiy reqctx->mode = mode; 2041a49e490cSVladimir Zapolskiy 2042a49e490cSVladimir Zapolskiy return s5p_aes_handle_req(dev, req); 2043a49e490cSVladimir Zapolskiy } 2044a49e490cSVladimir Zapolskiy 2045e6b98ce6SArd Biesheuvel static int s5p_aes_setkey(struct crypto_skcipher *cipher, 2046b1b4416fSChristoph Manszewski const u8 *key, unsigned int keylen) 2047a49e490cSVladimir Zapolskiy { 2048e6b98ce6SArd Biesheuvel struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 2049a49e490cSVladimir Zapolskiy struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm); 2050a49e490cSVladimir Zapolskiy 2051a49e490cSVladimir Zapolskiy if (keylen != AES_KEYSIZE_128 && 2052a49e490cSVladimir Zapolskiy keylen != AES_KEYSIZE_192 && 2053a49e490cSVladimir Zapolskiy keylen != AES_KEYSIZE_256) 2054a49e490cSVladimir Zapolskiy return -EINVAL; 2055a49e490cSVladimir Zapolskiy 2056a49e490cSVladimir Zapolskiy memcpy(ctx->aes_key, key, keylen); 2057a49e490cSVladimir Zapolskiy ctx->keylen = keylen; 2058a49e490cSVladimir Zapolskiy 2059a49e490cSVladimir Zapolskiy return 0; 2060a49e490cSVladimir Zapolskiy } 2061a49e490cSVladimir Zapolskiy 2062e6b98ce6SArd Biesheuvel static int s5p_aes_ecb_encrypt(struct skcipher_request *req) 2063a49e490cSVladimir Zapolskiy { 2064a49e490cSVladimir Zapolskiy return s5p_aes_crypt(req, 0); 2065a49e490cSVladimir Zapolskiy } 2066a49e490cSVladimir Zapolskiy 2067e6b98ce6SArd Biesheuvel static int s5p_aes_ecb_decrypt(struct skcipher_request *req) 2068a49e490cSVladimir Zapolskiy { 2069a49e490cSVladimir Zapolskiy return s5p_aes_crypt(req, FLAGS_AES_DECRYPT); 2070a49e490cSVladimir Zapolskiy } 2071a49e490cSVladimir Zapolskiy 2072e6b98ce6SArd Biesheuvel static int s5p_aes_cbc_encrypt(struct skcipher_request *req) 2073a49e490cSVladimir Zapolskiy { 2074a49e490cSVladimir Zapolskiy return s5p_aes_crypt(req, FLAGS_AES_CBC); 2075a49e490cSVladimir Zapolskiy } 2076a49e490cSVladimir Zapolskiy 2077e6b98ce6SArd Biesheuvel static int s5p_aes_cbc_decrypt(struct skcipher_request *req) 2078a49e490cSVladimir Zapolskiy { 2079a49e490cSVladimir Zapolskiy return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC); 2080a49e490cSVladimir Zapolskiy } 2081a49e490cSVladimir Zapolskiy 2082e6b98ce6SArd Biesheuvel static int s5p_aes_ctr_crypt(struct skcipher_request *req) 2083cdf640a6SChristoph Manszewski { 2084cdf640a6SChristoph Manszewski return s5p_aes_crypt(req, FLAGS_AES_CTR); 2085cdf640a6SChristoph Manszewski } 2086cdf640a6SChristoph Manszewski 2087e6b98ce6SArd Biesheuvel static int s5p_aes_init_tfm(struct crypto_skcipher *tfm) 2088a49e490cSVladimir Zapolskiy { 2089e6b98ce6SArd Biesheuvel struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 2090a49e490cSVladimir Zapolskiy 2091a49e490cSVladimir Zapolskiy ctx->dev = s5p_dev; 2092e6b98ce6SArd Biesheuvel crypto_skcipher_set_reqsize(tfm, sizeof(struct s5p_aes_reqctx)); 2093a49e490cSVladimir Zapolskiy 2094a49e490cSVladimir Zapolskiy return 0; 2095a49e490cSVladimir Zapolskiy } 2096a49e490cSVladimir Zapolskiy 2097e6b98ce6SArd Biesheuvel static struct skcipher_alg algs[] = { 2098a49e490cSVladimir Zapolskiy { 2099e6b98ce6SArd Biesheuvel .base.cra_name = "ecb(aes)", 2100e6b98ce6SArd Biesheuvel .base.cra_driver_name = "ecb-aes-s5p", 2101e6b98ce6SArd Biesheuvel .base.cra_priority = 100, 2102e6b98ce6SArd Biesheuvel .base.cra_flags = CRYPTO_ALG_ASYNC | 2103d912bb76SNikos Mavrogiannopoulos CRYPTO_ALG_KERN_DRIVER_ONLY, 2104e6b98ce6SArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 2105e6b98ce6SArd Biesheuvel .base.cra_ctxsize = sizeof(struct s5p_aes_ctx), 2106e6b98ce6SArd Biesheuvel .base.cra_alignmask = 0x0f, 2107e6b98ce6SArd Biesheuvel .base.cra_module = THIS_MODULE, 2108e6b98ce6SArd Biesheuvel 2109a49e490cSVladimir Zapolskiy .min_keysize = AES_MIN_KEY_SIZE, 2110a49e490cSVladimir Zapolskiy .max_keysize = AES_MAX_KEY_SIZE, 2111a49e490cSVladimir Zapolskiy .setkey = s5p_aes_setkey, 2112a49e490cSVladimir Zapolskiy .encrypt = s5p_aes_ecb_encrypt, 2113a49e490cSVladimir Zapolskiy .decrypt = s5p_aes_ecb_decrypt, 2114e6b98ce6SArd Biesheuvel .init = s5p_aes_init_tfm, 2115a49e490cSVladimir Zapolskiy }, 2116a49e490cSVladimir Zapolskiy { 2117e6b98ce6SArd Biesheuvel .base.cra_name = "cbc(aes)", 2118e6b98ce6SArd Biesheuvel .base.cra_driver_name = "cbc-aes-s5p", 2119e6b98ce6SArd Biesheuvel .base.cra_priority = 100, 2120e6b98ce6SArd Biesheuvel .base.cra_flags = CRYPTO_ALG_ASYNC | 2121d912bb76SNikos Mavrogiannopoulos CRYPTO_ALG_KERN_DRIVER_ONLY, 2122e6b98ce6SArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 2123e6b98ce6SArd Biesheuvel .base.cra_ctxsize = sizeof(struct s5p_aes_ctx), 2124e6b98ce6SArd Biesheuvel .base.cra_alignmask = 0x0f, 2125e6b98ce6SArd Biesheuvel .base.cra_module = THIS_MODULE, 2126e6b98ce6SArd Biesheuvel 2127a49e490cSVladimir Zapolskiy .min_keysize = AES_MIN_KEY_SIZE, 2128a49e490cSVladimir Zapolskiy .max_keysize = AES_MAX_KEY_SIZE, 2129a49e490cSVladimir Zapolskiy .ivsize = AES_BLOCK_SIZE, 2130a49e490cSVladimir Zapolskiy .setkey = s5p_aes_setkey, 2131a49e490cSVladimir Zapolskiy .encrypt = s5p_aes_cbc_encrypt, 2132a49e490cSVladimir Zapolskiy .decrypt = s5p_aes_cbc_decrypt, 2133e6b98ce6SArd Biesheuvel .init = s5p_aes_init_tfm, 2134a49e490cSVladimir Zapolskiy }, 2135cdf640a6SChristoph Manszewski { 2136e6b98ce6SArd Biesheuvel .base.cra_name = "ctr(aes)", 2137e6b98ce6SArd Biesheuvel .base.cra_driver_name = "ctr-aes-s5p", 2138e6b98ce6SArd Biesheuvel .base.cra_priority = 100, 2139e6b98ce6SArd Biesheuvel .base.cra_flags = CRYPTO_ALG_ASYNC | 2140cdf640a6SChristoph Manszewski CRYPTO_ALG_KERN_DRIVER_ONLY, 2141e6b98ce6SArd Biesheuvel .base.cra_blocksize = 1, 2142e6b98ce6SArd Biesheuvel .base.cra_ctxsize = sizeof(struct s5p_aes_ctx), 2143e6b98ce6SArd Biesheuvel .base.cra_alignmask = 0x0f, 2144e6b98ce6SArd Biesheuvel .base.cra_module = THIS_MODULE, 2145e6b98ce6SArd Biesheuvel 2146cdf640a6SChristoph Manszewski .min_keysize = AES_MIN_KEY_SIZE, 2147cdf640a6SChristoph Manszewski .max_keysize = AES_MAX_KEY_SIZE, 2148cdf640a6SChristoph Manszewski .ivsize = AES_BLOCK_SIZE, 2149cdf640a6SChristoph Manszewski .setkey = s5p_aes_setkey, 2150cdf640a6SChristoph Manszewski .encrypt = s5p_aes_ctr_crypt, 2151cdf640a6SChristoph Manszewski .decrypt = s5p_aes_ctr_crypt, 2152e6b98ce6SArd Biesheuvel .init = s5p_aes_init_tfm, 2153cdf640a6SChristoph Manszewski }, 2154a49e490cSVladimir Zapolskiy }; 2155a49e490cSVladimir Zapolskiy 2156a49e490cSVladimir Zapolskiy static int s5p_aes_probe(struct platform_device *pdev) 2157a49e490cSVladimir Zapolskiy { 2158a49e490cSVladimir Zapolskiy struct device *dev = &pdev->dev; 21593d3b3a00SKrzysztof Kozlowski int i, j, err; 21606584eacbSKrzysztof Kozlowski const struct samsung_aes_variant *variant; 21615318c53dSKrzysztof Kozlowski struct s5p_aes_dev *pdata; 21625318c53dSKrzysztof Kozlowski struct resource *res; 2163c2afad6cSKamil Konieczny unsigned int hash_i; 2164a49e490cSVladimir Zapolskiy 2165a49e490cSVladimir Zapolskiy if (s5p_dev) 2166a49e490cSVladimir Zapolskiy return -EEXIST; 2167a49e490cSVladimir Zapolskiy 2168a49e490cSVladimir Zapolskiy pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); 2169a49e490cSVladimir Zapolskiy if (!pdata) 2170a49e490cSVladimir Zapolskiy return -ENOMEM; 2171a49e490cSVladimir Zapolskiy 2172c2afad6cSKamil Konieczny variant = find_s5p_sss_version(pdev); 21730fdefe2cSJingoo Han res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2174c2afad6cSKamil Konieczny 2175c2afad6cSKamil Konieczny /* 2176c2afad6cSKamil Konieczny * Note: HASH and PRNG uses the same registers in secss, avoid 2177c2afad6cSKamil Konieczny * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG 2178c2afad6cSKamil Konieczny * is enabled in config. We need larger size for HASH registers in 2179c2afad6cSKamil Konieczny * secss, current describe only AES/DES 2180c2afad6cSKamil Konieczny */ 2181c2afad6cSKamil Konieczny if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) { 2182c2afad6cSKamil Konieczny if (variant == &exynos_aes_data) { 2183c2afad6cSKamil Konieczny res->end += 0x300; 2184c2afad6cSKamil Konieczny pdata->use_hash = true; 2185c2afad6cSKamil Konieczny } 2186c2afad6cSKamil Konieczny } 2187c2afad6cSKamil Konieczny 2188c2afad6cSKamil Konieczny pdata->res = res; 2189*87bff3d8SKrzysztof Kozlowski pdata->ioaddr = devm_ioremap_resource(dev, res); 2190c2afad6cSKamil Konieczny if (IS_ERR(pdata->ioaddr)) { 2191c2afad6cSKamil Konieczny if (!pdata->use_hash) 2192c2afad6cSKamil Konieczny return PTR_ERR(pdata->ioaddr); 2193c2afad6cSKamil Konieczny /* try AES without HASH */ 2194c2afad6cSKamil Konieczny res->end -= 0x300; 2195c2afad6cSKamil Konieczny pdata->use_hash = false; 2196*87bff3d8SKrzysztof Kozlowski pdata->ioaddr = devm_ioremap_resource(dev, res); 21970fdefe2cSJingoo Han if (IS_ERR(pdata->ioaddr)) 21980fdefe2cSJingoo Han return PTR_ERR(pdata->ioaddr); 2199c2afad6cSKamil Konieczny } 220089245107SNaveen Krishna Chatradhi 22010918f18cSKamil Konieczny pdata->clk = devm_clk_get(dev, variant->clk_names[0]); 2202b7da560eSKrzysztof Kozlowski if (IS_ERR(pdata->clk)) 2203b7da560eSKrzysztof Kozlowski return dev_err_probe(dev, PTR_ERR(pdata->clk), 2204b7da560eSKrzysztof Kozlowski "failed to find secss clock %s\n", 22050918f18cSKamil Konieczny variant->clk_names[0]); 2206a49e490cSVladimir Zapolskiy 2207c1eb7ef2SNaveen Krishna Chatradhi err = clk_prepare_enable(pdata->clk); 2208c1eb7ef2SNaveen Krishna Chatradhi if (err < 0) { 22090918f18cSKamil Konieczny dev_err(dev, "Enabling clock %s failed, err %d\n", 22100918f18cSKamil Konieczny variant->clk_names[0], err); 2211c1eb7ef2SNaveen Krishna Chatradhi return err; 2212c1eb7ef2SNaveen Krishna Chatradhi } 2213a49e490cSVladimir Zapolskiy 22140918f18cSKamil Konieczny if (variant->clk_names[1]) { 22150918f18cSKamil Konieczny pdata->pclk = devm_clk_get(dev, variant->clk_names[1]); 22160918f18cSKamil Konieczny if (IS_ERR(pdata->pclk)) { 2217b7da560eSKrzysztof Kozlowski err = dev_err_probe(dev, PTR_ERR(pdata->pclk), 2218b7da560eSKrzysztof Kozlowski "failed to find clock %s\n", 22190918f18cSKamil Konieczny variant->clk_names[1]); 22200918f18cSKamil Konieczny goto err_clk; 22210918f18cSKamil Konieczny } 22220918f18cSKamil Konieczny 22230918f18cSKamil Konieczny err = clk_prepare_enable(pdata->pclk); 22240918f18cSKamil Konieczny if (err < 0) { 22250918f18cSKamil Konieczny dev_err(dev, "Enabling clock %s failed, err %d\n", 22260918f18cSKamil Konieczny variant->clk_names[0], err); 22270918f18cSKamil Konieczny goto err_clk; 22280918f18cSKamil Konieczny } 22290918f18cSKamil Konieczny } else { 22300918f18cSKamil Konieczny pdata->pclk = NULL; 22310918f18cSKamil Konieczny } 22320918f18cSKamil Konieczny 2233a49e490cSVladimir Zapolskiy spin_lock_init(&pdata->lock); 2234c2afad6cSKamil Konieczny spin_lock_init(&pdata->hash_lock); 2235a49e490cSVladimir Zapolskiy 223689245107SNaveen Krishna Chatradhi pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset; 2237c2afad6cSKamil Konieczny pdata->io_hash_base = pdata->ioaddr + variant->hash_offset; 223889245107SNaveen Krishna Chatradhi 223996fc70b6SNaveen Krishna Chatradhi pdata->irq_fc = platform_get_irq(pdev, 0); 2240a49e490cSVladimir Zapolskiy if (pdata->irq_fc < 0) { 2241a49e490cSVladimir Zapolskiy err = pdata->irq_fc; 2242a49e490cSVladimir Zapolskiy dev_warn(dev, "feed control interrupt is not available.\n"); 2243a49e490cSVladimir Zapolskiy goto err_irq; 2244a49e490cSVladimir Zapolskiy } 224507de4bc8SKrzysztof Kozlowski err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL, 224607de4bc8SKrzysztof Kozlowski s5p_aes_interrupt, IRQF_ONESHOT, 224707de4bc8SKrzysztof Kozlowski pdev->name, pdev); 2248a49e490cSVladimir Zapolskiy if (err < 0) { 2249a49e490cSVladimir Zapolskiy dev_warn(dev, "feed control interrupt is not available.\n"); 2250a49e490cSVladimir Zapolskiy goto err_irq; 2251a49e490cSVladimir Zapolskiy } 2252a49e490cSVladimir Zapolskiy 2253dc5e3f19SNaveen Krishna Chatradhi pdata->busy = false; 2254a49e490cSVladimir Zapolskiy pdata->dev = dev; 2255a49e490cSVladimir Zapolskiy platform_set_drvdata(pdev, pdata); 2256a49e490cSVladimir Zapolskiy s5p_dev = pdata; 2257a49e490cSVladimir Zapolskiy 2258a49e490cSVladimir Zapolskiy tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata); 2259a49e490cSVladimir Zapolskiy crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN); 2260a49e490cSVladimir Zapolskiy 2261a49e490cSVladimir Zapolskiy for (i = 0; i < ARRAY_SIZE(algs); i++) { 2262e6b98ce6SArd Biesheuvel err = crypto_register_skcipher(&algs[i]); 2263a49e490cSVladimir Zapolskiy if (err) 2264a49e490cSVladimir Zapolskiy goto err_algs; 2265a49e490cSVladimir Zapolskiy } 2266a49e490cSVladimir Zapolskiy 2267c2afad6cSKamil Konieczny if (pdata->use_hash) { 2268c2afad6cSKamil Konieczny tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb, 2269c2afad6cSKamil Konieczny (unsigned long)pdata); 2270c2afad6cSKamil Konieczny crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH); 2271c2afad6cSKamil Konieczny 2272c2afad6cSKamil Konieczny for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256); 2273c2afad6cSKamil Konieczny hash_i++) { 2274c2afad6cSKamil Konieczny struct ahash_alg *alg; 2275c2afad6cSKamil Konieczny 2276c2afad6cSKamil Konieczny alg = &algs_sha1_md5_sha256[hash_i]; 2277c2afad6cSKamil Konieczny err = crypto_register_ahash(alg); 2278c2afad6cSKamil Konieczny if (err) { 2279c2afad6cSKamil Konieczny dev_err(dev, "can't register '%s': %d\n", 2280c2afad6cSKamil Konieczny alg->halg.base.cra_driver_name, err); 2281c2afad6cSKamil Konieczny goto err_hash; 2282c2afad6cSKamil Konieczny } 2283c2afad6cSKamil Konieczny } 2284c2afad6cSKamil Konieczny } 2285c2afad6cSKamil Konieczny 2286313becd1SKrzysztof Koz?owski dev_info(dev, "s5p-sss driver registered\n"); 2287a49e490cSVladimir Zapolskiy 2288a49e490cSVladimir Zapolskiy return 0; 2289a49e490cSVladimir Zapolskiy 2290c2afad6cSKamil Konieczny err_hash: 2291c2afad6cSKamil Konieczny for (j = hash_i - 1; j >= 0; j--) 2292c2afad6cSKamil Konieczny crypto_unregister_ahash(&algs_sha1_md5_sha256[j]); 2293c2afad6cSKamil Konieczny 2294c2afad6cSKamil Konieczny tasklet_kill(&pdata->hash_tasklet); 2295c2afad6cSKamil Konieczny res->end -= 0x300; 2296c2afad6cSKamil Konieczny 2297a49e490cSVladimir Zapolskiy err_algs: 2298c2afad6cSKamil Konieczny if (i < ARRAY_SIZE(algs)) 2299e6b98ce6SArd Biesheuvel dev_err(dev, "can't register '%s': %d\n", algs[i].base.cra_name, 2300c2afad6cSKamil Konieczny err); 2301a49e490cSVladimir Zapolskiy 2302a49e490cSVladimir Zapolskiy for (j = 0; j < i; j++) 2303e6b98ce6SArd Biesheuvel crypto_unregister_skcipher(&algs[j]); 2304a49e490cSVladimir Zapolskiy 2305a49e490cSVladimir Zapolskiy tasklet_kill(&pdata->tasklet); 2306a49e490cSVladimir Zapolskiy 2307a49e490cSVladimir Zapolskiy err_irq: 23080918f18cSKamil Konieczny clk_disable_unprepare(pdata->pclk); 2309a49e490cSVladimir Zapolskiy 23100918f18cSKamil Konieczny err_clk: 23110918f18cSKamil Konieczny clk_disable_unprepare(pdata->clk); 2312a49e490cSVladimir Zapolskiy s5p_dev = NULL; 2313a49e490cSVladimir Zapolskiy 2314a49e490cSVladimir Zapolskiy return err; 2315a49e490cSVladimir Zapolskiy } 2316a49e490cSVladimir Zapolskiy 2317a49e490cSVladimir Zapolskiy static int s5p_aes_remove(struct platform_device *pdev) 2318a49e490cSVladimir Zapolskiy { 2319a49e490cSVladimir Zapolskiy struct s5p_aes_dev *pdata = platform_get_drvdata(pdev); 2320a49e490cSVladimir Zapolskiy int i; 2321a49e490cSVladimir Zapolskiy 2322a49e490cSVladimir Zapolskiy if (!pdata) 2323a49e490cSVladimir Zapolskiy return -ENODEV; 2324a49e490cSVladimir Zapolskiy 2325a49e490cSVladimir Zapolskiy for (i = 0; i < ARRAY_SIZE(algs); i++) 2326e6b98ce6SArd Biesheuvel crypto_unregister_skcipher(&algs[i]); 2327a49e490cSVladimir Zapolskiy 2328a49e490cSVladimir Zapolskiy tasklet_kill(&pdata->tasklet); 2329c2afad6cSKamil Konieczny if (pdata->use_hash) { 2330c2afad6cSKamil Konieczny for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--) 2331c2afad6cSKamil Konieczny crypto_unregister_ahash(&algs_sha1_md5_sha256[i]); 2332c2afad6cSKamil Konieczny 2333c2afad6cSKamil Konieczny pdata->res->end -= 0x300; 2334c2afad6cSKamil Konieczny tasklet_kill(&pdata->hash_tasklet); 2335c2afad6cSKamil Konieczny pdata->use_hash = false; 2336c2afad6cSKamil Konieczny } 2337a49e490cSVladimir Zapolskiy 23380918f18cSKamil Konieczny clk_disable_unprepare(pdata->pclk); 23390918f18cSKamil Konieczny 2340c1eb7ef2SNaveen Krishna Chatradhi clk_disable_unprepare(pdata->clk); 2341a49e490cSVladimir Zapolskiy s5p_dev = NULL; 2342a49e490cSVladimir Zapolskiy 2343a49e490cSVladimir Zapolskiy return 0; 2344a49e490cSVladimir Zapolskiy } 2345a49e490cSVladimir Zapolskiy 2346a49e490cSVladimir Zapolskiy static struct platform_driver s5p_aes_crypto = { 2347a49e490cSVladimir Zapolskiy .probe = s5p_aes_probe, 2348a49e490cSVladimir Zapolskiy .remove = s5p_aes_remove, 2349a49e490cSVladimir Zapolskiy .driver = { 2350a49e490cSVladimir Zapolskiy .name = "s5p-secss", 23516b9f16e6SNaveen Krishna Chatradhi .of_match_table = s5p_sss_dt_match, 2352a49e490cSVladimir Zapolskiy }, 2353a49e490cSVladimir Zapolskiy }; 2354a49e490cSVladimir Zapolskiy 2355741e8c2dSAxel Lin module_platform_driver(s5p_aes_crypto); 2356a49e490cSVladimir Zapolskiy 2357a49e490cSVladimir Zapolskiy MODULE_DESCRIPTION("S5PV210 AES hw acceleration support."); 2358a49e490cSVladimir Zapolskiy MODULE_LICENSE("GPL v2"); 2359a49e490cSVladimir Zapolskiy MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>"); 2360c2afad6cSKamil Konieczny MODULE_AUTHOR("Kamil Konieczny <k.konieczny@partner.samsung.com>"); 2361