163ee04c8SGilad Ben-Yossef // SPDX-License-Identifier: GPL-2.0 203963caeSGilad Ben-Yossef /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 363ee04c8SGilad Ben-Yossef 463ee04c8SGilad Ben-Yossef #include <linux/kernel.h> 563ee04c8SGilad Ben-Yossef #include <linux/module.h> 663ee04c8SGilad Ben-Yossef #include <crypto/algapi.h> 763ee04c8SGilad Ben-Yossef #include <crypto/internal/skcipher.h> 800cd6b23SArd Biesheuvel #include <crypto/internal/des.h> 963ee04c8SGilad Ben-Yossef #include <crypto/xts.h> 109b8d51f8SGilad Ben-Yossef #include <crypto/sm4.h> 1163ee04c8SGilad Ben-Yossef #include <crypto/scatterwalk.h> 1263ee04c8SGilad Ben-Yossef 1363ee04c8SGilad Ben-Yossef #include "cc_driver.h" 1463ee04c8SGilad Ben-Yossef #include "cc_lli_defs.h" 1563ee04c8SGilad Ben-Yossef #include "cc_buffer_mgr.h" 1663ee04c8SGilad Ben-Yossef #include "cc_cipher.h" 1763ee04c8SGilad Ben-Yossef #include "cc_request_mgr.h" 1863ee04c8SGilad Ben-Yossef 195620eb6cSArd Biesheuvel #define MAX_SKCIPHER_SEQ_LEN 6 2063ee04c8SGilad Ben-Yossef 2163ee04c8SGilad Ben-Yossef #define template_skcipher template_u.skcipher 2263ee04c8SGilad Ben-Yossef 2363ee04c8SGilad Ben-Yossef struct cc_user_key_info { 2463ee04c8SGilad Ben-Yossef u8 *key; 2563ee04c8SGilad Ben-Yossef dma_addr_t key_dma_addr; 2663ee04c8SGilad Ben-Yossef }; 2763ee04c8SGilad Ben-Yossef 2863ee04c8SGilad Ben-Yossef struct cc_hw_key_info { 2963ee04c8SGilad Ben-Yossef enum cc_hw_crypto_key key1_slot; 3063ee04c8SGilad Ben-Yossef enum cc_hw_crypto_key key2_slot; 3163ee04c8SGilad Ben-Yossef }; 3263ee04c8SGilad Ben-Yossef 3352f42c65SGilad Ben-Yossef struct cc_cpp_key_info { 3452f42c65SGilad Ben-Yossef u8 slot; 3552f42c65SGilad Ben-Yossef enum cc_cpp_alg alg; 3652f42c65SGilad Ben-Yossef }; 3752f42c65SGilad Ben-Yossef 3852f42c65SGilad Ben-Yossef enum cc_key_type { 3952f42c65SGilad Ben-Yossef CC_UNPROTECTED_KEY, /* User key */ 4052f42c65SGilad Ben-Yossef CC_HW_PROTECTED_KEY, /* HW (FDE) key */ 4152f42c65SGilad Ben-Yossef CC_POLICY_PROTECTED_KEY, /* CPP key */ 4252f42c65SGilad Ben-Yossef CC_INVALID_PROTECTED_KEY /* Invalid key */ 4352f42c65SGilad Ben-Yossef }; 4452f42c65SGilad Ben-Yossef 4563ee04c8SGilad Ben-Yossef struct cc_cipher_ctx { 4663ee04c8SGilad Ben-Yossef struct cc_drvdata *drvdata; 4763ee04c8SGilad Ben-Yossef int keylen; 4863ee04c8SGilad Ben-Yossef int cipher_mode; 4963ee04c8SGilad Ben-Yossef int flow_mode; 5063ee04c8SGilad Ben-Yossef unsigned int flags; 5152f42c65SGilad Ben-Yossef enum cc_key_type key_type; 5263ee04c8SGilad Ben-Yossef struct cc_user_key_info user; 5352f42c65SGilad Ben-Yossef union { 5463ee04c8SGilad Ben-Yossef struct cc_hw_key_info hw; 5552f42c65SGilad Ben-Yossef struct cc_cpp_key_info cpp; 5652f42c65SGilad Ben-Yossef }; 5763ee04c8SGilad Ben-Yossef struct crypto_shash *shash_tfm; 58520f325aSGilad Ben-Yossef struct crypto_skcipher *fallback_tfm; 59520f325aSGilad Ben-Yossef bool fallback_on; 6063ee04c8SGilad Ben-Yossef }; 6163ee04c8SGilad Ben-Yossef 6263ee04c8SGilad Ben-Yossef static void cc_cipher_complete(struct device *dev, void *cc_req, int err); 6363ee04c8SGilad Ben-Yossef 6452f42c65SGilad Ben-Yossef static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm) 65a794d8d8SGilad Ben-Yossef { 66a794d8d8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 67a794d8d8SGilad Ben-Yossef 6852f42c65SGilad Ben-Yossef return ctx_p->key_type; 69a794d8d8SGilad Ben-Yossef } 70a794d8d8SGilad Ben-Yossef 7163ee04c8SGilad Ben-Yossef static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) 7263ee04c8SGilad Ben-Yossef { 7363ee04c8SGilad Ben-Yossef switch (ctx_p->flow_mode) { 7463ee04c8SGilad Ben-Yossef case S_DIN_to_AES: 7563ee04c8SGilad Ben-Yossef switch (size) { 7663ee04c8SGilad Ben-Yossef case CC_AES_128_BIT_KEY_SIZE: 7763ee04c8SGilad Ben-Yossef case CC_AES_192_BIT_KEY_SIZE: 781d8b41ffSGilad Ben-Yossef if (ctx_p->cipher_mode != DRV_CIPHER_XTS) 7963ee04c8SGilad Ben-Yossef return 0; 8063ee04c8SGilad Ben-Yossef break; 8163ee04c8SGilad Ben-Yossef case CC_AES_256_BIT_KEY_SIZE: 8263ee04c8SGilad Ben-Yossef return 0; 8363ee04c8SGilad Ben-Yossef case (CC_AES_192_BIT_KEY_SIZE * 2): 8463ee04c8SGilad Ben-Yossef case (CC_AES_256_BIT_KEY_SIZE * 2): 8563ee04c8SGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 861d8b41ffSGilad Ben-Yossef ctx_p->cipher_mode == DRV_CIPHER_ESSIV) 8763ee04c8SGilad Ben-Yossef return 0; 8863ee04c8SGilad Ben-Yossef break; 8963ee04c8SGilad Ben-Yossef default: 9063ee04c8SGilad Ben-Yossef break; 9163ee04c8SGilad Ben-Yossef } 92b5be8531SGustavo A. R. Silva break; 9363ee04c8SGilad Ben-Yossef case S_DIN_to_DES: 9463ee04c8SGilad Ben-Yossef if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE) 9563ee04c8SGilad Ben-Yossef return 0; 9663ee04c8SGilad Ben-Yossef break; 979b8d51f8SGilad Ben-Yossef case S_DIN_to_SM4: 989b8d51f8SGilad Ben-Yossef if (size == SM4_KEY_SIZE) 999b8d51f8SGilad Ben-Yossef return 0; 100abc6146aSGustavo A. R. Silva break; 10163ee04c8SGilad Ben-Yossef default: 10263ee04c8SGilad Ben-Yossef break; 10363ee04c8SGilad Ben-Yossef } 10463ee04c8SGilad Ben-Yossef return -EINVAL; 10563ee04c8SGilad Ben-Yossef } 10663ee04c8SGilad Ben-Yossef 10763ee04c8SGilad Ben-Yossef static int validate_data_size(struct cc_cipher_ctx *ctx_p, 10863ee04c8SGilad Ben-Yossef unsigned int size) 10963ee04c8SGilad Ben-Yossef { 11063ee04c8SGilad Ben-Yossef switch (ctx_p->flow_mode) { 11163ee04c8SGilad Ben-Yossef case S_DIN_to_AES: 11263ee04c8SGilad Ben-Yossef switch (ctx_p->cipher_mode) { 11363ee04c8SGilad Ben-Yossef case DRV_CIPHER_XTS: 11463ee04c8SGilad Ben-Yossef case DRV_CIPHER_CBC_CTS: 11563ee04c8SGilad Ben-Yossef if (size >= AES_BLOCK_SIZE) 11663ee04c8SGilad Ben-Yossef return 0; 11763ee04c8SGilad Ben-Yossef break; 11863ee04c8SGilad Ben-Yossef case DRV_CIPHER_OFB: 11963ee04c8SGilad Ben-Yossef case DRV_CIPHER_CTR: 12063ee04c8SGilad Ben-Yossef return 0; 12163ee04c8SGilad Ben-Yossef case DRV_CIPHER_ECB: 12263ee04c8SGilad Ben-Yossef case DRV_CIPHER_CBC: 12363ee04c8SGilad Ben-Yossef case DRV_CIPHER_ESSIV: 12463ee04c8SGilad Ben-Yossef if (IS_ALIGNED(size, AES_BLOCK_SIZE)) 12563ee04c8SGilad Ben-Yossef return 0; 12663ee04c8SGilad Ben-Yossef break; 12763ee04c8SGilad Ben-Yossef default: 12863ee04c8SGilad Ben-Yossef break; 12963ee04c8SGilad Ben-Yossef } 13063ee04c8SGilad Ben-Yossef break; 13163ee04c8SGilad Ben-Yossef case S_DIN_to_DES: 13263ee04c8SGilad Ben-Yossef if (IS_ALIGNED(size, DES_BLOCK_SIZE)) 13363ee04c8SGilad Ben-Yossef return 0; 13463ee04c8SGilad Ben-Yossef break; 1359b8d51f8SGilad Ben-Yossef case S_DIN_to_SM4: 1369b8d51f8SGilad Ben-Yossef switch (ctx_p->cipher_mode) { 1379b8d51f8SGilad Ben-Yossef case DRV_CIPHER_CTR: 1389b8d51f8SGilad Ben-Yossef return 0; 1399b8d51f8SGilad Ben-Yossef case DRV_CIPHER_ECB: 1409b8d51f8SGilad Ben-Yossef case DRV_CIPHER_CBC: 1419b8d51f8SGilad Ben-Yossef if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) 1429b8d51f8SGilad Ben-Yossef return 0; 143abc6146aSGustavo A. R. Silva break; 1449b8d51f8SGilad Ben-Yossef default: 1459b8d51f8SGilad Ben-Yossef break; 1469b8d51f8SGilad Ben-Yossef } 147abc6146aSGustavo A. R. Silva break; 14863ee04c8SGilad Ben-Yossef default: 14963ee04c8SGilad Ben-Yossef break; 15063ee04c8SGilad Ben-Yossef } 15163ee04c8SGilad Ben-Yossef return -EINVAL; 15263ee04c8SGilad Ben-Yossef } 15363ee04c8SGilad Ben-Yossef 15463ee04c8SGilad Ben-Yossef static int cc_cipher_init(struct crypto_tfm *tfm) 15563ee04c8SGilad Ben-Yossef { 15663ee04c8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 15763ee04c8SGilad Ben-Yossef struct cc_crypto_alg *cc_alg = 15863ee04c8SGilad Ben-Yossef container_of(tfm->__crt_alg, struct cc_crypto_alg, 15963ee04c8SGilad Ben-Yossef skcipher_alg.base); 16063ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(cc_alg->drvdata); 16163ee04c8SGilad Ben-Yossef unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 162520f325aSGilad Ben-Yossef unsigned int fallback_req_size = 0; 16363ee04c8SGilad Ben-Yossef 16463ee04c8SGilad Ben-Yossef dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p, 16563ee04c8SGilad Ben-Yossef crypto_tfm_alg_name(tfm)); 16663ee04c8SGilad Ben-Yossef 16763ee04c8SGilad Ben-Yossef ctx_p->cipher_mode = cc_alg->cipher_mode; 16863ee04c8SGilad Ben-Yossef ctx_p->flow_mode = cc_alg->flow_mode; 16963ee04c8SGilad Ben-Yossef ctx_p->drvdata = cc_alg->drvdata; 17063ee04c8SGilad Ben-Yossef 1719bc6165dSGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 172520f325aSGilad Ben-Yossef const char *name = crypto_tfm_alg_name(tfm); 173520f325aSGilad Ben-Yossef 1749bc6165dSGilad Ben-Yossef /* Alloc hash tfm for essiv */ 175520f325aSGilad Ben-Yossef ctx_p->shash_tfm = crypto_alloc_shash("sha256", 0, 0); 1769bc6165dSGilad Ben-Yossef if (IS_ERR(ctx_p->shash_tfm)) { 1779bc6165dSGilad Ben-Yossef dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); 1789bc6165dSGilad Ben-Yossef return PTR_ERR(ctx_p->shash_tfm); 1799bc6165dSGilad Ben-Yossef } 180520f325aSGilad Ben-Yossef max_key_buf_size <<= 1; 181520f325aSGilad Ben-Yossef 182520f325aSGilad Ben-Yossef /* Alloc fallabck tfm or essiv when key size != 256 bit */ 183520f325aSGilad Ben-Yossef ctx_p->fallback_tfm = 184520f325aSGilad Ben-Yossef crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); 185520f325aSGilad Ben-Yossef 186520f325aSGilad Ben-Yossef if (IS_ERR(ctx_p->fallback_tfm)) { 187520f325aSGilad Ben-Yossef /* Note we're still allowing registration with no fallback since it's 188520f325aSGilad Ben-Yossef * better to have most modes supported than none at all. 189520f325aSGilad Ben-Yossef */ 190520f325aSGilad Ben-Yossef dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n", 191520f325aSGilad Ben-Yossef name); 192520f325aSGilad Ben-Yossef ctx_p->fallback_tfm = NULL; 193520f325aSGilad Ben-Yossef } else { 194520f325aSGilad Ben-Yossef fallback_req_size = crypto_skcipher_reqsize(ctx_p->fallback_tfm); 195520f325aSGilad Ben-Yossef } 1969bc6165dSGilad Ben-Yossef } 1979bc6165dSGilad Ben-Yossef 198520f325aSGilad Ben-Yossef crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 199520f325aSGilad Ben-Yossef sizeof(struct cipher_req_ctx) + fallback_req_size); 200520f325aSGilad Ben-Yossef 20163ee04c8SGilad Ben-Yossef /* Allocate key buffer, cache line aligned */ 202520f325aSGilad Ben-Yossef ctx_p->user.key = kzalloc(max_key_buf_size, GFP_KERNEL); 20363ee04c8SGilad Ben-Yossef if (!ctx_p->user.key) 204520f325aSGilad Ben-Yossef goto free_fallback; 20563ee04c8SGilad Ben-Yossef 20663ee04c8SGilad Ben-Yossef dev_dbg(dev, "Allocated key buffer in context. key=@%p\n", 20763ee04c8SGilad Ben-Yossef ctx_p->user.key); 20863ee04c8SGilad Ben-Yossef 20963ee04c8SGilad Ben-Yossef /* Map key buffer */ 210f4274eecSGeert Uytterhoeven ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key, 21163ee04c8SGilad Ben-Yossef max_key_buf_size, 21263ee04c8SGilad Ben-Yossef DMA_TO_DEVICE); 21363ee04c8SGilad Ben-Yossef if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { 21463ee04c8SGilad Ben-Yossef dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n", 21563ee04c8SGilad Ben-Yossef max_key_buf_size, ctx_p->user.key); 2169bc6165dSGilad Ben-Yossef goto free_key; 21763ee04c8SGilad Ben-Yossef } 21863ee04c8SGilad Ben-Yossef dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n", 21963ee04c8SGilad Ben-Yossef max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); 22063ee04c8SGilad Ben-Yossef 2219bc6165dSGilad Ben-Yossef return 0; 22263ee04c8SGilad Ben-Yossef 2239bc6165dSGilad Ben-Yossef free_key: 2249bc6165dSGilad Ben-Yossef kfree(ctx_p->user.key); 225520f325aSGilad Ben-Yossef free_fallback: 226520f325aSGilad Ben-Yossef crypto_free_skcipher(ctx_p->fallback_tfm); 2279bc6165dSGilad Ben-Yossef crypto_free_shash(ctx_p->shash_tfm); 2289bc6165dSGilad Ben-Yossef 2299bc6165dSGilad Ben-Yossef return -ENOMEM; 23063ee04c8SGilad Ben-Yossef } 23163ee04c8SGilad Ben-Yossef 23263ee04c8SGilad Ben-Yossef static void cc_cipher_exit(struct crypto_tfm *tfm) 23363ee04c8SGilad Ben-Yossef { 23463ee04c8SGilad Ben-Yossef struct crypto_alg *alg = tfm->__crt_alg; 23563ee04c8SGilad Ben-Yossef struct cc_crypto_alg *cc_alg = 23663ee04c8SGilad Ben-Yossef container_of(alg, struct cc_crypto_alg, 23763ee04c8SGilad Ben-Yossef skcipher_alg.base); 23863ee04c8SGilad Ben-Yossef unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 23963ee04c8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 24063ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 24163ee04c8SGilad Ben-Yossef 24263ee04c8SGilad Ben-Yossef dev_dbg(dev, "Clearing context @%p for %s\n", 24363ee04c8SGilad Ben-Yossef crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm)); 24463ee04c8SGilad Ben-Yossef 24563ee04c8SGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 24663ee04c8SGilad Ben-Yossef /* Free hash tfm for essiv */ 24763ee04c8SGilad Ben-Yossef crypto_free_shash(ctx_p->shash_tfm); 24863ee04c8SGilad Ben-Yossef ctx_p->shash_tfm = NULL; 249520f325aSGilad Ben-Yossef crypto_free_skcipher(ctx_p->fallback_tfm); 250520f325aSGilad Ben-Yossef ctx_p->fallback_tfm = NULL; 25163ee04c8SGilad Ben-Yossef } 25263ee04c8SGilad Ben-Yossef 25363ee04c8SGilad Ben-Yossef /* Unmap key buffer */ 25463ee04c8SGilad Ben-Yossef dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size, 25563ee04c8SGilad Ben-Yossef DMA_TO_DEVICE); 25663ee04c8SGilad Ben-Yossef dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n", 25763ee04c8SGilad Ben-Yossef &ctx_p->user.key_dma_addr); 25863ee04c8SGilad Ben-Yossef 25963ee04c8SGilad Ben-Yossef /* Free key buffer in context */ 26063ee04c8SGilad Ben-Yossef dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key); 261*3d950c34SJianglei Nie kfree_sensitive(ctx_p->user.key); 26263ee04c8SGilad Ben-Yossef } 26363ee04c8SGilad Ben-Yossef 26463ee04c8SGilad Ben-Yossef struct tdes_keys { 26563ee04c8SGilad Ben-Yossef u8 key1[DES_KEY_SIZE]; 26663ee04c8SGilad Ben-Yossef u8 key2[DES_KEY_SIZE]; 26763ee04c8SGilad Ben-Yossef u8 key3[DES_KEY_SIZE]; 26863ee04c8SGilad Ben-Yossef }; 26963ee04c8SGilad Ben-Yossef 27052f42c65SGilad Ben-Yossef static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num) 27163ee04c8SGilad Ben-Yossef { 27263ee04c8SGilad Ben-Yossef switch (slot_num) { 27363ee04c8SGilad Ben-Yossef case 0: 27463ee04c8SGilad Ben-Yossef return KFDE0_KEY; 27563ee04c8SGilad Ben-Yossef case 1: 27663ee04c8SGilad Ben-Yossef return KFDE1_KEY; 27763ee04c8SGilad Ben-Yossef case 2: 27863ee04c8SGilad Ben-Yossef return KFDE2_KEY; 27963ee04c8SGilad Ben-Yossef case 3: 28063ee04c8SGilad Ben-Yossef return KFDE3_KEY; 28163ee04c8SGilad Ben-Yossef } 28263ee04c8SGilad Ben-Yossef return END_OF_KEYS; 28363ee04c8SGilad Ben-Yossef } 28463ee04c8SGilad Ben-Yossef 28552f42c65SGilad Ben-Yossef static u8 cc_slot_to_cpp_key(u8 slot_num) 28652f42c65SGilad Ben-Yossef { 28752f42c65SGilad Ben-Yossef return (slot_num - CC_FIRST_CPP_KEY_SLOT); 28852f42c65SGilad Ben-Yossef } 28952f42c65SGilad Ben-Yossef 29052f42c65SGilad Ben-Yossef static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num) 29152f42c65SGilad Ben-Yossef { 29252f42c65SGilad Ben-Yossef if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT) 29352f42c65SGilad Ben-Yossef return CC_HW_PROTECTED_KEY; 29452f42c65SGilad Ben-Yossef else if (slot_num >= CC_FIRST_CPP_KEY_SLOT && 29552f42c65SGilad Ben-Yossef slot_num <= CC_LAST_CPP_KEY_SLOT) 29652f42c65SGilad Ben-Yossef return CC_POLICY_PROTECTED_KEY; 29752f42c65SGilad Ben-Yossef else 29852f42c65SGilad Ben-Yossef return CC_INVALID_PROTECTED_KEY; 29952f42c65SGilad Ben-Yossef } 30052f42c65SGilad Ben-Yossef 301a794d8d8SGilad Ben-Yossef static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, 302a794d8d8SGilad Ben-Yossef unsigned int keylen) 303a794d8d8SGilad Ben-Yossef { 304a794d8d8SGilad Ben-Yossef struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 305a794d8d8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 306a794d8d8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 307a794d8d8SGilad Ben-Yossef struct cc_hkey_info hki; 308a794d8d8SGilad Ben-Yossef 309a794d8d8SGilad Ben-Yossef dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n", 310a794d8d8SGilad Ben-Yossef ctx_p, crypto_tfm_alg_name(tfm), keylen); 311f4274eecSGeert Uytterhoeven dump_byte_array("key", key, keylen); 312a794d8d8SGilad Ben-Yossef 313a794d8d8SGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 314a794d8d8SGilad Ben-Yossef 31552f42c65SGilad Ben-Yossef /* This check the size of the protected key token */ 316a794d8d8SGilad Ben-Yossef if (keylen != sizeof(hki)) { 31752f42c65SGilad Ben-Yossef dev_err(dev, "Unsupported protected key size %d.\n", keylen); 318a794d8d8SGilad Ben-Yossef return -EINVAL; 319a794d8d8SGilad Ben-Yossef } 320a794d8d8SGilad Ben-Yossef 321a794d8d8SGilad Ben-Yossef memcpy(&hki, key, keylen); 322a794d8d8SGilad Ben-Yossef 323a794d8d8SGilad Ben-Yossef /* The real key len for crypto op is the size of the HW key 324a794d8d8SGilad Ben-Yossef * referenced by the HW key slot, not the hardware key token 325a794d8d8SGilad Ben-Yossef */ 326a794d8d8SGilad Ben-Yossef keylen = hki.keylen; 327a794d8d8SGilad Ben-Yossef 328a794d8d8SGilad Ben-Yossef if (validate_keys_sizes(ctx_p, keylen)) { 329c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Unsupported key size %d.\n", keylen); 330a794d8d8SGilad Ben-Yossef return -EINVAL; 331a794d8d8SGilad Ben-Yossef } 332a794d8d8SGilad Ben-Yossef 33352f42c65SGilad Ben-Yossef ctx_p->keylen = keylen; 334520f325aSGilad Ben-Yossef ctx_p->fallback_on = false; 33552f42c65SGilad Ben-Yossef 33652f42c65SGilad Ben-Yossef switch (cc_slot_to_key_type(hki.hw_key1)) { 33752f42c65SGilad Ben-Yossef case CC_HW_PROTECTED_KEY: 33852f42c65SGilad Ben-Yossef if (ctx_p->flow_mode == S_DIN_to_SM4) { 33952f42c65SGilad Ben-Yossef dev_err(dev, "Only AES HW protected keys are supported\n"); 34052f42c65SGilad Ben-Yossef return -EINVAL; 34152f42c65SGilad Ben-Yossef } 34252f42c65SGilad Ben-Yossef 343a794d8d8SGilad Ben-Yossef ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); 344a794d8d8SGilad Ben-Yossef if (ctx_p->hw.key1_slot == END_OF_KEYS) { 34552f42c65SGilad Ben-Yossef dev_err(dev, "Unsupported hw key1 number (%d)\n", 34652f42c65SGilad Ben-Yossef hki.hw_key1); 347a794d8d8SGilad Ben-Yossef return -EINVAL; 348a794d8d8SGilad Ben-Yossef } 349a794d8d8SGilad Ben-Yossef 350a794d8d8SGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 3511d8b41ffSGilad Ben-Yossef ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 352a794d8d8SGilad Ben-Yossef if (hki.hw_key1 == hki.hw_key2) { 353a794d8d8SGilad Ben-Yossef dev_err(dev, "Illegal hw key numbers (%d,%d)\n", 354a794d8d8SGilad Ben-Yossef hki.hw_key1, hki.hw_key2); 355a794d8d8SGilad Ben-Yossef return -EINVAL; 356a794d8d8SGilad Ben-Yossef } 35752f42c65SGilad Ben-Yossef 358a794d8d8SGilad Ben-Yossef ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); 359a794d8d8SGilad Ben-Yossef if (ctx_p->hw.key2_slot == END_OF_KEYS) { 360a794d8d8SGilad Ben-Yossef dev_err(dev, "Unsupported hw key2 number (%d)\n", 361a794d8d8SGilad Ben-Yossef hki.hw_key2); 362a794d8d8SGilad Ben-Yossef return -EINVAL; 363a794d8d8SGilad Ben-Yossef } 364a794d8d8SGilad Ben-Yossef } 365a794d8d8SGilad Ben-Yossef 36652f42c65SGilad Ben-Yossef ctx_p->key_type = CC_HW_PROTECTED_KEY; 36752f42c65SGilad Ben-Yossef dev_dbg(dev, "HW protected key %d/%d set\n.", 36852f42c65SGilad Ben-Yossef ctx_p->hw.key1_slot, ctx_p->hw.key2_slot); 36952f42c65SGilad Ben-Yossef break; 37052f42c65SGilad Ben-Yossef 37152f42c65SGilad Ben-Yossef case CC_POLICY_PROTECTED_KEY: 37252f42c65SGilad Ben-Yossef if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) { 37352f42c65SGilad Ben-Yossef dev_err(dev, "CPP keys not supported in this hardware revision.\n"); 37452f42c65SGilad Ben-Yossef return -EINVAL; 37552f42c65SGilad Ben-Yossef } 37652f42c65SGilad Ben-Yossef 37752f42c65SGilad Ben-Yossef if (ctx_p->cipher_mode != DRV_CIPHER_CBC && 37852f42c65SGilad Ben-Yossef ctx_p->cipher_mode != DRV_CIPHER_CTR) { 37952f42c65SGilad Ben-Yossef dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n"); 38052f42c65SGilad Ben-Yossef return -EINVAL; 38152f42c65SGilad Ben-Yossef } 38252f42c65SGilad Ben-Yossef 38352f42c65SGilad Ben-Yossef ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1); 38452f42c65SGilad Ben-Yossef if (ctx_p->flow_mode == S_DIN_to_AES) 38552f42c65SGilad Ben-Yossef ctx_p->cpp.alg = CC_CPP_AES; 38652f42c65SGilad Ben-Yossef else /* Must be SM4 since due to sethkey registration */ 38752f42c65SGilad Ben-Yossef ctx_p->cpp.alg = CC_CPP_SM4; 38852f42c65SGilad Ben-Yossef ctx_p->key_type = CC_POLICY_PROTECTED_KEY; 38922e2db68SColin Ian King dev_dbg(dev, "policy protected key alg: %d slot: %d.\n", 39052f42c65SGilad Ben-Yossef ctx_p->cpp.alg, ctx_p->cpp.slot); 39152f42c65SGilad Ben-Yossef break; 39252f42c65SGilad Ben-Yossef 39352f42c65SGilad Ben-Yossef default: 39452f42c65SGilad Ben-Yossef dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1); 39552f42c65SGilad Ben-Yossef return -EINVAL; 39652f42c65SGilad Ben-Yossef } 397a794d8d8SGilad Ben-Yossef 398a794d8d8SGilad Ben-Yossef return 0; 399a794d8d8SGilad Ben-Yossef } 400a794d8d8SGilad Ben-Yossef 40163ee04c8SGilad Ben-Yossef static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, 40263ee04c8SGilad Ben-Yossef unsigned int keylen) 40363ee04c8SGilad Ben-Yossef { 40463ee04c8SGilad Ben-Yossef struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 40563ee04c8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 40663ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 40763ee04c8SGilad Ben-Yossef struct cc_crypto_alg *cc_alg = 40863ee04c8SGilad Ben-Yossef container_of(tfm->__crt_alg, struct cc_crypto_alg, 40963ee04c8SGilad Ben-Yossef skcipher_alg.base); 41063ee04c8SGilad Ben-Yossef unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 41163ee04c8SGilad Ben-Yossef 41263ee04c8SGilad Ben-Yossef dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n", 41363ee04c8SGilad Ben-Yossef ctx_p, crypto_tfm_alg_name(tfm), keylen); 414f4274eecSGeert Uytterhoeven dump_byte_array("key", key, keylen); 41563ee04c8SGilad Ben-Yossef 41663ee04c8SGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 41763ee04c8SGilad Ben-Yossef 41863ee04c8SGilad Ben-Yossef if (validate_keys_sizes(ctx_p, keylen)) { 419520f325aSGilad Ben-Yossef dev_dbg(dev, "Invalid key size %d.\n", keylen); 42063ee04c8SGilad Ben-Yossef return -EINVAL; 42163ee04c8SGilad Ben-Yossef } 42263ee04c8SGilad Ben-Yossef 423520f325aSGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 424520f325aSGilad Ben-Yossef 425520f325aSGilad Ben-Yossef /* We only support 256 bit ESSIV-CBC-AES keys */ 426520f325aSGilad Ben-Yossef if (keylen != AES_KEYSIZE_256) { 427520f325aSGilad Ben-Yossef unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK; 428520f325aSGilad Ben-Yossef 429520f325aSGilad Ben-Yossef if (likely(ctx_p->fallback_tfm)) { 430520f325aSGilad Ben-Yossef ctx_p->fallback_on = true; 431520f325aSGilad Ben-Yossef crypto_skcipher_clear_flags(ctx_p->fallback_tfm, 432520f325aSGilad Ben-Yossef CRYPTO_TFM_REQ_MASK); 433520f325aSGilad Ben-Yossef crypto_skcipher_clear_flags(ctx_p->fallback_tfm, flags); 434520f325aSGilad Ben-Yossef return crypto_skcipher_setkey(ctx_p->fallback_tfm, key, keylen); 435520f325aSGilad Ben-Yossef } 436520f325aSGilad Ben-Yossef 437520f325aSGilad Ben-Yossef dev_dbg(dev, "Unsupported key size %d and no fallback.\n", keylen); 438520f325aSGilad Ben-Yossef return -EINVAL; 439520f325aSGilad Ben-Yossef } 440520f325aSGilad Ben-Yossef 441520f325aSGilad Ben-Yossef /* Internal ESSIV key buffer is double sized */ 442520f325aSGilad Ben-Yossef max_key_buf_size <<= 1; 443520f325aSGilad Ben-Yossef } 444520f325aSGilad Ben-Yossef 445520f325aSGilad Ben-Yossef ctx_p->fallback_on = false; 44652f42c65SGilad Ben-Yossef ctx_p->key_type = CC_UNPROTECTED_KEY; 44763ee04c8SGilad Ben-Yossef 44863ee04c8SGilad Ben-Yossef /* 44963ee04c8SGilad Ben-Yossef * Verify DES weak keys 45063ee04c8SGilad Ben-Yossef * Note that we're dropping the expanded key since the 45163ee04c8SGilad Ben-Yossef * HW does the expansion on its own. 45263ee04c8SGilad Ben-Yossef */ 45363ee04c8SGilad Ben-Yossef if (ctx_p->flow_mode == S_DIN_to_DES) { 45400cd6b23SArd Biesheuvel if ((keylen == DES3_EDE_KEY_SIZE && 45500cd6b23SArd Biesheuvel verify_skcipher_des3_key(sktfm, key)) || 45600cd6b23SArd Biesheuvel verify_skcipher_des_key(sktfm, key)) { 45763ee04c8SGilad Ben-Yossef dev_dbg(dev, "weak DES key"); 45863ee04c8SGilad Ben-Yossef return -EINVAL; 45963ee04c8SGilad Ben-Yossef } 46063ee04c8SGilad Ben-Yossef } 46163ee04c8SGilad Ben-Yossef 46263ee04c8SGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_XTS && 46363ee04c8SGilad Ben-Yossef xts_check_key(tfm, key, keylen)) { 46463ee04c8SGilad Ben-Yossef dev_dbg(dev, "weak XTS key"); 46563ee04c8SGilad Ben-Yossef return -EINVAL; 46663ee04c8SGilad Ben-Yossef } 46763ee04c8SGilad Ben-Yossef 46863ee04c8SGilad Ben-Yossef /* STAT_PHASE_1: Copy key to ctx */ 46963ee04c8SGilad Ben-Yossef dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr, 47063ee04c8SGilad Ben-Yossef max_key_buf_size, DMA_TO_DEVICE); 47163ee04c8SGilad Ben-Yossef 47263ee04c8SGilad Ben-Yossef memcpy(ctx_p->user.key, key, keylen); 47363ee04c8SGilad Ben-Yossef 47463ee04c8SGilad Ben-Yossef if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 47563ee04c8SGilad Ben-Yossef /* sha256 for key2 - use sw implementation */ 47663ee04c8SGilad Ben-Yossef int err; 47763ee04c8SGilad Ben-Yossef 4788cbb8097SEric Biggers err = crypto_shash_tfm_digest(ctx_p->shash_tfm, 479520f325aSGilad Ben-Yossef ctx_p->user.key, keylen, 480520f325aSGilad Ben-Yossef ctx_p->user.key + keylen); 48163ee04c8SGilad Ben-Yossef if (err) { 48263ee04c8SGilad Ben-Yossef dev_err(dev, "Failed to hash ESSIV key.\n"); 48363ee04c8SGilad Ben-Yossef return err; 48463ee04c8SGilad Ben-Yossef } 485520f325aSGilad Ben-Yossef 486520f325aSGilad Ben-Yossef keylen <<= 1; 48763ee04c8SGilad Ben-Yossef } 48863ee04c8SGilad Ben-Yossef dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr, 48963ee04c8SGilad Ben-Yossef max_key_buf_size, DMA_TO_DEVICE); 49063ee04c8SGilad Ben-Yossef ctx_p->keylen = keylen; 49163ee04c8SGilad Ben-Yossef 49263ee04c8SGilad Ben-Yossef dev_dbg(dev, "return safely"); 49363ee04c8SGilad Ben-Yossef return 0; 49463ee04c8SGilad Ben-Yossef } 49563ee04c8SGilad Ben-Yossef 4966f17e00fSGilad Ben-Yossef static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p) 4976f17e00fSGilad Ben-Yossef { 4986f17e00fSGilad Ben-Yossef switch (ctx_p->flow_mode) { 4996f17e00fSGilad Ben-Yossef case S_DIN_to_AES: 5006f17e00fSGilad Ben-Yossef return S_AES_to_DOUT; 5016f17e00fSGilad Ben-Yossef case S_DIN_to_DES: 5026f17e00fSGilad Ben-Yossef return S_DES_to_DOUT; 5036f17e00fSGilad Ben-Yossef case S_DIN_to_SM4: 5046f17e00fSGilad Ben-Yossef return S_SM4_to_DOUT; 5056f17e00fSGilad Ben-Yossef default: 5066f17e00fSGilad Ben-Yossef return ctx_p->flow_mode; 5076f17e00fSGilad Ben-Yossef } 5086f17e00fSGilad Ben-Yossef } 5096f17e00fSGilad Ben-Yossef 5106f17e00fSGilad Ben-Yossef static void cc_setup_readiv_desc(struct crypto_tfm *tfm, 5116f17e00fSGilad Ben-Yossef struct cipher_req_ctx *req_ctx, 5126f17e00fSGilad Ben-Yossef unsigned int ivsize, struct cc_hw_desc desc[], 5136f17e00fSGilad Ben-Yossef unsigned int *seq_size) 5146f17e00fSGilad Ben-Yossef { 5156f17e00fSGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 5166f17e00fSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 5176f17e00fSGilad Ben-Yossef int cipher_mode = ctx_p->cipher_mode; 5186f17e00fSGilad Ben-Yossef int flow_mode = cc_out_setup_mode(ctx_p); 5196f17e00fSGilad Ben-Yossef int direction = req_ctx->gen_ctx.op_type; 5206f17e00fSGilad Ben-Yossef dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 5216f17e00fSGilad Ben-Yossef 5226f17e00fSGilad Ben-Yossef if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) 5236f17e00fSGilad Ben-Yossef return; 5246f17e00fSGilad Ben-Yossef 5256f17e00fSGilad Ben-Yossef switch (cipher_mode) { 5266f17e00fSGilad Ben-Yossef case DRV_CIPHER_ECB: 5276f17e00fSGilad Ben-Yossef break; 5286f17e00fSGilad Ben-Yossef case DRV_CIPHER_CBC: 5296f17e00fSGilad Ben-Yossef case DRV_CIPHER_CBC_CTS: 5306f17e00fSGilad Ben-Yossef case DRV_CIPHER_CTR: 5316f17e00fSGilad Ben-Yossef case DRV_CIPHER_OFB: 5326f17e00fSGilad Ben-Yossef /* Read next IV */ 5336f17e00fSGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 5346f17e00fSGilad Ben-Yossef set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1); 5356f17e00fSGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 5366f17e00fSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 5376f17e00fSGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 5386f17e00fSGilad Ben-Yossef if (cipher_mode == DRV_CIPHER_CTR || 5396f17e00fSGilad Ben-Yossef cipher_mode == DRV_CIPHER_OFB) { 5406f17e00fSGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); 5416f17e00fSGilad Ben-Yossef } else { 5426f17e00fSGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0); 5436f17e00fSGilad Ben-Yossef } 5446f17e00fSGilad Ben-Yossef set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 5456f17e00fSGilad Ben-Yossef (*seq_size)++; 5466f17e00fSGilad Ben-Yossef break; 5476f17e00fSGilad Ben-Yossef case DRV_CIPHER_XTS: 5486f17e00fSGilad Ben-Yossef case DRV_CIPHER_ESSIV: 5496f17e00fSGilad Ben-Yossef /* IV */ 5506f17e00fSGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 5516f17e00fSGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); 5526f17e00fSGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 5536f17e00fSGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 5546f17e00fSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 5556f17e00fSGilad Ben-Yossef set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE, 5566f17e00fSGilad Ben-Yossef NS_BIT, 1); 5576f17e00fSGilad Ben-Yossef set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 5586f17e00fSGilad Ben-Yossef (*seq_size)++; 5596f17e00fSGilad Ben-Yossef break; 5606f17e00fSGilad Ben-Yossef default: 5616f17e00fSGilad Ben-Yossef dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 5626f17e00fSGilad Ben-Yossef } 5636f17e00fSGilad Ben-Yossef } 5646f17e00fSGilad Ben-Yossef 5655c83e8ecSOfir Drang 566dd8486c7SGilad Ben-Yossef static void cc_setup_state_desc(struct crypto_tfm *tfm, 56763ee04c8SGilad Ben-Yossef struct cipher_req_ctx *req_ctx, 56863ee04c8SGilad Ben-Yossef unsigned int ivsize, unsigned int nbytes, 56963ee04c8SGilad Ben-Yossef struct cc_hw_desc desc[], 57063ee04c8SGilad Ben-Yossef unsigned int *seq_size) 57163ee04c8SGilad Ben-Yossef { 57263ee04c8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 57363ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 57463ee04c8SGilad Ben-Yossef int cipher_mode = ctx_p->cipher_mode; 57563ee04c8SGilad Ben-Yossef int flow_mode = ctx_p->flow_mode; 57663ee04c8SGilad Ben-Yossef int direction = req_ctx->gen_ctx.op_type; 57763ee04c8SGilad Ben-Yossef dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 57863ee04c8SGilad Ben-Yossef 57963ee04c8SGilad Ben-Yossef switch (cipher_mode) { 580dd8486c7SGilad Ben-Yossef case DRV_CIPHER_ECB: 581dd8486c7SGilad Ben-Yossef break; 58263ee04c8SGilad Ben-Yossef case DRV_CIPHER_CBC: 58363ee04c8SGilad Ben-Yossef case DRV_CIPHER_CBC_CTS: 58463ee04c8SGilad Ben-Yossef case DRV_CIPHER_CTR: 58563ee04c8SGilad Ben-Yossef case DRV_CIPHER_OFB: 586dd8486c7SGilad Ben-Yossef /* Load IV */ 58763ee04c8SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 58863ee04c8SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize, 58963ee04c8SGilad Ben-Yossef NS_BIT); 59063ee04c8SGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 59163ee04c8SGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 59263ee04c8SGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 59363ee04c8SGilad Ben-Yossef if (cipher_mode == DRV_CIPHER_CTR || 59463ee04c8SGilad Ben-Yossef cipher_mode == DRV_CIPHER_OFB) { 59563ee04c8SGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 59663ee04c8SGilad Ben-Yossef } else { 59763ee04c8SGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0); 59863ee04c8SGilad Ben-Yossef } 59963ee04c8SGilad Ben-Yossef (*seq_size)++; 600dd8486c7SGilad Ben-Yossef break; 601dd8486c7SGilad Ben-Yossef case DRV_CIPHER_XTS: 602dd8486c7SGilad Ben-Yossef case DRV_CIPHER_ESSIV: 6035c83e8ecSOfir Drang break; 6045c83e8ecSOfir Drang default: 6055c83e8ecSOfir Drang dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 6065c83e8ecSOfir Drang } 6075c83e8ecSOfir Drang } 6085c83e8ecSOfir Drang 6095c83e8ecSOfir Drang 6105c83e8ecSOfir Drang static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, 6115c83e8ecSOfir Drang struct cipher_req_ctx *req_ctx, 6125c83e8ecSOfir Drang unsigned int ivsize, unsigned int nbytes, 6135c83e8ecSOfir Drang struct cc_hw_desc desc[], 6145c83e8ecSOfir Drang unsigned int *seq_size) 6155c83e8ecSOfir Drang { 6165c83e8ecSOfir Drang struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 6175c83e8ecSOfir Drang struct device *dev = drvdata_to_dev(ctx_p->drvdata); 6185c83e8ecSOfir Drang int cipher_mode = ctx_p->cipher_mode; 6195c83e8ecSOfir Drang int flow_mode = ctx_p->flow_mode; 6205c83e8ecSOfir Drang int direction = req_ctx->gen_ctx.op_type; 6215c83e8ecSOfir Drang dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 622520f325aSGilad Ben-Yossef unsigned int key_len = (ctx_p->keylen / 2); 6235c83e8ecSOfir Drang dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 624520f325aSGilad Ben-Yossef unsigned int key_offset = key_len; 6255c83e8ecSOfir Drang 6265c83e8ecSOfir Drang switch (cipher_mode) { 6275c83e8ecSOfir Drang case DRV_CIPHER_ECB: 6285c83e8ecSOfir Drang break; 6295c83e8ecSOfir Drang case DRV_CIPHER_CBC: 6305c83e8ecSOfir Drang case DRV_CIPHER_CBC_CTS: 6315c83e8ecSOfir Drang case DRV_CIPHER_CTR: 6325c83e8ecSOfir Drang case DRV_CIPHER_OFB: 6335c83e8ecSOfir Drang break; 6345c83e8ecSOfir Drang case DRV_CIPHER_XTS: 6355c83e8ecSOfir Drang case DRV_CIPHER_ESSIV: 636520f325aSGilad Ben-Yossef 637520f325aSGilad Ben-Yossef if (cipher_mode == DRV_CIPHER_ESSIV) 638520f325aSGilad Ben-Yossef key_len = SHA256_DIGEST_SIZE; 639520f325aSGilad Ben-Yossef 640dd8486c7SGilad Ben-Yossef /* load XEX key */ 641dd8486c7SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 642dd8486c7SGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 643dd8486c7SGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 64452f42c65SGilad Ben-Yossef if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 645dd8486c7SGilad Ben-Yossef set_hw_crypto_key(&desc[*seq_size], 646dd8486c7SGilad Ben-Yossef ctx_p->hw.key2_slot); 647dd8486c7SGilad Ben-Yossef } else { 648dd8486c7SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, 649520f325aSGilad Ben-Yossef (key_dma_addr + key_offset), 650520f325aSGilad Ben-Yossef key_len, NS_BIT); 651dd8486c7SGilad Ben-Yossef } 652a93492caSGilad Ben-Yossef set_xex_data_unit_size(&desc[*seq_size], nbytes); 653dd8486c7SGilad Ben-Yossef set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); 654520f325aSGilad Ben-Yossef set_key_size_aes(&desc[*seq_size], key_len); 655dd8486c7SGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); 656dd8486c7SGilad Ben-Yossef (*seq_size)++; 657dd8486c7SGilad Ben-Yossef 658dd8486c7SGilad Ben-Yossef /* Load IV */ 659dd8486c7SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 660dd8486c7SGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 661dd8486c7SGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 662dd8486c7SGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 663520f325aSGilad Ben-Yossef set_key_size_aes(&desc[*seq_size], key_len); 664dd8486c7SGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 665dd8486c7SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, 666dd8486c7SGilad Ben-Yossef CC_AES_BLOCK_SIZE, NS_BIT); 667dd8486c7SGilad Ben-Yossef (*seq_size)++; 668dd8486c7SGilad Ben-Yossef break; 669dd8486c7SGilad Ben-Yossef default: 670dd8486c7SGilad Ben-Yossef dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 671dd8486c7SGilad Ben-Yossef } 672dd8486c7SGilad Ben-Yossef } 673dd8486c7SGilad Ben-Yossef 674533edf9fSGilad Ben-Yossef static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p) 675533edf9fSGilad Ben-Yossef { 676533edf9fSGilad Ben-Yossef switch (ctx_p->flow_mode) { 677533edf9fSGilad Ben-Yossef case S_DIN_to_AES: 678533edf9fSGilad Ben-Yossef return DIN_AES_DOUT; 679533edf9fSGilad Ben-Yossef case S_DIN_to_DES: 680533edf9fSGilad Ben-Yossef return DIN_DES_DOUT; 681533edf9fSGilad Ben-Yossef case S_DIN_to_SM4: 682533edf9fSGilad Ben-Yossef return DIN_SM4_DOUT; 683533edf9fSGilad Ben-Yossef default: 684533edf9fSGilad Ben-Yossef return ctx_p->flow_mode; 685533edf9fSGilad Ben-Yossef } 686533edf9fSGilad Ben-Yossef } 687dd8486c7SGilad Ben-Yossef 688dd8486c7SGilad Ben-Yossef static void cc_setup_key_desc(struct crypto_tfm *tfm, 689dd8486c7SGilad Ben-Yossef struct cipher_req_ctx *req_ctx, 690dd8486c7SGilad Ben-Yossef unsigned int nbytes, struct cc_hw_desc desc[], 691dd8486c7SGilad Ben-Yossef unsigned int *seq_size) 692dd8486c7SGilad Ben-Yossef { 693dd8486c7SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 694dd8486c7SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 695dd8486c7SGilad Ben-Yossef int cipher_mode = ctx_p->cipher_mode; 696dd8486c7SGilad Ben-Yossef int flow_mode = ctx_p->flow_mode; 697dd8486c7SGilad Ben-Yossef int direction = req_ctx->gen_ctx.op_type; 698dd8486c7SGilad Ben-Yossef dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 699dd8486c7SGilad Ben-Yossef unsigned int key_len = ctx_p->keylen; 70052f42c65SGilad Ben-Yossef unsigned int din_size; 701dd8486c7SGilad Ben-Yossef 702dd8486c7SGilad Ben-Yossef switch (cipher_mode) { 703dd8486c7SGilad Ben-Yossef case DRV_CIPHER_CBC: 704dd8486c7SGilad Ben-Yossef case DRV_CIPHER_CBC_CTS: 705dd8486c7SGilad Ben-Yossef case DRV_CIPHER_CTR: 706dd8486c7SGilad Ben-Yossef case DRV_CIPHER_OFB: 70763ee04c8SGilad Ben-Yossef case DRV_CIPHER_ECB: 70863ee04c8SGilad Ben-Yossef /* Load key */ 70963ee04c8SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 71063ee04c8SGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 71163ee04c8SGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 712533edf9fSGilad Ben-Yossef 713533edf9fSGilad Ben-Yossef if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) { 714533edf9fSGilad Ben-Yossef /* We use the AES key size coding for all CPP algs */ 715533edf9fSGilad Ben-Yossef set_key_size_aes(&desc[*seq_size], key_len); 716533edf9fSGilad Ben-Yossef set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot); 717533edf9fSGilad Ben-Yossef flow_mode = cc_out_flow_mode(ctx_p); 718533edf9fSGilad Ben-Yossef } else { 71963ee04c8SGilad Ben-Yossef if (flow_mode == S_DIN_to_AES) { 72052f42c65SGilad Ben-Yossef if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 72163ee04c8SGilad Ben-Yossef set_hw_crypto_key(&desc[*seq_size], 72263ee04c8SGilad Ben-Yossef ctx_p->hw.key1_slot); 72363ee04c8SGilad Ben-Yossef } else { 72452f42c65SGilad Ben-Yossef /* CC_POLICY_UNPROTECTED_KEY 72552f42c65SGilad Ben-Yossef * Invalid keys are filtered out in 72652f42c65SGilad Ben-Yossef * sethkey() 72752f42c65SGilad Ben-Yossef */ 72852f42c65SGilad Ben-Yossef din_size = (key_len == 24) ? 72952f42c65SGilad Ben-Yossef AES_MAX_KEY_SIZE : key_len; 73052f42c65SGilad Ben-Yossef 73163ee04c8SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, 73252f42c65SGilad Ben-Yossef key_dma_addr, din_size, 73352f42c65SGilad Ben-Yossef NS_BIT); 73463ee04c8SGilad Ben-Yossef } 73563ee04c8SGilad Ben-Yossef set_key_size_aes(&desc[*seq_size], key_len); 73663ee04c8SGilad Ben-Yossef } else { 73763ee04c8SGilad Ben-Yossef /*des*/ 73852f42c65SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, 73952f42c65SGilad Ben-Yossef key_dma_addr, key_len, NS_BIT); 74063ee04c8SGilad Ben-Yossef set_key_size_des(&desc[*seq_size], key_len); 74163ee04c8SGilad Ben-Yossef } 74263ee04c8SGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 74352f42c65SGilad Ben-Yossef } 744533edf9fSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 74563ee04c8SGilad Ben-Yossef (*seq_size)++; 74663ee04c8SGilad Ben-Yossef break; 74763ee04c8SGilad Ben-Yossef case DRV_CIPHER_XTS: 74863ee04c8SGilad Ben-Yossef case DRV_CIPHER_ESSIV: 74963ee04c8SGilad Ben-Yossef /* Load AES key */ 75063ee04c8SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 75163ee04c8SGilad Ben-Yossef set_cipher_mode(&desc[*seq_size], cipher_mode); 75263ee04c8SGilad Ben-Yossef set_cipher_config0(&desc[*seq_size], direction); 75352f42c65SGilad Ben-Yossef if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 75463ee04c8SGilad Ben-Yossef set_hw_crypto_key(&desc[*seq_size], 75563ee04c8SGilad Ben-Yossef ctx_p->hw.key1_slot); 75663ee04c8SGilad Ben-Yossef } else { 75763ee04c8SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, 75863ee04c8SGilad Ben-Yossef (key_len / 2), NS_BIT); 75963ee04c8SGilad Ben-Yossef } 76063ee04c8SGilad Ben-Yossef set_key_size_aes(&desc[*seq_size], (key_len / 2)); 76163ee04c8SGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 76263ee04c8SGilad Ben-Yossef set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 76363ee04c8SGilad Ben-Yossef (*seq_size)++; 76463ee04c8SGilad Ben-Yossef break; 76563ee04c8SGilad Ben-Yossef default: 76663ee04c8SGilad Ben-Yossef dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 76763ee04c8SGilad Ben-Yossef } 76863ee04c8SGilad Ben-Yossef } 76963ee04c8SGilad Ben-Yossef 7704b1d7debSGilad Ben-Yossef static void cc_setup_mlli_desc(struct crypto_tfm *tfm, 77163ee04c8SGilad Ben-Yossef struct cipher_req_ctx *req_ctx, 7724b1d7debSGilad Ben-Yossef struct scatterlist *dst, struct scatterlist *src, 7734b1d7debSGilad Ben-Yossef unsigned int nbytes, void *areq, 7744b1d7debSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 7754b1d7debSGilad Ben-Yossef { 7764b1d7debSGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 7774b1d7debSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 7784b1d7debSGilad Ben-Yossef 7794b1d7debSGilad Ben-Yossef if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { 7804b1d7debSGilad Ben-Yossef /* bypass */ 7814b1d7debSGilad Ben-Yossef dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n", 7824b1d7debSGilad Ben-Yossef &req_ctx->mlli_params.mlli_dma_addr, 7834b1d7debSGilad Ben-Yossef req_ctx->mlli_params.mlli_len, 7841a895f1dSGeert Uytterhoeven ctx_p->drvdata->mlli_sram_addr); 7854b1d7debSGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 7864b1d7debSGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, 7874b1d7debSGilad Ben-Yossef req_ctx->mlli_params.mlli_dma_addr, 7884b1d7debSGilad Ben-Yossef req_ctx->mlli_params.mlli_len, NS_BIT); 7894b1d7debSGilad Ben-Yossef set_dout_sram(&desc[*seq_size], 7904b1d7debSGilad Ben-Yossef ctx_p->drvdata->mlli_sram_addr, 7914b1d7debSGilad Ben-Yossef req_ctx->mlli_params.mlli_len); 7924b1d7debSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], BYPASS); 7934b1d7debSGilad Ben-Yossef (*seq_size)++; 7944b1d7debSGilad Ben-Yossef } 7954b1d7debSGilad Ben-Yossef } 7964b1d7debSGilad Ben-Yossef 7974b1d7debSGilad Ben-Yossef static void cc_setup_flow_desc(struct crypto_tfm *tfm, 7984b1d7debSGilad Ben-Yossef struct cipher_req_ctx *req_ctx, 7994b1d7debSGilad Ben-Yossef struct scatterlist *dst, struct scatterlist *src, 8006f17e00fSGilad Ben-Yossef unsigned int nbytes, struct cc_hw_desc desc[], 8016f17e00fSGilad Ben-Yossef unsigned int *seq_size) 80263ee04c8SGilad Ben-Yossef { 80363ee04c8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 80463ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 805533edf9fSGilad Ben-Yossef unsigned int flow_mode = cc_out_flow_mode(ctx_p); 8066f17e00fSGilad Ben-Yossef bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY || 8076f17e00fSGilad Ben-Yossef ctx_p->cipher_mode == DRV_CIPHER_ECB); 80863ee04c8SGilad Ben-Yossef 80963ee04c8SGilad Ben-Yossef /* Process */ 81063ee04c8SGilad Ben-Yossef if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { 81163ee04c8SGilad Ben-Yossef dev_dbg(dev, " data params addr %pad length 0x%X\n", 81263ee04c8SGilad Ben-Yossef &sg_dma_address(src), nbytes); 81363ee04c8SGilad Ben-Yossef dev_dbg(dev, " data params addr %pad length 0x%X\n", 81463ee04c8SGilad Ben-Yossef &sg_dma_address(dst), nbytes); 81563ee04c8SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 81663ee04c8SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), 81763ee04c8SGilad Ben-Yossef nbytes, NS_BIT); 81863ee04c8SGilad Ben-Yossef set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), 8196f17e00fSGilad Ben-Yossef nbytes, NS_BIT, (!last_desc ? 0 : 1)); 8206f17e00fSGilad Ben-Yossef if (last_desc) 82127b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 82263ee04c8SGilad Ben-Yossef 82363ee04c8SGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 82463ee04c8SGilad Ben-Yossef (*seq_size)++; 82563ee04c8SGilad Ben-Yossef } else { 82663ee04c8SGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 82763ee04c8SGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_MLLI, 82863ee04c8SGilad Ben-Yossef ctx_p->drvdata->mlli_sram_addr, 82963ee04c8SGilad Ben-Yossef req_ctx->in_mlli_nents, NS_BIT); 83063ee04c8SGilad Ben-Yossef if (req_ctx->out_nents == 0) { 83163ee04c8SGilad Ben-Yossef dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 8321a895f1dSGeert Uytterhoeven ctx_p->drvdata->mlli_sram_addr, 8331a895f1dSGeert Uytterhoeven ctx_p->drvdata->mlli_sram_addr); 83463ee04c8SGilad Ben-Yossef set_dout_mlli(&desc[*seq_size], 83563ee04c8SGilad Ben-Yossef ctx_p->drvdata->mlli_sram_addr, 83663ee04c8SGilad Ben-Yossef req_ctx->in_mlli_nents, NS_BIT, 8376f17e00fSGilad Ben-Yossef (!last_desc ? 0 : 1)); 83863ee04c8SGilad Ben-Yossef } else { 83963ee04c8SGilad Ben-Yossef dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 8401a895f1dSGeert Uytterhoeven ctx_p->drvdata->mlli_sram_addr, 8411a895f1dSGeert Uytterhoeven ctx_p->drvdata->mlli_sram_addr + 84263ee04c8SGilad Ben-Yossef (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents); 84363ee04c8SGilad Ben-Yossef set_dout_mlli(&desc[*seq_size], 84463ee04c8SGilad Ben-Yossef (ctx_p->drvdata->mlli_sram_addr + 84563ee04c8SGilad Ben-Yossef (LLI_ENTRY_BYTE_SIZE * 84663ee04c8SGilad Ben-Yossef req_ctx->in_mlli_nents)), 84763ee04c8SGilad Ben-Yossef req_ctx->out_mlli_nents, NS_BIT, 8486f17e00fSGilad Ben-Yossef (!last_desc ? 0 : 1)); 84963ee04c8SGilad Ben-Yossef } 8506f17e00fSGilad Ben-Yossef if (last_desc) 85127b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 85263ee04c8SGilad Ben-Yossef 85363ee04c8SGilad Ben-Yossef set_flow_mode(&desc[*seq_size], flow_mode); 85463ee04c8SGilad Ben-Yossef (*seq_size)++; 85563ee04c8SGilad Ben-Yossef } 85663ee04c8SGilad Ben-Yossef } 85763ee04c8SGilad Ben-Yossef 85863ee04c8SGilad Ben-Yossef static void cc_cipher_complete(struct device *dev, void *cc_req, int err) 85963ee04c8SGilad Ben-Yossef { 86063ee04c8SGilad Ben-Yossef struct skcipher_request *req = (struct skcipher_request *)cc_req; 86163ee04c8SGilad Ben-Yossef struct scatterlist *dst = req->dst; 86263ee04c8SGilad Ben-Yossef struct scatterlist *src = req->src; 86363ee04c8SGilad Ben-Yossef struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 86400904aa0SGilad Ben-Yossef struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 86500904aa0SGilad Ben-Yossef unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 86663ee04c8SGilad Ben-Yossef 867a108f931SGilad Ben-Yossef if (err != -EINPROGRESS) { 868a108f931SGilad Ben-Yossef /* Not a BACKLOG notification */ 869c139c72eSGilad Ben-Yossef cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 8706f17e00fSGilad Ben-Yossef memcpy(req->iv, req_ctx->iv, ivsize); 871453431a5SWaiman Long kfree_sensitive(req_ctx->iv); 872a108f931SGilad Ben-Yossef } 873a108f931SGilad Ben-Yossef 87463ee04c8SGilad Ben-Yossef skcipher_request_complete(req, err); 87563ee04c8SGilad Ben-Yossef } 87663ee04c8SGilad Ben-Yossef 87763ee04c8SGilad Ben-Yossef static int cc_cipher_process(struct skcipher_request *req, 87863ee04c8SGilad Ben-Yossef enum drv_crypto_direction direction) 87963ee04c8SGilad Ben-Yossef { 88063ee04c8SGilad Ben-Yossef struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 88163ee04c8SGilad Ben-Yossef struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); 88263ee04c8SGilad Ben-Yossef struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 88363ee04c8SGilad Ben-Yossef unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 88463ee04c8SGilad Ben-Yossef struct scatterlist *dst = req->dst; 88563ee04c8SGilad Ben-Yossef struct scatterlist *src = req->src; 88663ee04c8SGilad Ben-Yossef unsigned int nbytes = req->cryptlen; 88763ee04c8SGilad Ben-Yossef void *iv = req->iv; 88863ee04c8SGilad Ben-Yossef struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 88963ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx_p->drvdata); 8905620eb6cSArd Biesheuvel struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN]; 89163ee04c8SGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 89284f366daSGilad Ben-Yossef int rc; 89363ee04c8SGilad Ben-Yossef unsigned int seq_len = 0; 89463ee04c8SGilad Ben-Yossef gfp_t flags = cc_gfp_flags(&req->base); 89563ee04c8SGilad Ben-Yossef 89663ee04c8SGilad Ben-Yossef dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n", 89763ee04c8SGilad Ben-Yossef ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 89863ee04c8SGilad Ben-Yossef "Encrypt" : "Decrypt"), req, iv, nbytes); 89963ee04c8SGilad Ben-Yossef 90063ee04c8SGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 90163ee04c8SGilad Ben-Yossef 90263ee04c8SGilad Ben-Yossef if (validate_data_size(ctx_p, nbytes)) { 903c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Unsupported data size %d.\n", nbytes); 90463ee04c8SGilad Ben-Yossef rc = -EINVAL; 90563ee04c8SGilad Ben-Yossef goto exit_process; 90663ee04c8SGilad Ben-Yossef } 90763ee04c8SGilad Ben-Yossef if (nbytes == 0) { 90863ee04c8SGilad Ben-Yossef /* No data to process is valid */ 90963ee04c8SGilad Ben-Yossef rc = 0; 91063ee04c8SGilad Ben-Yossef goto exit_process; 91163ee04c8SGilad Ben-Yossef } 91263ee04c8SGilad Ben-Yossef 913520f325aSGilad Ben-Yossef if (ctx_p->fallback_on) { 914520f325aSGilad Ben-Yossef struct skcipher_request *subreq = skcipher_request_ctx(req); 915520f325aSGilad Ben-Yossef 916520f325aSGilad Ben-Yossef *subreq = *req; 917520f325aSGilad Ben-Yossef skcipher_request_set_tfm(subreq, ctx_p->fallback_tfm); 918520f325aSGilad Ben-Yossef if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT) 919520f325aSGilad Ben-Yossef return crypto_skcipher_encrypt(subreq); 920520f325aSGilad Ben-Yossef else 921520f325aSGilad Ben-Yossef return crypto_skcipher_decrypt(subreq); 922520f325aSGilad Ben-Yossef } 923520f325aSGilad Ben-Yossef 924b2cd1d81Sdingsenjie /* The IV we are handed may be allocated from the stack so 92563ee04c8SGilad Ben-Yossef * we must copy it to a DMAable buffer before use. 92663ee04c8SGilad Ben-Yossef */ 92701745706SFengguang Wu req_ctx->iv = kmemdup(iv, ivsize, flags); 92863ee04c8SGilad Ben-Yossef if (!req_ctx->iv) { 92963ee04c8SGilad Ben-Yossef rc = -ENOMEM; 93063ee04c8SGilad Ben-Yossef goto exit_process; 93163ee04c8SGilad Ben-Yossef } 93263ee04c8SGilad Ben-Yossef 93363ee04c8SGilad Ben-Yossef /* Setup request structure */ 934f4274eecSGeert Uytterhoeven cc_req.user_cb = cc_cipher_complete; 935f4274eecSGeert Uytterhoeven cc_req.user_arg = req; 93663ee04c8SGilad Ben-Yossef 93752f42c65SGilad Ben-Yossef /* Setup CPP operation details */ 93852f42c65SGilad Ben-Yossef if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) { 93952f42c65SGilad Ben-Yossef cc_req.cpp.is_cpp = true; 94052f42c65SGilad Ben-Yossef cc_req.cpp.alg = ctx_p->cpp.alg; 94152f42c65SGilad Ben-Yossef cc_req.cpp.slot = ctx_p->cpp.slot; 94252f42c65SGilad Ben-Yossef } 94352f42c65SGilad Ben-Yossef 94463ee04c8SGilad Ben-Yossef /* Setup request context */ 94563ee04c8SGilad Ben-Yossef req_ctx->gen_ctx.op_type = direction; 94663ee04c8SGilad Ben-Yossef 94763ee04c8SGilad Ben-Yossef /* STAT_PHASE_1: Map buffers */ 94863ee04c8SGilad Ben-Yossef 94963ee04c8SGilad Ben-Yossef rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes, 95063ee04c8SGilad Ben-Yossef req_ctx->iv, src, dst, flags); 95163ee04c8SGilad Ben-Yossef if (rc) { 95263ee04c8SGilad Ben-Yossef dev_err(dev, "map_request() failed\n"); 95363ee04c8SGilad Ben-Yossef goto exit_process; 95463ee04c8SGilad Ben-Yossef } 95563ee04c8SGilad Ben-Yossef 95663ee04c8SGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */ 95763ee04c8SGilad Ben-Yossef 9585c83e8ecSOfir Drang /* Setup state (IV) */ 959dd8486c7SGilad Ben-Yossef cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 9604b1d7debSGilad Ben-Yossef /* Setup MLLI line, if needed */ 9614b1d7debSGilad Ben-Yossef cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len); 962dd8486c7SGilad Ben-Yossef /* Setup key */ 963dd8486c7SGilad Ben-Yossef cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len); 9645c83e8ecSOfir Drang /* Setup state (IV and XEX key) */ 9655c83e8ecSOfir Drang cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 96663ee04c8SGilad Ben-Yossef /* Data processing */ 9676f17e00fSGilad Ben-Yossef cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len); 9686f17e00fSGilad Ben-Yossef /* Read next IV */ 9696f17e00fSGilad Ben-Yossef cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len); 97063ee04c8SGilad Ben-Yossef 97163ee04c8SGilad Ben-Yossef /* STAT_PHASE_3: Lock HW and push sequence */ 97263ee04c8SGilad Ben-Yossef 97363ee04c8SGilad Ben-Yossef rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len, 97463ee04c8SGilad Ben-Yossef &req->base); 97563ee04c8SGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) { 97663ee04c8SGilad Ben-Yossef /* Failed to send the request or request completed 97763ee04c8SGilad Ben-Yossef * synchronously 97863ee04c8SGilad Ben-Yossef */ 97963ee04c8SGilad Ben-Yossef cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 98063ee04c8SGilad Ben-Yossef } 98163ee04c8SGilad Ben-Yossef 98263ee04c8SGilad Ben-Yossef exit_process: 98363ee04c8SGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) { 984453431a5SWaiman Long kfree_sensitive(req_ctx->iv); 98563ee04c8SGilad Ben-Yossef } 98663ee04c8SGilad Ben-Yossef 98763ee04c8SGilad Ben-Yossef return rc; 98863ee04c8SGilad Ben-Yossef } 98963ee04c8SGilad Ben-Yossef 99063ee04c8SGilad Ben-Yossef static int cc_cipher_encrypt(struct skcipher_request *req) 99163ee04c8SGilad Ben-Yossef { 99263ee04c8SGilad Ben-Yossef struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 99363ee04c8SGilad Ben-Yossef 994e30368f3SGilad Ben-Yossef memset(req_ctx, 0, sizeof(*req_ctx)); 99563ee04c8SGilad Ben-Yossef 99663ee04c8SGilad Ben-Yossef return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 99763ee04c8SGilad Ben-Yossef } 99863ee04c8SGilad Ben-Yossef 99963ee04c8SGilad Ben-Yossef static int cc_cipher_decrypt(struct skcipher_request *req) 100063ee04c8SGilad Ben-Yossef { 100163ee04c8SGilad Ben-Yossef struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 100263ee04c8SGilad Ben-Yossef 1003e30368f3SGilad Ben-Yossef memset(req_ctx, 0, sizeof(*req_ctx)); 1004e30368f3SGilad Ben-Yossef 100563ee04c8SGilad Ben-Yossef return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT); 100663ee04c8SGilad Ben-Yossef } 100763ee04c8SGilad Ben-Yossef 100863ee04c8SGilad Ben-Yossef /* Block cipher alg */ 100963ee04c8SGilad Ben-Yossef static const struct cc_alg_template skcipher_algs[] = { 101063ee04c8SGilad Ben-Yossef { 1011a794d8d8SGilad Ben-Yossef .name = "xts(paes)", 1012a794d8d8SGilad Ben-Yossef .driver_name = "xts-paes-ccree", 101367caef08SUri Shir .blocksize = 1, 1014a794d8d8SGilad Ben-Yossef .template_skcipher = { 1015a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1016a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1017a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1018a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1019a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1020a794d8d8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 1021a794d8d8SGilad Ben-Yossef }, 1022a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_XTS, 1023a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1024a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 10251c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1026f98f6e21SGilad Ben-Yossef .sec_func = true, 1027a794d8d8SGilad Ben-Yossef }, 1028a794d8d8SGilad Ben-Yossef { 1029520f325aSGilad Ben-Yossef .name = "essiv(cbc(paes),sha256)", 1030a794d8d8SGilad Ben-Yossef .driver_name = "essiv-paes-ccree", 1031a794d8d8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 1032a794d8d8SGilad Ben-Yossef .template_skcipher = { 1033a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1034a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1035a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1036a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1037a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1038a794d8d8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 1039a794d8d8SGilad Ben-Yossef }, 1040a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ESSIV, 1041a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1042a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 10431c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1044f98f6e21SGilad Ben-Yossef .sec_func = true, 1045a794d8d8SGilad Ben-Yossef }, 1046a794d8d8SGilad Ben-Yossef { 1047a794d8d8SGilad Ben-Yossef .name = "ecb(paes)", 1048a794d8d8SGilad Ben-Yossef .driver_name = "ecb-paes-ccree", 1049a794d8d8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 1050a794d8d8SGilad Ben-Yossef .template_skcipher = { 1051a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1052a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1053a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1054a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1055a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1056a794d8d8SGilad Ben-Yossef .ivsize = 0, 1057a794d8d8SGilad Ben-Yossef }, 1058a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ECB, 1059a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1060a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 10611c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1062f98f6e21SGilad Ben-Yossef .sec_func = true, 1063a794d8d8SGilad Ben-Yossef }, 1064a794d8d8SGilad Ben-Yossef { 1065a794d8d8SGilad Ben-Yossef .name = "cbc(paes)", 1066a794d8d8SGilad Ben-Yossef .driver_name = "cbc-paes-ccree", 1067a794d8d8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 1068a794d8d8SGilad Ben-Yossef .template_skcipher = { 1069a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1070a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1071a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1072a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1073a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1074a794d8d8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 1075a794d8d8SGilad Ben-Yossef }, 1076a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 1077a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1078a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 10791c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1080f98f6e21SGilad Ben-Yossef .sec_func = true, 1081a794d8d8SGilad Ben-Yossef }, 1082a794d8d8SGilad Ben-Yossef { 1083a794d8d8SGilad Ben-Yossef .name = "ofb(paes)", 1084a794d8d8SGilad Ben-Yossef .driver_name = "ofb-paes-ccree", 1085a794d8d8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 1086a794d8d8SGilad Ben-Yossef .template_skcipher = { 1087a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1088a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1089a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1090a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1091a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1092a794d8d8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 1093a794d8d8SGilad Ben-Yossef }, 1094a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_OFB, 1095a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1096a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 10971c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1098f98f6e21SGilad Ben-Yossef .sec_func = true, 1099a794d8d8SGilad Ben-Yossef }, 1100a794d8d8SGilad Ben-Yossef { 110184f366daSGilad Ben-Yossef .name = "cts(cbc(paes))", 110284f366daSGilad Ben-Yossef .driver_name = "cts-cbc-paes-ccree", 1103a794d8d8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 1104a794d8d8SGilad Ben-Yossef .template_skcipher = { 1105a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1106a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1107a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1108a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1109a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1110a794d8d8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 1111a794d8d8SGilad Ben-Yossef }, 1112a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC_CTS, 1113a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1114a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 11151c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1116f98f6e21SGilad Ben-Yossef .sec_func = true, 1117a794d8d8SGilad Ben-Yossef }, 1118a794d8d8SGilad Ben-Yossef { 1119a794d8d8SGilad Ben-Yossef .name = "ctr(paes)", 1120a794d8d8SGilad Ben-Yossef .driver_name = "ctr-paes-ccree", 1121a794d8d8SGilad Ben-Yossef .blocksize = 1, 1122a794d8d8SGilad Ben-Yossef .template_skcipher = { 1123a794d8d8SGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1124a794d8d8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1125a794d8d8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1126a794d8d8SGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1127a794d8d8SGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1128a794d8d8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 1129a794d8d8SGilad Ben-Yossef }, 1130a794d8d8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 1131a794d8d8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 1132a794d8d8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 11331c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 1134f98f6e21SGilad Ben-Yossef .sec_func = true, 1135a794d8d8SGilad Ben-Yossef }, 1136a794d8d8SGilad Ben-Yossef { 11374aaefb62SGilad Ben-Yossef /* See https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg40576.html 11384aaefb62SGilad Ben-Yossef * for the reason why this differs from the generic 11394aaefb62SGilad Ben-Yossef * implementation. 11404aaefb62SGilad Ben-Yossef */ 114163ee04c8SGilad Ben-Yossef .name = "xts(aes)", 114263ee04c8SGilad Ben-Yossef .driver_name = "xts-aes-ccree", 114367caef08SUri Shir .blocksize = 1, 114463ee04c8SGilad Ben-Yossef .template_skcipher = { 114563ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 114663ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 114763ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 114863ee04c8SGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE * 2, 114963ee04c8SGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE * 2, 115063ee04c8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 115163ee04c8SGilad Ben-Yossef }, 115263ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_XTS, 115363ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 115427b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 11551c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 115663ee04c8SGilad Ben-Yossef }, 115763ee04c8SGilad Ben-Yossef { 1158520f325aSGilad Ben-Yossef .name = "essiv(cbc(aes),sha256)", 115963ee04c8SGilad Ben-Yossef .driver_name = "essiv-aes-ccree", 116063ee04c8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 116163ee04c8SGilad Ben-Yossef .template_skcipher = { 116263ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 116363ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 116463ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1165520f325aSGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE, 1166520f325aSGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE, 116763ee04c8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 116863ee04c8SGilad Ben-Yossef }, 116963ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ESSIV, 117063ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 117127b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_712, 11721c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 117363ee04c8SGilad Ben-Yossef }, 117463ee04c8SGilad Ben-Yossef { 117563ee04c8SGilad Ben-Yossef .name = "ecb(aes)", 117663ee04c8SGilad Ben-Yossef .driver_name = "ecb-aes-ccree", 117763ee04c8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 117863ee04c8SGilad Ben-Yossef .template_skcipher = { 117963ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 118063ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 118163ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 118263ee04c8SGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE, 118363ee04c8SGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE, 118463ee04c8SGilad Ben-Yossef .ivsize = 0, 118563ee04c8SGilad Ben-Yossef }, 118663ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ECB, 118763ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 118827b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 11891c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 119063ee04c8SGilad Ben-Yossef }, 119163ee04c8SGilad Ben-Yossef { 119263ee04c8SGilad Ben-Yossef .name = "cbc(aes)", 119363ee04c8SGilad Ben-Yossef .driver_name = "cbc-aes-ccree", 119463ee04c8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 119563ee04c8SGilad Ben-Yossef .template_skcipher = { 119663ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 119763ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 119863ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 119963ee04c8SGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE, 120063ee04c8SGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE, 120163ee04c8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 120263ee04c8SGilad Ben-Yossef }, 120363ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 120463ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 120527b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 12061c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 120763ee04c8SGilad Ben-Yossef }, 120863ee04c8SGilad Ben-Yossef { 120963ee04c8SGilad Ben-Yossef .name = "ofb(aes)", 121063ee04c8SGilad Ben-Yossef .driver_name = "ofb-aes-ccree", 12114aaefb62SGilad Ben-Yossef .blocksize = 1, 121263ee04c8SGilad Ben-Yossef .template_skcipher = { 121363ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 121463ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 121563ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 121663ee04c8SGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE, 121763ee04c8SGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE, 121863ee04c8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 121963ee04c8SGilad Ben-Yossef }, 122063ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_OFB, 122163ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 122227b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 12231c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 122463ee04c8SGilad Ben-Yossef }, 122563ee04c8SGilad Ben-Yossef { 122684f366daSGilad Ben-Yossef .name = "cts(cbc(aes))", 122784f366daSGilad Ben-Yossef .driver_name = "cts-cbc-aes-ccree", 122863ee04c8SGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 122963ee04c8SGilad Ben-Yossef .template_skcipher = { 123063ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 123163ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 123263ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 123363ee04c8SGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE, 123463ee04c8SGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE, 123563ee04c8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 123663ee04c8SGilad Ben-Yossef }, 123763ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC_CTS, 123863ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 123927b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 12401c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 124163ee04c8SGilad Ben-Yossef }, 124263ee04c8SGilad Ben-Yossef { 124363ee04c8SGilad Ben-Yossef .name = "ctr(aes)", 124463ee04c8SGilad Ben-Yossef .driver_name = "ctr-aes-ccree", 124563ee04c8SGilad Ben-Yossef .blocksize = 1, 124663ee04c8SGilad Ben-Yossef .template_skcipher = { 124763ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 124863ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 124963ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 125063ee04c8SGilad Ben-Yossef .min_keysize = AES_MIN_KEY_SIZE, 125163ee04c8SGilad Ben-Yossef .max_keysize = AES_MAX_KEY_SIZE, 125263ee04c8SGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 125363ee04c8SGilad Ben-Yossef }, 125463ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 125563ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 125627b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 12571c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 125863ee04c8SGilad Ben-Yossef }, 125963ee04c8SGilad Ben-Yossef { 126063ee04c8SGilad Ben-Yossef .name = "cbc(des3_ede)", 126163ee04c8SGilad Ben-Yossef .driver_name = "cbc-3des-ccree", 126263ee04c8SGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE, 126363ee04c8SGilad Ben-Yossef .template_skcipher = { 126463ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 126563ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 126663ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 126763ee04c8SGilad Ben-Yossef .min_keysize = DES3_EDE_KEY_SIZE, 126863ee04c8SGilad Ben-Yossef .max_keysize = DES3_EDE_KEY_SIZE, 126963ee04c8SGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE, 127063ee04c8SGilad Ben-Yossef }, 127163ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 127263ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 127327b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 12741c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 127563ee04c8SGilad Ben-Yossef }, 127663ee04c8SGilad Ben-Yossef { 127763ee04c8SGilad Ben-Yossef .name = "ecb(des3_ede)", 127863ee04c8SGilad Ben-Yossef .driver_name = "ecb-3des-ccree", 127963ee04c8SGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE, 128063ee04c8SGilad Ben-Yossef .template_skcipher = { 128163ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 128263ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 128363ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 128463ee04c8SGilad Ben-Yossef .min_keysize = DES3_EDE_KEY_SIZE, 128563ee04c8SGilad Ben-Yossef .max_keysize = DES3_EDE_KEY_SIZE, 128663ee04c8SGilad Ben-Yossef .ivsize = 0, 128763ee04c8SGilad Ben-Yossef }, 128863ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ECB, 128963ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 129027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 12911c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 129263ee04c8SGilad Ben-Yossef }, 129363ee04c8SGilad Ben-Yossef { 129463ee04c8SGilad Ben-Yossef .name = "cbc(des)", 129563ee04c8SGilad Ben-Yossef .driver_name = "cbc-des-ccree", 129663ee04c8SGilad Ben-Yossef .blocksize = DES_BLOCK_SIZE, 129763ee04c8SGilad Ben-Yossef .template_skcipher = { 129863ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 129963ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 130063ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 130163ee04c8SGilad Ben-Yossef .min_keysize = DES_KEY_SIZE, 130263ee04c8SGilad Ben-Yossef .max_keysize = DES_KEY_SIZE, 130363ee04c8SGilad Ben-Yossef .ivsize = DES_BLOCK_SIZE, 130463ee04c8SGilad Ben-Yossef }, 130563ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 130663ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 130727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 13081c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 130963ee04c8SGilad Ben-Yossef }, 131063ee04c8SGilad Ben-Yossef { 131163ee04c8SGilad Ben-Yossef .name = "ecb(des)", 131263ee04c8SGilad Ben-Yossef .driver_name = "ecb-des-ccree", 131363ee04c8SGilad Ben-Yossef .blocksize = DES_BLOCK_SIZE, 131463ee04c8SGilad Ben-Yossef .template_skcipher = { 131563ee04c8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 131663ee04c8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 131763ee04c8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 131863ee04c8SGilad Ben-Yossef .min_keysize = DES_KEY_SIZE, 131963ee04c8SGilad Ben-Yossef .max_keysize = DES_KEY_SIZE, 132063ee04c8SGilad Ben-Yossef .ivsize = 0, 132163ee04c8SGilad Ben-Yossef }, 132263ee04c8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ECB, 132363ee04c8SGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 132427b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 13251c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 132663ee04c8SGilad Ben-Yossef }, 13279b8d51f8SGilad Ben-Yossef { 13289b8d51f8SGilad Ben-Yossef .name = "cbc(sm4)", 13299b8d51f8SGilad Ben-Yossef .driver_name = "cbc-sm4-ccree", 13309b8d51f8SGilad Ben-Yossef .blocksize = SM4_BLOCK_SIZE, 13319b8d51f8SGilad Ben-Yossef .template_skcipher = { 13329b8d51f8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 13339b8d51f8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 13349b8d51f8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 13359b8d51f8SGilad Ben-Yossef .min_keysize = SM4_KEY_SIZE, 13369b8d51f8SGilad Ben-Yossef .max_keysize = SM4_KEY_SIZE, 13379b8d51f8SGilad Ben-Yossef .ivsize = SM4_BLOCK_SIZE, 13389b8d51f8SGilad Ben-Yossef }, 13399b8d51f8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 13409b8d51f8SGilad Ben-Yossef .flow_mode = S_DIN_to_SM4, 13419b8d51f8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_713, 13421c876a90SGilad Ben-Yossef .std_body = CC_STD_OSCCA, 13439b8d51f8SGilad Ben-Yossef }, 13449b8d51f8SGilad Ben-Yossef { 13459b8d51f8SGilad Ben-Yossef .name = "ecb(sm4)", 13469b8d51f8SGilad Ben-Yossef .driver_name = "ecb-sm4-ccree", 13479b8d51f8SGilad Ben-Yossef .blocksize = SM4_BLOCK_SIZE, 13489b8d51f8SGilad Ben-Yossef .template_skcipher = { 13499b8d51f8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 13509b8d51f8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 13519b8d51f8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 13529b8d51f8SGilad Ben-Yossef .min_keysize = SM4_KEY_SIZE, 13539b8d51f8SGilad Ben-Yossef .max_keysize = SM4_KEY_SIZE, 13549b8d51f8SGilad Ben-Yossef .ivsize = 0, 13559b8d51f8SGilad Ben-Yossef }, 13569b8d51f8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_ECB, 13579b8d51f8SGilad Ben-Yossef .flow_mode = S_DIN_to_SM4, 13589b8d51f8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_713, 13591c876a90SGilad Ben-Yossef .std_body = CC_STD_OSCCA, 13609b8d51f8SGilad Ben-Yossef }, 13619b8d51f8SGilad Ben-Yossef { 13629b8d51f8SGilad Ben-Yossef .name = "ctr(sm4)", 13639b8d51f8SGilad Ben-Yossef .driver_name = "ctr-sm4-ccree", 13644aaefb62SGilad Ben-Yossef .blocksize = 1, 13659b8d51f8SGilad Ben-Yossef .template_skcipher = { 13669b8d51f8SGilad Ben-Yossef .setkey = cc_cipher_setkey, 13679b8d51f8SGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 13689b8d51f8SGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 13699b8d51f8SGilad Ben-Yossef .min_keysize = SM4_KEY_SIZE, 13709b8d51f8SGilad Ben-Yossef .max_keysize = SM4_KEY_SIZE, 13719b8d51f8SGilad Ben-Yossef .ivsize = SM4_BLOCK_SIZE, 13729b8d51f8SGilad Ben-Yossef }, 13739b8d51f8SGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 13749b8d51f8SGilad Ben-Yossef .flow_mode = S_DIN_to_SM4, 13759b8d51f8SGilad Ben-Yossef .min_hw_rev = CC_HW_REV_713, 13761c876a90SGilad Ben-Yossef .std_body = CC_STD_OSCCA, 13779b8d51f8SGilad Ben-Yossef }, 1378bee711faSGilad Ben-Yossef { 1379bee711faSGilad Ben-Yossef .name = "cbc(psm4)", 1380bee711faSGilad Ben-Yossef .driver_name = "cbc-psm4-ccree", 1381bee711faSGilad Ben-Yossef .blocksize = SM4_BLOCK_SIZE, 1382bee711faSGilad Ben-Yossef .template_skcipher = { 1383bee711faSGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1384bee711faSGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1385bee711faSGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1386bee711faSGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1387bee711faSGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1388bee711faSGilad Ben-Yossef .ivsize = SM4_BLOCK_SIZE, 1389bee711faSGilad Ben-Yossef }, 1390bee711faSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 1391bee711faSGilad Ben-Yossef .flow_mode = S_DIN_to_SM4, 1392bee711faSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_713, 1393bee711faSGilad Ben-Yossef .std_body = CC_STD_OSCCA, 1394bee711faSGilad Ben-Yossef .sec_func = true, 1395bee711faSGilad Ben-Yossef }, 1396bee711faSGilad Ben-Yossef { 1397bee711faSGilad Ben-Yossef .name = "ctr(psm4)", 1398bee711faSGilad Ben-Yossef .driver_name = "ctr-psm4-ccree", 1399bee711faSGilad Ben-Yossef .blocksize = SM4_BLOCK_SIZE, 1400bee711faSGilad Ben-Yossef .template_skcipher = { 1401bee711faSGilad Ben-Yossef .setkey = cc_cipher_sethkey, 1402bee711faSGilad Ben-Yossef .encrypt = cc_cipher_encrypt, 1403bee711faSGilad Ben-Yossef .decrypt = cc_cipher_decrypt, 1404bee711faSGilad Ben-Yossef .min_keysize = CC_HW_KEY_SIZE, 1405bee711faSGilad Ben-Yossef .max_keysize = CC_HW_KEY_SIZE, 1406bee711faSGilad Ben-Yossef .ivsize = SM4_BLOCK_SIZE, 1407bee711faSGilad Ben-Yossef }, 1408bee711faSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 1409bee711faSGilad Ben-Yossef .flow_mode = S_DIN_to_SM4, 1410bee711faSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_713, 1411bee711faSGilad Ben-Yossef .std_body = CC_STD_OSCCA, 1412bee711faSGilad Ben-Yossef .sec_func = true, 1413bee711faSGilad Ben-Yossef }, 141463ee04c8SGilad Ben-Yossef }; 141563ee04c8SGilad Ben-Yossef 141663ee04c8SGilad Ben-Yossef static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, 141763ee04c8SGilad Ben-Yossef struct device *dev) 141863ee04c8SGilad Ben-Yossef { 141963ee04c8SGilad Ben-Yossef struct cc_crypto_alg *t_alg; 142063ee04c8SGilad Ben-Yossef struct skcipher_alg *alg; 142163ee04c8SGilad Ben-Yossef 1422f7c8f992SGeert Uytterhoeven t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); 142363ee04c8SGilad Ben-Yossef if (!t_alg) 142463ee04c8SGilad Ben-Yossef return ERR_PTR(-ENOMEM); 142563ee04c8SGilad Ben-Yossef 142663ee04c8SGilad Ben-Yossef alg = &t_alg->skcipher_alg; 142763ee04c8SGilad Ben-Yossef 142863ee04c8SGilad Ben-Yossef memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); 142963ee04c8SGilad Ben-Yossef 143063ee04c8SGilad Ben-Yossef snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 143163ee04c8SGilad Ben-Yossef snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 143263ee04c8SGilad Ben-Yossef tmpl->driver_name); 143363ee04c8SGilad Ben-Yossef alg->base.cra_module = THIS_MODULE; 143463ee04c8SGilad Ben-Yossef alg->base.cra_priority = CC_CRA_PRIO; 143563ee04c8SGilad Ben-Yossef alg->base.cra_blocksize = tmpl->blocksize; 143663ee04c8SGilad Ben-Yossef alg->base.cra_alignmask = 0; 143763ee04c8SGilad Ben-Yossef alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx); 143863ee04c8SGilad Ben-Yossef 143963ee04c8SGilad Ben-Yossef alg->base.cra_init = cc_cipher_init; 144063ee04c8SGilad Ben-Yossef alg->base.cra_exit = cc_cipher_exit; 14412c95e6d9SEric Biggers alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 144263ee04c8SGilad Ben-Yossef 144363ee04c8SGilad Ben-Yossef t_alg->cipher_mode = tmpl->cipher_mode; 144463ee04c8SGilad Ben-Yossef t_alg->flow_mode = tmpl->flow_mode; 144563ee04c8SGilad Ben-Yossef 144663ee04c8SGilad Ben-Yossef return t_alg; 144763ee04c8SGilad Ben-Yossef } 144863ee04c8SGilad Ben-Yossef 144963ee04c8SGilad Ben-Yossef int cc_cipher_free(struct cc_drvdata *drvdata) 145063ee04c8SGilad Ben-Yossef { 145163ee04c8SGilad Ben-Yossef struct cc_crypto_alg *t_alg, *n; 145263ee04c8SGilad Ben-Yossef 145363ee04c8SGilad Ben-Yossef /* Remove registered algs */ 1454c23d7997SGeert Uytterhoeven list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) { 145563ee04c8SGilad Ben-Yossef crypto_unregister_skcipher(&t_alg->skcipher_alg); 145663ee04c8SGilad Ben-Yossef list_del(&t_alg->entry); 145763ee04c8SGilad Ben-Yossef } 145863ee04c8SGilad Ben-Yossef return 0; 145963ee04c8SGilad Ben-Yossef } 146063ee04c8SGilad Ben-Yossef 146163ee04c8SGilad Ben-Yossef int cc_cipher_alloc(struct cc_drvdata *drvdata) 146263ee04c8SGilad Ben-Yossef { 146363ee04c8SGilad Ben-Yossef struct cc_crypto_alg *t_alg; 146463ee04c8SGilad Ben-Yossef struct device *dev = drvdata_to_dev(drvdata); 146563ee04c8SGilad Ben-Yossef int rc = -ENOMEM; 146663ee04c8SGilad Ben-Yossef int alg; 146763ee04c8SGilad Ben-Yossef 1468c23d7997SGeert Uytterhoeven INIT_LIST_HEAD(&drvdata->alg_list); 146963ee04c8SGilad Ben-Yossef 147063ee04c8SGilad Ben-Yossef /* Linux crypto */ 147163ee04c8SGilad Ben-Yossef dev_dbg(dev, "Number of algorithms = %zu\n", 147263ee04c8SGilad Ben-Yossef ARRAY_SIZE(skcipher_algs)); 147363ee04c8SGilad Ben-Yossef for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { 14741c876a90SGilad Ben-Yossef if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) || 1475f98f6e21SGilad Ben-Yossef !(drvdata->std_bodies & skcipher_algs[alg].std_body) || 1476f98f6e21SGilad Ben-Yossef (drvdata->sec_disabled && skcipher_algs[alg].sec_func)) 147727b3b22dSGilad Ben-Yossef continue; 147827b3b22dSGilad Ben-Yossef 147963ee04c8SGilad Ben-Yossef dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name); 148063ee04c8SGilad Ben-Yossef t_alg = cc_create_alg(&skcipher_algs[alg], dev); 148163ee04c8SGilad Ben-Yossef if (IS_ERR(t_alg)) { 148263ee04c8SGilad Ben-Yossef rc = PTR_ERR(t_alg); 148363ee04c8SGilad Ben-Yossef dev_err(dev, "%s alg allocation failed\n", 148463ee04c8SGilad Ben-Yossef skcipher_algs[alg].driver_name); 148563ee04c8SGilad Ben-Yossef goto fail0; 148663ee04c8SGilad Ben-Yossef } 148763ee04c8SGilad Ben-Yossef t_alg->drvdata = drvdata; 148863ee04c8SGilad Ben-Yossef 148963ee04c8SGilad Ben-Yossef dev_dbg(dev, "registering %s\n", 149063ee04c8SGilad Ben-Yossef skcipher_algs[alg].driver_name); 149163ee04c8SGilad Ben-Yossef rc = crypto_register_skcipher(&t_alg->skcipher_alg); 149263ee04c8SGilad Ben-Yossef dev_dbg(dev, "%s alg registration rc = %x\n", 149363ee04c8SGilad Ben-Yossef t_alg->skcipher_alg.base.cra_driver_name, rc); 149463ee04c8SGilad Ben-Yossef if (rc) { 149563ee04c8SGilad Ben-Yossef dev_err(dev, "%s alg registration failed\n", 149663ee04c8SGilad Ben-Yossef t_alg->skcipher_alg.base.cra_driver_name); 149763ee04c8SGilad Ben-Yossef goto fail0; 1498ff4d719aSGeert Uytterhoeven } 1499ff4d719aSGeert Uytterhoeven 1500c23d7997SGeert Uytterhoeven list_add_tail(&t_alg->entry, &drvdata->alg_list); 150163ee04c8SGilad Ben-Yossef dev_dbg(dev, "Registered %s\n", 150263ee04c8SGilad Ben-Yossef t_alg->skcipher_alg.base.cra_driver_name); 150363ee04c8SGilad Ben-Yossef } 150463ee04c8SGilad Ben-Yossef return 0; 150563ee04c8SGilad Ben-Yossef 150663ee04c8SGilad Ben-Yossef fail0: 150763ee04c8SGilad Ben-Yossef cc_cipher_free(drvdata); 150863ee04c8SGilad Ben-Yossef return rc; 150963ee04c8SGilad Ben-Yossef } 1510