1ff27e85aSGilad Ben-Yossef // SPDX-License-Identifier: GPL-2.0 203963caeSGilad Ben-Yossef /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3ff27e85aSGilad Ben-Yossef 4ff27e85aSGilad Ben-Yossef #include <linux/kernel.h> 5ff27e85aSGilad Ben-Yossef #include <linux/module.h> 6ff27e85aSGilad Ben-Yossef #include <crypto/algapi.h> 7ff27e85aSGilad Ben-Yossef #include <crypto/internal/aead.h> 8ff27e85aSGilad Ben-Yossef #include <crypto/authenc.h> 900cd6b23SArd Biesheuvel #include <crypto/internal/des.h> 10ff27e85aSGilad Ben-Yossef #include <linux/rtnetlink.h> 11ff27e85aSGilad Ben-Yossef #include "cc_driver.h" 12ff27e85aSGilad Ben-Yossef #include "cc_buffer_mgr.h" 13ff27e85aSGilad Ben-Yossef #include "cc_aead.h" 14ff27e85aSGilad Ben-Yossef #include "cc_request_mgr.h" 15ff27e85aSGilad Ben-Yossef #include "cc_hash.h" 16ff27e85aSGilad Ben-Yossef #include "cc_sram_mgr.h" 17ff27e85aSGilad Ben-Yossef 18ff27e85aSGilad Ben-Yossef #define template_aead template_u.aead 19ff27e85aSGilad Ben-Yossef 20ff27e85aSGilad Ben-Yossef #define MAX_AEAD_SETKEY_SEQ 12 21ff27e85aSGilad Ben-Yossef #define MAX_AEAD_PROCESS_SEQ 23 22ff27e85aSGilad Ben-Yossef 23ff27e85aSGilad Ben-Yossef #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 24ff27e85aSGilad Ben-Yossef #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 25ff27e85aSGilad Ben-Yossef 26ff27e85aSGilad Ben-Yossef #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 27ff27e85aSGilad Ben-Yossef 28ff27e85aSGilad Ben-Yossef struct cc_aead_handle { 29ff27e85aSGilad Ben-Yossef cc_sram_addr_t sram_workspace_addr; 30ff27e85aSGilad Ben-Yossef struct list_head aead_list; 31ff27e85aSGilad Ben-Yossef }; 32ff27e85aSGilad Ben-Yossef 33ff27e85aSGilad Ben-Yossef struct cc_hmac_s { 34ff27e85aSGilad Ben-Yossef u8 *padded_authkey; 35ff27e85aSGilad Ben-Yossef u8 *ipad_opad; /* IPAD, OPAD*/ 36ff27e85aSGilad Ben-Yossef dma_addr_t padded_authkey_dma_addr; 37ff27e85aSGilad Ben-Yossef dma_addr_t ipad_opad_dma_addr; 38ff27e85aSGilad Ben-Yossef }; 39ff27e85aSGilad Ben-Yossef 40ff27e85aSGilad Ben-Yossef struct cc_xcbc_s { 41ff27e85aSGilad Ben-Yossef u8 *xcbc_keys; /* K1,K2,K3 */ 42ff27e85aSGilad Ben-Yossef dma_addr_t xcbc_keys_dma_addr; 43ff27e85aSGilad Ben-Yossef }; 44ff27e85aSGilad Ben-Yossef 45ff27e85aSGilad Ben-Yossef struct cc_aead_ctx { 46ff27e85aSGilad Ben-Yossef struct cc_drvdata *drvdata; 47ff27e85aSGilad Ben-Yossef u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 48ff27e85aSGilad Ben-Yossef u8 *enckey; 49ff27e85aSGilad Ben-Yossef dma_addr_t enckey_dma_addr; 50ff27e85aSGilad Ben-Yossef union { 51ff27e85aSGilad Ben-Yossef struct cc_hmac_s hmac; 52ff27e85aSGilad Ben-Yossef struct cc_xcbc_s xcbc; 53ff27e85aSGilad Ben-Yossef } auth_state; 54ff27e85aSGilad Ben-Yossef unsigned int enc_keylen; 55ff27e85aSGilad Ben-Yossef unsigned int auth_keylen; 56ff27e85aSGilad Ben-Yossef unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 57f1e52fd0SYael Chemla unsigned int hash_len; 58ff27e85aSGilad Ben-Yossef enum drv_cipher_mode cipher_mode; 59ff27e85aSGilad Ben-Yossef enum cc_flow_mode flow_mode; 60ff27e85aSGilad Ben-Yossef enum drv_hash_mode auth_mode; 61ff27e85aSGilad Ben-Yossef }; 62ff27e85aSGilad Ben-Yossef 63ff27e85aSGilad Ben-Yossef static inline bool valid_assoclen(struct aead_request *req) 64ff27e85aSGilad Ben-Yossef { 65ff27e85aSGilad Ben-Yossef return ((req->assoclen == 16) || (req->assoclen == 20)); 66ff27e85aSGilad Ben-Yossef } 67ff27e85aSGilad Ben-Yossef 68ff27e85aSGilad Ben-Yossef static void cc_aead_exit(struct crypto_aead *tfm) 69ff27e85aSGilad Ben-Yossef { 70ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 71ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 72ff27e85aSGilad Ben-Yossef 73ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 74ff27e85aSGilad Ben-Yossef crypto_tfm_alg_name(&tfm->base)); 75ff27e85aSGilad Ben-Yossef 76ff27e85aSGilad Ben-Yossef /* Unmap enckey buffer */ 77ff27e85aSGilad Ben-Yossef if (ctx->enckey) { 78ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 79ff27e85aSGilad Ben-Yossef ctx->enckey_dma_addr); 80ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 81ff27e85aSGilad Ben-Yossef &ctx->enckey_dma_addr); 82ff27e85aSGilad Ben-Yossef ctx->enckey_dma_addr = 0; 83ff27e85aSGilad Ben-Yossef ctx->enckey = NULL; 84ff27e85aSGilad Ben-Yossef } 85ff27e85aSGilad Ben-Yossef 86ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 87ff27e85aSGilad Ben-Yossef struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 88ff27e85aSGilad Ben-Yossef 89ff27e85aSGilad Ben-Yossef if (xcbc->xcbc_keys) { 90ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 91ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys, 92ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys_dma_addr); 93ff27e85aSGilad Ben-Yossef } 94ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 95ff27e85aSGilad Ben-Yossef &xcbc->xcbc_keys_dma_addr); 96ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys_dma_addr = 0; 97ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys = NULL; 98ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 99ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 100ff27e85aSGilad Ben-Yossef 101ff27e85aSGilad Ben-Yossef if (hmac->ipad_opad) { 102ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 103ff27e85aSGilad Ben-Yossef hmac->ipad_opad, 104ff27e85aSGilad Ben-Yossef hmac->ipad_opad_dma_addr); 105ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 106ff27e85aSGilad Ben-Yossef &hmac->ipad_opad_dma_addr); 107ff27e85aSGilad Ben-Yossef hmac->ipad_opad_dma_addr = 0; 108ff27e85aSGilad Ben-Yossef hmac->ipad_opad = NULL; 109ff27e85aSGilad Ben-Yossef } 110ff27e85aSGilad Ben-Yossef if (hmac->padded_authkey) { 111ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 112ff27e85aSGilad Ben-Yossef hmac->padded_authkey, 113ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr); 114ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 115ff27e85aSGilad Ben-Yossef &hmac->padded_authkey_dma_addr); 116ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr = 0; 117ff27e85aSGilad Ben-Yossef hmac->padded_authkey = NULL; 118ff27e85aSGilad Ben-Yossef } 119ff27e85aSGilad Ben-Yossef } 120ff27e85aSGilad Ben-Yossef } 121ff27e85aSGilad Ben-Yossef 122f1e52fd0SYael Chemla static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 123f1e52fd0SYael Chemla { 124f1e52fd0SYael Chemla struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 125f1e52fd0SYael Chemla 126f1e52fd0SYael Chemla return cc_get_default_hash_len(ctx->drvdata); 127f1e52fd0SYael Chemla } 128f1e52fd0SYael Chemla 129ff27e85aSGilad Ben-Yossef static int cc_aead_init(struct crypto_aead *tfm) 130ff27e85aSGilad Ben-Yossef { 131ff27e85aSGilad Ben-Yossef struct aead_alg *alg = crypto_aead_alg(tfm); 132ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 133ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *cc_alg = 134ff27e85aSGilad Ben-Yossef container_of(alg, struct cc_crypto_alg, aead_alg); 135ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(cc_alg->drvdata); 136ff27e85aSGilad Ben-Yossef 137ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 138ff27e85aSGilad Ben-Yossef crypto_tfm_alg_name(&tfm->base)); 139ff27e85aSGilad Ben-Yossef 140ff27e85aSGilad Ben-Yossef /* Initialize modes in instance */ 141ff27e85aSGilad Ben-Yossef ctx->cipher_mode = cc_alg->cipher_mode; 142ff27e85aSGilad Ben-Yossef ctx->flow_mode = cc_alg->flow_mode; 143ff27e85aSGilad Ben-Yossef ctx->auth_mode = cc_alg->auth_mode; 144ff27e85aSGilad Ben-Yossef ctx->drvdata = cc_alg->drvdata; 145ff27e85aSGilad Ben-Yossef crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx)); 146ff27e85aSGilad Ben-Yossef 147ff27e85aSGilad Ben-Yossef /* Allocate key buffer, cache line aligned */ 148ff27e85aSGilad Ben-Yossef ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 149ff27e85aSGilad Ben-Yossef &ctx->enckey_dma_addr, GFP_KERNEL); 150ff27e85aSGilad Ben-Yossef if (!ctx->enckey) { 151ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating key buffer\n"); 152ff27e85aSGilad Ben-Yossef goto init_failed; 153ff27e85aSGilad Ben-Yossef } 154ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 155ff27e85aSGilad Ben-Yossef ctx->enckey); 156ff27e85aSGilad Ben-Yossef 157ff27e85aSGilad Ben-Yossef /* Set default authlen value */ 158ff27e85aSGilad Ben-Yossef 159ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 160ff27e85aSGilad Ben-Yossef struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 161ff27e85aSGilad Ben-Yossef const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 162ff27e85aSGilad Ben-Yossef 163ff27e85aSGilad Ben-Yossef /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 164ff27e85aSGilad Ben-Yossef /* (and temporary for user key - up to 256b) */ 165ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 166ff27e85aSGilad Ben-Yossef &xcbc->xcbc_keys_dma_addr, 167ff27e85aSGilad Ben-Yossef GFP_KERNEL); 168ff27e85aSGilad Ben-Yossef if (!xcbc->xcbc_keys) { 169ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 170ff27e85aSGilad Ben-Yossef goto init_failed; 171ff27e85aSGilad Ben-Yossef } 172ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 173ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 174ff27e85aSGilad Ben-Yossef const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 175ff27e85aSGilad Ben-Yossef dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 176ff27e85aSGilad Ben-Yossef 177ff27e85aSGilad Ben-Yossef /* Allocate dma-coherent buffer for IPAD + OPAD */ 178ff27e85aSGilad Ben-Yossef hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 179ff27e85aSGilad Ben-Yossef &hmac->ipad_opad_dma_addr, 180ff27e85aSGilad Ben-Yossef GFP_KERNEL); 181ff27e85aSGilad Ben-Yossef 182ff27e85aSGilad Ben-Yossef if (!hmac->ipad_opad) { 183ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 184ff27e85aSGilad Ben-Yossef goto init_failed; 185ff27e85aSGilad Ben-Yossef } 186ff27e85aSGilad Ben-Yossef 187ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 188ff27e85aSGilad Ben-Yossef hmac->ipad_opad); 189ff27e85aSGilad Ben-Yossef 190ff27e85aSGilad Ben-Yossef hmac->padded_authkey = dma_alloc_coherent(dev, 191ff27e85aSGilad Ben-Yossef MAX_HMAC_BLOCK_SIZE, 192ff27e85aSGilad Ben-Yossef pkey_dma, 193ff27e85aSGilad Ben-Yossef GFP_KERNEL); 194ff27e85aSGilad Ben-Yossef 195ff27e85aSGilad Ben-Yossef if (!hmac->padded_authkey) { 196ff27e85aSGilad Ben-Yossef dev_err(dev, "failed to allocate padded_authkey\n"); 197ff27e85aSGilad Ben-Yossef goto init_failed; 198ff27e85aSGilad Ben-Yossef } 199ff27e85aSGilad Ben-Yossef } else { 200ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.ipad_opad = NULL; 201ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.padded_authkey = NULL; 202ff27e85aSGilad Ben-Yossef } 203f1e52fd0SYael Chemla ctx->hash_len = cc_get_aead_hash_len(tfm); 204ff27e85aSGilad Ben-Yossef 205ff27e85aSGilad Ben-Yossef return 0; 206ff27e85aSGilad Ben-Yossef 207ff27e85aSGilad Ben-Yossef init_failed: 208ff27e85aSGilad Ben-Yossef cc_aead_exit(tfm); 209ff27e85aSGilad Ben-Yossef return -ENOMEM; 210ff27e85aSGilad Ben-Yossef } 211ff27e85aSGilad Ben-Yossef 212ff27e85aSGilad Ben-Yossef static void cc_aead_complete(struct device *dev, void *cc_req, int err) 213ff27e85aSGilad Ben-Yossef { 214ff27e85aSGilad Ben-Yossef struct aead_request *areq = (struct aead_request *)cc_req; 215ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 216ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 217ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 218ff27e85aSGilad Ben-Yossef 219a108f931SGilad Ben-Yossef /* BACKLOG notification */ 220a108f931SGilad Ben-Yossef if (err == -EINPROGRESS) 221a108f931SGilad Ben-Yossef goto done; 222a108f931SGilad Ben-Yossef 223ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, areq); 224ff27e85aSGilad Ben-Yossef 225ff27e85aSGilad Ben-Yossef /* Restore ordinary iv pointer */ 226ff27e85aSGilad Ben-Yossef areq->iv = areq_ctx->backup_iv; 227ff27e85aSGilad Ben-Yossef 228ff27e85aSGilad Ben-Yossef if (err) 229ff27e85aSGilad Ben-Yossef goto done; 230ff27e85aSGilad Ben-Yossef 231ff27e85aSGilad Ben-Yossef if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 232ff27e85aSGilad Ben-Yossef if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 233ff27e85aSGilad Ben-Yossef ctx->authsize) != 0) { 234ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 235ff27e85aSGilad Ben-Yossef ctx->authsize, ctx->cipher_mode); 236ff27e85aSGilad Ben-Yossef /* In case of payload authentication failure, MUST NOT 237ff27e85aSGilad Ben-Yossef * revealed the decrypted message --> zero its memory. 238ff27e85aSGilad Ben-Yossef */ 239e88b27c8SGilad Ben-Yossef sg_zero_buffer(areq->dst, sg_nents(areq->dst), 2402a6bc713SGilad Ben-Yossef areq->cryptlen, areq->assoclen); 241ff27e85aSGilad Ben-Yossef err = -EBADMSG; 242ff27e85aSGilad Ben-Yossef } 243e6e6600cSGilad Ben-Yossef /*ENCRYPT*/ 244e6e6600cSGilad Ben-Yossef } else if (areq_ctx->is_icv_fragmented) { 245ff27e85aSGilad Ben-Yossef u32 skip = areq->cryptlen + areq_ctx->dst_offset; 246ff27e85aSGilad Ben-Yossef 247e6e6600cSGilad Ben-Yossef cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl, 248e6e6600cSGilad Ben-Yossef skip, (skip + ctx->authsize), 249ff27e85aSGilad Ben-Yossef CC_SG_FROM_BUF); 250ff27e85aSGilad Ben-Yossef } 251ff27e85aSGilad Ben-Yossef done: 252ff27e85aSGilad Ben-Yossef aead_request_complete(areq, err); 253ff27e85aSGilad Ben-Yossef } 254ff27e85aSGilad Ben-Yossef 255ff27e85aSGilad Ben-Yossef static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 256ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx) 257ff27e85aSGilad Ben-Yossef { 258ff27e85aSGilad Ben-Yossef /* Load the AES key */ 259ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[0]); 260ff27e85aSGilad Ben-Yossef /* We are using for the source/user key the same buffer 261ff27e85aSGilad Ben-Yossef * as for the output keys, * because after this key loading it 262ff27e85aSGilad Ben-Yossef * is not needed anymore 263ff27e85aSGilad Ben-Yossef */ 264ff27e85aSGilad Ben-Yossef set_din_type(&desc[0], DMA_DLLI, 265ff27e85aSGilad Ben-Yossef ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 266ff27e85aSGilad Ben-Yossef NS_BIT); 267ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 268ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 269ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[0], ctx->auth_keylen); 270ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[0], S_DIN_to_AES); 271ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 272ff27e85aSGilad Ben-Yossef 273ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[1]); 274ff27e85aSGilad Ben-Yossef set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 275ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[1], DIN_AES_DOUT); 276ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 277ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0); 278ff27e85aSGilad Ben-Yossef 279ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[2]); 280ff27e85aSGilad Ben-Yossef set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 281ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[2], DIN_AES_DOUT); 282ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 283ff27e85aSGilad Ben-Yossef + AES_KEYSIZE_128), 284ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0); 285ff27e85aSGilad Ben-Yossef 286ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[3]); 287ff27e85aSGilad Ben-Yossef set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 288ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[3], DIN_AES_DOUT); 289ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 290ff27e85aSGilad Ben-Yossef + 2 * AES_KEYSIZE_128), 291ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0); 292ff27e85aSGilad Ben-Yossef 293ff27e85aSGilad Ben-Yossef return 4; 294ff27e85aSGilad Ben-Yossef } 295ff27e85aSGilad Ben-Yossef 296798ac398STian Tao static unsigned int hmac_setkey(struct cc_hw_desc *desc, 297798ac398STian Tao struct cc_aead_ctx *ctx) 298ff27e85aSGilad Ben-Yossef { 299ff27e85aSGilad Ben-Yossef unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 300ff27e85aSGilad Ben-Yossef unsigned int digest_ofs = 0; 301ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 302ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 303ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 304ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 305ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 306ff27e85aSGilad Ben-Yossef 307ff27e85aSGilad Ben-Yossef unsigned int idx = 0; 308ff27e85aSGilad Ben-Yossef int i; 309ff27e85aSGilad Ben-Yossef 310ff27e85aSGilad Ben-Yossef /* calc derived HMAC key */ 311ff27e85aSGilad Ben-Yossef for (i = 0; i < 2; i++) { 312ff27e85aSGilad Ben-Yossef /* Load hash initial state */ 313ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 314ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 315ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], 316ff27e85aSGilad Ben-Yossef cc_larval_digest_addr(ctx->drvdata, 317ff27e85aSGilad Ben-Yossef ctx->auth_mode), 318ff27e85aSGilad Ben-Yossef digest_size); 319ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 320ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 321ff27e85aSGilad Ben-Yossef idx++; 322ff27e85aSGilad Ben-Yossef 323ff27e85aSGilad Ben-Yossef /* Load the hash current length*/ 324ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 325ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 326f1e52fd0SYael Chemla set_din_const(&desc[idx], 0, ctx->hash_len); 327ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 328ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 329ff27e85aSGilad Ben-Yossef idx++; 330ff27e85aSGilad Ben-Yossef 331ff27e85aSGilad Ben-Yossef /* Prepare ipad key */ 332ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 333ff27e85aSGilad Ben-Yossef set_xor_val(&desc[idx], hmac_pad_const[i]); 334ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 335ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 336ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 337ff27e85aSGilad Ben-Yossef idx++; 338ff27e85aSGilad Ben-Yossef 339ff27e85aSGilad Ben-Yossef /* Perform HASH update */ 340ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 341ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 342ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr, 343ff27e85aSGilad Ben-Yossef SHA256_BLOCK_SIZE, NS_BIT); 344ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 345ff27e85aSGilad Ben-Yossef set_xor_active(&desc[idx]); 346ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 347ff27e85aSGilad Ben-Yossef idx++; 348ff27e85aSGilad Ben-Yossef 349ff27e85aSGilad Ben-Yossef /* Get the digset */ 350ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 351ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 352ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], 353ff27e85aSGilad Ben-Yossef (hmac->ipad_opad_dma_addr + digest_ofs), 354ff27e85aSGilad Ben-Yossef digest_size, NS_BIT, 0); 355ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 356ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 357ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 358ff27e85aSGilad Ben-Yossef idx++; 359ff27e85aSGilad Ben-Yossef 360ff27e85aSGilad Ben-Yossef digest_ofs += digest_size; 361ff27e85aSGilad Ben-Yossef } 362ff27e85aSGilad Ben-Yossef 363ff27e85aSGilad Ben-Yossef return idx; 364ff27e85aSGilad Ben-Yossef } 365ff27e85aSGilad Ben-Yossef 366ff27e85aSGilad Ben-Yossef static int validate_keys_sizes(struct cc_aead_ctx *ctx) 367ff27e85aSGilad Ben-Yossef { 368ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 369ff27e85aSGilad Ben-Yossef 370ff27e85aSGilad Ben-Yossef dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 371ff27e85aSGilad Ben-Yossef ctx->enc_keylen, ctx->auth_keylen); 372ff27e85aSGilad Ben-Yossef 373ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { 374ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 375ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 376ff27e85aSGilad Ben-Yossef break; 377ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC: 378ff27e85aSGilad Ben-Yossef if (ctx->auth_keylen != AES_KEYSIZE_128 && 379ff27e85aSGilad Ben-Yossef ctx->auth_keylen != AES_KEYSIZE_192 && 380ff27e85aSGilad Ben-Yossef ctx->auth_keylen != AES_KEYSIZE_256) 381ff27e85aSGilad Ben-Yossef return -ENOTSUPP; 382ff27e85aSGilad Ben-Yossef break; 383ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 384ff27e85aSGilad Ben-Yossef if (ctx->auth_keylen > 0) 385ff27e85aSGilad Ben-Yossef return -EINVAL; 386ff27e85aSGilad Ben-Yossef break; 387ff27e85aSGilad Ben-Yossef default: 388c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 389ff27e85aSGilad Ben-Yossef return -EINVAL; 390ff27e85aSGilad Ben-Yossef } 391ff27e85aSGilad Ben-Yossef /* Check cipher key size */ 392ff27e85aSGilad Ben-Yossef if (ctx->flow_mode == S_DIN_to_DES) { 393ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 394c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Invalid cipher(3DES) key size: %u\n", 395ff27e85aSGilad Ben-Yossef ctx->enc_keylen); 396ff27e85aSGilad Ben-Yossef return -EINVAL; 397ff27e85aSGilad Ben-Yossef } 398ff27e85aSGilad Ben-Yossef } else { /* Default assumed to be AES ciphers */ 399ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen != AES_KEYSIZE_128 && 400ff27e85aSGilad Ben-Yossef ctx->enc_keylen != AES_KEYSIZE_192 && 401ff27e85aSGilad Ben-Yossef ctx->enc_keylen != AES_KEYSIZE_256) { 402c7b31c88SGilad Ben-Yossef dev_dbg(dev, "Invalid cipher(AES) key size: %u\n", 403ff27e85aSGilad Ben-Yossef ctx->enc_keylen); 404ff27e85aSGilad Ben-Yossef return -EINVAL; 405ff27e85aSGilad Ben-Yossef } 406ff27e85aSGilad Ben-Yossef } 407ff27e85aSGilad Ben-Yossef 408ff27e85aSGilad Ben-Yossef return 0; /* All tests of keys sizes passed */ 409ff27e85aSGilad Ben-Yossef } 410ff27e85aSGilad Ben-Yossef 411ff27e85aSGilad Ben-Yossef /* This function prepers the user key so it can pass to the hmac processing 412ff27e85aSGilad Ben-Yossef * (copy to intenral buffer or hash in case of key longer than block 413ff27e85aSGilad Ben-Yossef */ 414e8662a6aSGilad Ben-Yossef static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, 415ff27e85aSGilad Ben-Yossef unsigned int keylen) 416ff27e85aSGilad Ben-Yossef { 417ff27e85aSGilad Ben-Yossef dma_addr_t key_dma_addr = 0; 418ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 419ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 420e431cc04SGeert Uytterhoeven u32 larval_addr; 421ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 422ff27e85aSGilad Ben-Yossef unsigned int blocksize; 423ff27e85aSGilad Ben-Yossef unsigned int digestsize; 424ff27e85aSGilad Ben-Yossef unsigned int hashmode; 425ff27e85aSGilad Ben-Yossef unsigned int idx = 0; 426ff27e85aSGilad Ben-Yossef int rc = 0; 427e8662a6aSGilad Ben-Yossef u8 *key = NULL; 428ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 429ff27e85aSGilad Ben-Yossef dma_addr_t padded_authkey_dma_addr = 430ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.padded_authkey_dma_addr; 431ff27e85aSGilad Ben-Yossef 432ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { /* auth_key required and >0 */ 433ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 434ff27e85aSGilad Ben-Yossef blocksize = SHA1_BLOCK_SIZE; 435ff27e85aSGilad Ben-Yossef digestsize = SHA1_DIGEST_SIZE; 436ff27e85aSGilad Ben-Yossef hashmode = DRV_HASH_HW_SHA1; 437ff27e85aSGilad Ben-Yossef break; 438ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 439ff27e85aSGilad Ben-Yossef default: 440ff27e85aSGilad Ben-Yossef blocksize = SHA256_BLOCK_SIZE; 441ff27e85aSGilad Ben-Yossef digestsize = SHA256_DIGEST_SIZE; 442ff27e85aSGilad Ben-Yossef hashmode = DRV_HASH_HW_SHA256; 443ff27e85aSGilad Ben-Yossef } 444ff27e85aSGilad Ben-Yossef 445ff27e85aSGilad Ben-Yossef if (keylen != 0) { 446e8662a6aSGilad Ben-Yossef 447e8662a6aSGilad Ben-Yossef key = kmemdup(authkey, keylen, GFP_KERNEL); 448e8662a6aSGilad Ben-Yossef if (!key) 449e8662a6aSGilad Ben-Yossef return -ENOMEM; 450e8662a6aSGilad Ben-Yossef 451f4274eecSGeert Uytterhoeven key_dma_addr = dma_map_single(dev, key, keylen, DMA_TO_DEVICE); 452ff27e85aSGilad Ben-Yossef if (dma_mapping_error(dev, key_dma_addr)) { 453ff27e85aSGilad Ben-Yossef dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 454ff27e85aSGilad Ben-Yossef key, keylen); 455e8662a6aSGilad Ben-Yossef kzfree(key); 456ff27e85aSGilad Ben-Yossef return -ENOMEM; 457ff27e85aSGilad Ben-Yossef } 458ff27e85aSGilad Ben-Yossef if (keylen > blocksize) { 459ff27e85aSGilad Ben-Yossef /* Load hash initial state */ 460ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 461ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode); 462e431cc04SGeert Uytterhoeven larval_addr = cc_larval_digest_addr(ctx->drvdata, 463e431cc04SGeert Uytterhoeven ctx->auth_mode); 464ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], larval_addr, digestsize); 465ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 466ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 467ff27e85aSGilad Ben-Yossef idx++; 468ff27e85aSGilad Ben-Yossef 469ff27e85aSGilad Ben-Yossef /* Load the hash current length*/ 470ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 471ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode); 472f1e52fd0SYael Chemla set_din_const(&desc[idx], 0, ctx->hash_len); 473ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 474ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 475ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 476ff27e85aSGilad Ben-Yossef idx++; 477ff27e85aSGilad Ben-Yossef 478ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 479ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 480ff27e85aSGilad Ben-Yossef key_dma_addr, keylen, NS_BIT); 481ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 482ff27e85aSGilad Ben-Yossef idx++; 483ff27e85aSGilad Ben-Yossef 484ff27e85aSGilad Ben-Yossef /* Get hashed key */ 485ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 486ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode); 487ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 488ff27e85aSGilad Ben-Yossef digestsize, NS_BIT, 0); 489ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 490ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 491ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 492ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], 493ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN); 494ff27e85aSGilad Ben-Yossef idx++; 495ff27e85aSGilad Ben-Yossef 496ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 497ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, (blocksize - digestsize)); 498ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 499ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 500ff27e85aSGilad Ben-Yossef digestsize), (blocksize - digestsize), 501ff27e85aSGilad Ben-Yossef NS_BIT, 0); 502ff27e85aSGilad Ben-Yossef idx++; 503ff27e85aSGilad Ben-Yossef } else { 504ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 505ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 506ff27e85aSGilad Ben-Yossef keylen, NS_BIT); 507ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 508ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 509ff27e85aSGilad Ben-Yossef keylen, NS_BIT, 0); 510ff27e85aSGilad Ben-Yossef idx++; 511ff27e85aSGilad Ben-Yossef 512ff27e85aSGilad Ben-Yossef if ((blocksize - keylen) != 0) { 513ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 514ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, 515ff27e85aSGilad Ben-Yossef (blocksize - keylen)); 516ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 517ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], 518ff27e85aSGilad Ben-Yossef (padded_authkey_dma_addr + 519ff27e85aSGilad Ben-Yossef keylen), 520ff27e85aSGilad Ben-Yossef (blocksize - keylen), NS_BIT, 0); 521ff27e85aSGilad Ben-Yossef idx++; 522ff27e85aSGilad Ben-Yossef } 523ff27e85aSGilad Ben-Yossef } 524ff27e85aSGilad Ben-Yossef } else { 525ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 526ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, (blocksize - keylen)); 527ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 528ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 529ff27e85aSGilad Ben-Yossef blocksize, NS_BIT, 0); 530ff27e85aSGilad Ben-Yossef idx++; 531ff27e85aSGilad Ben-Yossef } 532ff27e85aSGilad Ben-Yossef 533ff27e85aSGilad Ben-Yossef rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 534ff27e85aSGilad Ben-Yossef if (rc) 535ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc); 536ff27e85aSGilad Ben-Yossef 537ff27e85aSGilad Ben-Yossef if (key_dma_addr) 538ff27e85aSGilad Ben-Yossef dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 539ff27e85aSGilad Ben-Yossef 540e8662a6aSGilad Ben-Yossef kzfree(key); 541e8662a6aSGilad Ben-Yossef 542ff27e85aSGilad Ben-Yossef return rc; 543ff27e85aSGilad Ben-Yossef } 544ff27e85aSGilad Ben-Yossef 545ff27e85aSGilad Ben-Yossef static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 546ff27e85aSGilad Ben-Yossef unsigned int keylen) 547ff27e85aSGilad Ben-Yossef { 548ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 549ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 550ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 551ff27e85aSGilad Ben-Yossef unsigned int seq_len = 0; 552ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 553dc95b535SEric Biggers const u8 *enckey, *authkey; 554dc95b535SEric Biggers int rc; 555ff27e85aSGilad Ben-Yossef 556ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 557ff27e85aSGilad Ben-Yossef ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 558ff27e85aSGilad Ben-Yossef 559ff27e85aSGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 560ff27e85aSGilad Ben-Yossef 561ff27e85aSGilad Ben-Yossef if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 562dc95b535SEric Biggers struct crypto_authenc_keys keys; 563dc95b535SEric Biggers 564dc95b535SEric Biggers rc = crypto_authenc_extractkeys(&keys, key, keylen); 565dc95b535SEric Biggers if (rc) 566674f368aSEric Biggers return rc; 567dc95b535SEric Biggers enckey = keys.enckey; 568dc95b535SEric Biggers authkey = keys.authkey; 569dc95b535SEric Biggers ctx->enc_keylen = keys.enckeylen; 570dc95b535SEric Biggers ctx->auth_keylen = keys.authkeylen; 571ff27e85aSGilad Ben-Yossef 572ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) { 573ff27e85aSGilad Ben-Yossef /* the nonce is stored in bytes at end of key */ 574ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen < 575ff27e85aSGilad Ben-Yossef (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 576674f368aSEric Biggers return -EINVAL; 577ff27e85aSGilad Ben-Yossef /* Copy nonce from last 4 bytes in CTR key to 578ff27e85aSGilad Ben-Yossef * first 4 bytes in CTR IV 579ff27e85aSGilad Ben-Yossef */ 580dc95b535SEric Biggers memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - 581dc95b535SEric Biggers CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); 582ff27e85aSGilad Ben-Yossef /* Set CTR key size */ 583ff27e85aSGilad Ben-Yossef ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 584ff27e85aSGilad Ben-Yossef } 585ff27e85aSGilad Ben-Yossef } else { /* non-authenc - has just one key */ 586dc95b535SEric Biggers enckey = key; 587dc95b535SEric Biggers authkey = NULL; 588ff27e85aSGilad Ben-Yossef ctx->enc_keylen = keylen; 589ff27e85aSGilad Ben-Yossef ctx->auth_keylen = 0; 590ff27e85aSGilad Ben-Yossef } 591ff27e85aSGilad Ben-Yossef 592ff27e85aSGilad Ben-Yossef rc = validate_keys_sizes(ctx); 593ff27e85aSGilad Ben-Yossef if (rc) 594674f368aSEric Biggers return rc; 595ff27e85aSGilad Ben-Yossef 596ff27e85aSGilad Ben-Yossef /* STAT_PHASE_1: Copy key to ctx */ 597ff27e85aSGilad Ben-Yossef 598ff27e85aSGilad Ben-Yossef /* Get key material */ 599dc95b535SEric Biggers memcpy(ctx->enckey, enckey, ctx->enc_keylen); 600ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen == 24) 601ff27e85aSGilad Ben-Yossef memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 602ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 603dc95b535SEric Biggers memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, 604dc95b535SEric Biggers ctx->auth_keylen); 605ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 606dc95b535SEric Biggers rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); 607ff27e85aSGilad Ben-Yossef if (rc) 608674f368aSEric Biggers return rc; 609ff27e85aSGilad Ben-Yossef } 610ff27e85aSGilad Ben-Yossef 611ff27e85aSGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */ 612ff27e85aSGilad Ben-Yossef 613ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { 614ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 615ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 616ff27e85aSGilad Ben-Yossef seq_len = hmac_setkey(desc, ctx); 617ff27e85aSGilad Ben-Yossef break; 618ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC: 619ff27e85aSGilad Ben-Yossef seq_len = xcbc_setkey(desc, ctx); 620ff27e85aSGilad Ben-Yossef break; 621ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 622ff27e85aSGilad Ben-Yossef break; /* No auth. key setup */ 623ff27e85aSGilad Ben-Yossef default: 624ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 625674f368aSEric Biggers return -ENOTSUPP; 626ff27e85aSGilad Ben-Yossef } 627ff27e85aSGilad Ben-Yossef 628ff27e85aSGilad Ben-Yossef /* STAT_PHASE_3: Submit sequence to HW */ 629ff27e85aSGilad Ben-Yossef 630ff27e85aSGilad Ben-Yossef if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 631ff27e85aSGilad Ben-Yossef rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 632ff27e85aSGilad Ben-Yossef if (rc) { 633ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc); 634674f368aSEric Biggers return rc; 635ff27e85aSGilad Ben-Yossef } 636ff27e85aSGilad Ben-Yossef } 637ff27e85aSGilad Ben-Yossef 638ff27e85aSGilad Ben-Yossef /* Update STAT_PHASE_3 */ 639ff27e85aSGilad Ben-Yossef return rc; 640ff27e85aSGilad Ben-Yossef } 641ff27e85aSGilad Ben-Yossef 6429fbfcefcSHerbert Xu static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 6439fbfcefcSHerbert Xu unsigned int keylen) 6449fbfcefcSHerbert Xu { 6459fbfcefcSHerbert Xu struct crypto_authenc_keys keys; 6469fbfcefcSHerbert Xu int err; 6479fbfcefcSHerbert Xu 6489fbfcefcSHerbert Xu err = crypto_authenc_extractkeys(&keys, key, keylen); 6499fbfcefcSHerbert Xu if (unlikely(err)) 6509fbfcefcSHerbert Xu return err; 6519fbfcefcSHerbert Xu 65200cd6b23SArd Biesheuvel err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 65300cd6b23SArd Biesheuvel cc_aead_setkey(aead, key, keylen); 65400cd6b23SArd Biesheuvel 65500cd6b23SArd Biesheuvel memzero_explicit(&keys, sizeof(keys)); 65600cd6b23SArd Biesheuvel return err; 6579fbfcefcSHerbert Xu } 6589fbfcefcSHerbert Xu 659ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 660ff27e85aSGilad Ben-Yossef unsigned int keylen) 661ff27e85aSGilad Ben-Yossef { 662ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 663ff27e85aSGilad Ben-Yossef 664ff27e85aSGilad Ben-Yossef if (keylen < 3) 665ff27e85aSGilad Ben-Yossef return -EINVAL; 666ff27e85aSGilad Ben-Yossef 667ff27e85aSGilad Ben-Yossef keylen -= 3; 668ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 3); 669ff27e85aSGilad Ben-Yossef 670ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen); 671ff27e85aSGilad Ben-Yossef } 672ff27e85aSGilad Ben-Yossef 673ff27e85aSGilad Ben-Yossef static int cc_aead_setauthsize(struct crypto_aead *authenc, 674ff27e85aSGilad Ben-Yossef unsigned int authsize) 675ff27e85aSGilad Ben-Yossef { 676ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 677ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 678ff27e85aSGilad Ben-Yossef 679ff27e85aSGilad Ben-Yossef /* Unsupported auth. sizes */ 680ff27e85aSGilad Ben-Yossef if (authsize == 0 || 681ff27e85aSGilad Ben-Yossef authsize > crypto_aead_maxauthsize(authenc)) { 682ff27e85aSGilad Ben-Yossef return -ENOTSUPP; 683ff27e85aSGilad Ben-Yossef } 684ff27e85aSGilad Ben-Yossef 685ff27e85aSGilad Ben-Yossef ctx->authsize = authsize; 686ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authlen=%d\n", ctx->authsize); 687ff27e85aSGilad Ben-Yossef 688ff27e85aSGilad Ben-Yossef return 0; 689ff27e85aSGilad Ben-Yossef } 690ff27e85aSGilad Ben-Yossef 691ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 692ff27e85aSGilad Ben-Yossef unsigned int authsize) 693ff27e85aSGilad Ben-Yossef { 694ff27e85aSGilad Ben-Yossef switch (authsize) { 695ff27e85aSGilad Ben-Yossef case 8: 696ff27e85aSGilad Ben-Yossef case 12: 697ff27e85aSGilad Ben-Yossef case 16: 698ff27e85aSGilad Ben-Yossef break; 699ff27e85aSGilad Ben-Yossef default: 700ff27e85aSGilad Ben-Yossef return -EINVAL; 701ff27e85aSGilad Ben-Yossef } 702ff27e85aSGilad Ben-Yossef 703ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 704ff27e85aSGilad Ben-Yossef } 705ff27e85aSGilad Ben-Yossef 706ff27e85aSGilad Ben-Yossef static int cc_ccm_setauthsize(struct crypto_aead *authenc, 707ff27e85aSGilad Ben-Yossef unsigned int authsize) 708ff27e85aSGilad Ben-Yossef { 709ff27e85aSGilad Ben-Yossef switch (authsize) { 710ff27e85aSGilad Ben-Yossef case 4: 711ff27e85aSGilad Ben-Yossef case 6: 712ff27e85aSGilad Ben-Yossef case 8: 713ff27e85aSGilad Ben-Yossef case 10: 714ff27e85aSGilad Ben-Yossef case 12: 715ff27e85aSGilad Ben-Yossef case 14: 716ff27e85aSGilad Ben-Yossef case 16: 717ff27e85aSGilad Ben-Yossef break; 718ff27e85aSGilad Ben-Yossef default: 719ff27e85aSGilad Ben-Yossef return -EINVAL; 720ff27e85aSGilad Ben-Yossef } 721ff27e85aSGilad Ben-Yossef 722ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 723ff27e85aSGilad Ben-Yossef } 724ff27e85aSGilad Ben-Yossef 725ff27e85aSGilad Ben-Yossef static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 726ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 727ff27e85aSGilad Ben-Yossef { 728ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 729ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 730ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 731ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 732ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 733ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 734ff27e85aSGilad Ben-Yossef 735ff27e85aSGilad Ben-Yossef switch (assoc_dma_type) { 736ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI: 737ff27e85aSGilad Ben-Yossef dev_dbg(dev, "ASSOC buffer type DLLI\n"); 738ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 739ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 740da3cf67fSGilad Ben-Yossef areq_ctx->assoclen, NS_BIT); 741ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 742ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 743ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen > 0) 744ff27e85aSGilad Ben-Yossef set_din_not_last_indication(&desc[idx]); 745ff27e85aSGilad Ben-Yossef break; 746ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI: 747ff27e85aSGilad Ben-Yossef dev_dbg(dev, "ASSOC buffer type MLLI\n"); 748ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 749ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 750ff27e85aSGilad Ben-Yossef areq_ctx->assoc.mlli_nents, NS_BIT); 751ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 752ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 753ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen > 0) 754ff27e85aSGilad Ben-Yossef set_din_not_last_indication(&desc[idx]); 755ff27e85aSGilad Ben-Yossef break; 756ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL: 757ff27e85aSGilad Ben-Yossef default: 758ff27e85aSGilad Ben-Yossef dev_err(dev, "Invalid ASSOC buffer type\n"); 759ff27e85aSGilad Ben-Yossef } 760ff27e85aSGilad Ben-Yossef 761ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 762ff27e85aSGilad Ben-Yossef } 763ff27e85aSGilad Ben-Yossef 764ff27e85aSGilad Ben-Yossef static void cc_proc_authen_desc(struct aead_request *areq, 765ff27e85aSGilad Ben-Yossef unsigned int flow_mode, 766ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 767ff27e85aSGilad Ben-Yossef unsigned int *seq_size, int direct) 768ff27e85aSGilad Ben-Yossef { 769ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 770ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 771ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 772ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 773ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 774ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 775ff27e85aSGilad Ben-Yossef 776ff27e85aSGilad Ben-Yossef switch (data_dma_type) { 777ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI: 778ff27e85aSGilad Ben-Yossef { 779ff27e85aSGilad Ben-Yossef struct scatterlist *cipher = 780ff27e85aSGilad Ben-Yossef (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 781ff27e85aSGilad Ben-Yossef areq_ctx->dst_sgl : areq_ctx->src_sgl; 782ff27e85aSGilad Ben-Yossef 783ff27e85aSGilad Ben-Yossef unsigned int offset = 784ff27e85aSGilad Ben-Yossef (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 785ff27e85aSGilad Ben-Yossef areq_ctx->dst_offset : areq_ctx->src_offset; 786ff27e85aSGilad Ben-Yossef dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 787ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 788ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 789ff27e85aSGilad Ben-Yossef (sg_dma_address(cipher) + offset), 790ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen, NS_BIT); 791ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 792ff27e85aSGilad Ben-Yossef break; 793ff27e85aSGilad Ben-Yossef } 794ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI: 795ff27e85aSGilad Ben-Yossef { 796ff27e85aSGilad Ben-Yossef /* DOUBLE-PASS flow (as default) 797ff27e85aSGilad Ben-Yossef * assoc. + iv + data -compact in one table 798ff27e85aSGilad Ben-Yossef * if assoclen is ZERO only IV perform 799ff27e85aSGilad Ben-Yossef */ 800ff27e85aSGilad Ben-Yossef cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr; 801ff27e85aSGilad Ben-Yossef u32 mlli_nents = areq_ctx->assoc.mlli_nents; 802ff27e85aSGilad Ben-Yossef 803ff27e85aSGilad Ben-Yossef if (areq_ctx->is_single_pass) { 804ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 805ff27e85aSGilad Ben-Yossef mlli_addr = areq_ctx->dst.sram_addr; 806ff27e85aSGilad Ben-Yossef mlli_nents = areq_ctx->dst.mlli_nents; 807ff27e85aSGilad Ben-Yossef } else { 808ff27e85aSGilad Ben-Yossef mlli_addr = areq_ctx->src.sram_addr; 809ff27e85aSGilad Ben-Yossef mlli_nents = areq_ctx->src.mlli_nents; 810ff27e85aSGilad Ben-Yossef } 811ff27e85aSGilad Ben-Yossef } 812ff27e85aSGilad Ben-Yossef 813ff27e85aSGilad Ben-Yossef dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 814ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 815ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 816ff27e85aSGilad Ben-Yossef NS_BIT); 817ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 818ff27e85aSGilad Ben-Yossef break; 819ff27e85aSGilad Ben-Yossef } 820ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL: 821ff27e85aSGilad Ben-Yossef default: 822ff27e85aSGilad Ben-Yossef dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 823ff27e85aSGilad Ben-Yossef } 824ff27e85aSGilad Ben-Yossef 825ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 826ff27e85aSGilad Ben-Yossef } 827ff27e85aSGilad Ben-Yossef 828ff27e85aSGilad Ben-Yossef static void cc_proc_cipher_desc(struct aead_request *areq, 829ff27e85aSGilad Ben-Yossef unsigned int flow_mode, 830ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 831ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 832ff27e85aSGilad Ben-Yossef { 833ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 834ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 835ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 836ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 837ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 838ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 839ff27e85aSGilad Ben-Yossef 840ff27e85aSGilad Ben-Yossef if (areq_ctx->cryptlen == 0) 841ff27e85aSGilad Ben-Yossef return; /*null processing*/ 842ff27e85aSGilad Ben-Yossef 843ff27e85aSGilad Ben-Yossef switch (data_dma_type) { 844ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI: 845ff27e85aSGilad Ben-Yossef dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 846ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 847ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 848ff27e85aSGilad Ben-Yossef (sg_dma_address(areq_ctx->src_sgl) + 849ff27e85aSGilad Ben-Yossef areq_ctx->src_offset), areq_ctx->cryptlen, 850ff27e85aSGilad Ben-Yossef NS_BIT); 851ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], 852ff27e85aSGilad Ben-Yossef (sg_dma_address(areq_ctx->dst_sgl) + 853ff27e85aSGilad Ben-Yossef areq_ctx->dst_offset), 854ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen, NS_BIT, 0); 855ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 856ff27e85aSGilad Ben-Yossef break; 857ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI: 858ff27e85aSGilad Ben-Yossef dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 859ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 860ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 861ff27e85aSGilad Ben-Yossef areq_ctx->src.mlli_nents, NS_BIT); 862ff27e85aSGilad Ben-Yossef set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 863ff27e85aSGilad Ben-Yossef areq_ctx->dst.mlli_nents, NS_BIT, 0); 864ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 865ff27e85aSGilad Ben-Yossef break; 866ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL: 867ff27e85aSGilad Ben-Yossef default: 868ff27e85aSGilad Ben-Yossef dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 869ff27e85aSGilad Ben-Yossef } 870ff27e85aSGilad Ben-Yossef 871ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 872ff27e85aSGilad Ben-Yossef } 873ff27e85aSGilad Ben-Yossef 874ff27e85aSGilad Ben-Yossef static void cc_proc_digest_desc(struct aead_request *req, 875ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 876ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 877ff27e85aSGilad Ben-Yossef { 878ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 879ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 880ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 881ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 882ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 883ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 884ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 885ff27e85aSGilad Ben-Yossef 886ff27e85aSGilad Ben-Yossef /* Get final ICV result */ 887ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 888ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 889ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 890ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 891ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 892ff27e85aSGilad Ben-Yossef NS_BIT, 1); 89327b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 894ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 895ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 896ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 897ff27e85aSGilad Ben-Yossef } else { 898ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], 899ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN); 900ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 901ff27e85aSGilad Ben-Yossef } 902ff27e85aSGilad Ben-Yossef } else { /*Decrypt*/ 903ff27e85aSGilad Ben-Yossef /* Get ICV out from hardware */ 904ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 905ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 906ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 907ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 908ff27e85aSGilad Ben-Yossef ctx->authsize, NS_BIT, 1); 90927b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 910ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], 911ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN); 912ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 913ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 914ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 915ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 916ff27e85aSGilad Ben-Yossef } else { 917ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 918ff27e85aSGilad Ben-Yossef } 919ff27e85aSGilad Ben-Yossef } 920ff27e85aSGilad Ben-Yossef 921ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 922ff27e85aSGilad Ben-Yossef } 923ff27e85aSGilad Ben-Yossef 924ff27e85aSGilad Ben-Yossef static void cc_set_cipher_desc(struct aead_request *req, 925ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 926ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 927ff27e85aSGilad Ben-Yossef { 928ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 929ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 930ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 931ff27e85aSGilad Ben-Yossef unsigned int hw_iv_size = req_ctx->hw_iv_size; 932ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 933ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 934ff27e85aSGilad Ben-Yossef 935ff27e85aSGilad Ben-Yossef /* Setup cipher state */ 936ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 937ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], direct); 938ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], ctx->flow_mode); 939ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 940ff27e85aSGilad Ben-Yossef hw_iv_size, NS_BIT); 941ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) 942ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 943ff27e85aSGilad Ben-Yossef else 944ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 945ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], ctx->cipher_mode); 946ff27e85aSGilad Ben-Yossef idx++; 947ff27e85aSGilad Ben-Yossef 948ff27e85aSGilad Ben-Yossef /* Setup enc. key */ 949ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 950ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], direct); 951ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 952ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], ctx->flow_mode); 953ff27e85aSGilad Ben-Yossef if (ctx->flow_mode == S_DIN_to_AES) { 954ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 955ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 956ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT); 957ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 958ff27e85aSGilad Ben-Yossef } else { 959ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 960ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT); 961ff27e85aSGilad Ben-Yossef set_key_size_des(&desc[idx], ctx->enc_keylen); 962ff27e85aSGilad Ben-Yossef } 963ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], ctx->cipher_mode); 964ff27e85aSGilad Ben-Yossef idx++; 965ff27e85aSGilad Ben-Yossef 966ff27e85aSGilad Ben-Yossef *seq_size = idx; 967ff27e85aSGilad Ben-Yossef } 968ff27e85aSGilad Ben-Yossef 969ff27e85aSGilad Ben-Yossef static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 970ff27e85aSGilad Ben-Yossef unsigned int *seq_size, unsigned int data_flow_mode) 971ff27e85aSGilad Ben-Yossef { 972ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 973ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 974ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 975ff27e85aSGilad Ben-Yossef 976ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen == 0) 977ff27e85aSGilad Ben-Yossef return; /*null processing*/ 978ff27e85aSGilad Ben-Yossef 979ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, &idx); 980ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 981ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 982ff27e85aSGilad Ben-Yossef /* We must wait for DMA to write all cipher */ 983ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 984ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 985ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 986ff27e85aSGilad Ben-Yossef idx++; 987ff27e85aSGilad Ben-Yossef } 988ff27e85aSGilad Ben-Yossef 989ff27e85aSGilad Ben-Yossef *seq_size = idx; 990ff27e85aSGilad Ben-Yossef } 991ff27e85aSGilad Ben-Yossef 992ff27e85aSGilad Ben-Yossef static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 993ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 994ff27e85aSGilad Ben-Yossef { 995ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 996ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 997ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 998ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 999ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1000ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1001ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1002ff27e85aSGilad Ben-Yossef 1003ff27e85aSGilad Ben-Yossef /* Loading hash ipad xor key state */ 1004ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1005ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1006ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1007ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1008ff27e85aSGilad Ben-Yossef NS_BIT); 1009ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1010ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1011ff27e85aSGilad Ben-Yossef idx++; 1012ff27e85aSGilad Ben-Yossef 1013ff27e85aSGilad Ben-Yossef /* Load init. digest len (64 bytes) */ 1014ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1015ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1016ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1017f1e52fd0SYael Chemla ctx->hash_len); 1018ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1019ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1020ff27e85aSGilad Ben-Yossef idx++; 1021ff27e85aSGilad Ben-Yossef 1022ff27e85aSGilad Ben-Yossef *seq_size = idx; 1023ff27e85aSGilad Ben-Yossef } 1024ff27e85aSGilad Ben-Yossef 1025ff27e85aSGilad Ben-Yossef static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1026ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1027ff27e85aSGilad Ben-Yossef { 1028ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1029ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1030ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1031ff27e85aSGilad Ben-Yossef 1032ff27e85aSGilad Ben-Yossef /* Loading MAC state */ 1033ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1034ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1035ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1036ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1037ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1038ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1039ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1040ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1041ff27e85aSGilad Ben-Yossef idx++; 1042ff27e85aSGilad Ben-Yossef 1043ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K1 */ 1044ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1045ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1046ff27e85aSGilad Ben-Yossef ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1047ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT); 1048ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1049ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1050ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1051ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1052ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1053ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1054ff27e85aSGilad Ben-Yossef idx++; 1055ff27e85aSGilad Ben-Yossef 1056ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K2 */ 1057ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1058ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1059ff27e85aSGilad Ben-Yossef (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1060ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1061ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1062ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1063ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1064ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1065ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1066ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1067ff27e85aSGilad Ben-Yossef idx++; 1068ff27e85aSGilad Ben-Yossef 1069ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K3 */ 1070ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1071ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1072ff27e85aSGilad Ben-Yossef (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1073ff27e85aSGilad Ben-Yossef 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1074ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1075ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1076ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1077ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1078ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1079ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1080ff27e85aSGilad Ben-Yossef idx++; 1081ff27e85aSGilad Ben-Yossef 1082ff27e85aSGilad Ben-Yossef *seq_size = idx; 1083ff27e85aSGilad Ben-Yossef } 1084ff27e85aSGilad Ben-Yossef 1085ff27e85aSGilad Ben-Yossef static void cc_proc_header_desc(struct aead_request *req, 1086ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 1087ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1088ff27e85aSGilad Ben-Yossef { 1089da3cf67fSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1090ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1091da3cf67fSGilad Ben-Yossef 1092ff27e85aSGilad Ben-Yossef /* Hash associated data */ 1093da3cf67fSGilad Ben-Yossef if (areq_ctx->assoclen > 0) 1094ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1095ff27e85aSGilad Ben-Yossef 1096ff27e85aSGilad Ben-Yossef /* Hash IV */ 1097ff27e85aSGilad Ben-Yossef *seq_size = idx; 1098ff27e85aSGilad Ben-Yossef } 1099ff27e85aSGilad Ben-Yossef 1100ff27e85aSGilad Ben-Yossef static void cc_proc_scheme_desc(struct aead_request *req, 1101ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 1102ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1103ff27e85aSGilad Ben-Yossef { 1104ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1105ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1106ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1107ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1108ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1109ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1110ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1111ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1112ff27e85aSGilad Ben-Yossef 1113ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1114ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1115ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1116f1e52fd0SYael Chemla ctx->hash_len); 1117ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1118ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1119ff27e85aSGilad Ben-Yossef set_cipher_do(&desc[idx], DO_PAD); 1120ff27e85aSGilad Ben-Yossef idx++; 1121ff27e85aSGilad Ben-Yossef 1122ff27e85aSGilad Ben-Yossef /* Get final ICV result */ 1123ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1124ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1125ff27e85aSGilad Ben-Yossef digest_size); 1126ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1127ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1128ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1129ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1130ff27e85aSGilad Ben-Yossef idx++; 1131ff27e85aSGilad Ben-Yossef 1132ff27e85aSGilad Ben-Yossef /* Loading hash opad xor key state */ 1133ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1134ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1135ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1136ff27e85aSGilad Ben-Yossef (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1137ff27e85aSGilad Ben-Yossef digest_size, NS_BIT); 1138ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1139ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1140ff27e85aSGilad Ben-Yossef idx++; 1141ff27e85aSGilad Ben-Yossef 1142ff27e85aSGilad Ben-Yossef /* Load init. digest len (64 bytes) */ 1143ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1144ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1145ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1146f1e52fd0SYael Chemla ctx->hash_len); 1147ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1148ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1149ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1150ff27e85aSGilad Ben-Yossef idx++; 1151ff27e85aSGilad Ben-Yossef 1152ff27e85aSGilad Ben-Yossef /* Perform HASH update */ 1153ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1154ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1155ff27e85aSGilad Ben-Yossef digest_size); 1156ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 1157ff27e85aSGilad Ben-Yossef idx++; 1158ff27e85aSGilad Ben-Yossef 1159ff27e85aSGilad Ben-Yossef *seq_size = idx; 1160ff27e85aSGilad Ben-Yossef } 1161ff27e85aSGilad Ben-Yossef 1162ff27e85aSGilad Ben-Yossef static void cc_mlli_to_sram(struct aead_request *req, 1163ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 1164ff27e85aSGilad Ben-Yossef { 1165ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1166ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1167ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1168ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1169ff27e85aSGilad Ben-Yossef 1170d2d34fb5SGilad Ben-Yossef if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1171ff27e85aSGilad Ben-Yossef req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1172d2d34fb5SGilad Ben-Yossef !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { 1173ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1174ff27e85aSGilad Ben-Yossef (unsigned int)ctx->drvdata->mlli_sram_addr, 1175ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len); 1176ff27e85aSGilad Ben-Yossef /* Copy MLLI table host-to-sram */ 1177ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 1178ff27e85aSGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, 1179ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_dma_addr, 1180ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len, NS_BIT); 1181ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[*seq_size], 1182ff27e85aSGilad Ben-Yossef ctx->drvdata->mlli_sram_addr, 1183ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len); 1184ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], BYPASS); 1185ff27e85aSGilad Ben-Yossef (*seq_size)++; 1186ff27e85aSGilad Ben-Yossef } 1187ff27e85aSGilad Ben-Yossef } 1188ff27e85aSGilad Ben-Yossef 1189ff27e85aSGilad Ben-Yossef static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1190ff27e85aSGilad Ben-Yossef enum cc_flow_mode setup_flow_mode, 1191ff27e85aSGilad Ben-Yossef bool is_single_pass) 1192ff27e85aSGilad Ben-Yossef { 1193ff27e85aSGilad Ben-Yossef enum cc_flow_mode data_flow_mode; 1194ff27e85aSGilad Ben-Yossef 1195ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1196ff27e85aSGilad Ben-Yossef if (setup_flow_mode == S_DIN_to_AES) 1197ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1198ff27e85aSGilad Ben-Yossef AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1199ff27e85aSGilad Ben-Yossef else 1200ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1201ff27e85aSGilad Ben-Yossef DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1202ff27e85aSGilad Ben-Yossef } else { /* Decrypt */ 1203ff27e85aSGilad Ben-Yossef if (setup_flow_mode == S_DIN_to_AES) 1204ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1205ff27e85aSGilad Ben-Yossef AES_and_HASH : DIN_AES_DOUT; 1206ff27e85aSGilad Ben-Yossef else 1207ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1208ff27e85aSGilad Ben-Yossef DES_and_HASH : DIN_DES_DOUT; 1209ff27e85aSGilad Ben-Yossef } 1210ff27e85aSGilad Ben-Yossef 1211ff27e85aSGilad Ben-Yossef return data_flow_mode; 1212ff27e85aSGilad Ben-Yossef } 1213ff27e85aSGilad Ben-Yossef 1214ff27e85aSGilad Ben-Yossef static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1215ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1216ff27e85aSGilad Ben-Yossef { 1217ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1218ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1219ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1220ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 1221ff27e85aSGilad Ben-Yossef unsigned int data_flow_mode = 1222ff27e85aSGilad Ben-Yossef cc_get_data_flow(direct, ctx->flow_mode, 1223ff27e85aSGilad Ben-Yossef req_ctx->is_single_pass); 1224ff27e85aSGilad Ben-Yossef 1225ff27e85aSGilad Ben-Yossef if (req_ctx->is_single_pass) { 1226ff27e85aSGilad Ben-Yossef /** 1227ff27e85aSGilad Ben-Yossef * Single-pass flow 1228ff27e85aSGilad Ben-Yossef */ 1229ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size); 1230ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, seq_size); 1231ff27e85aSGilad Ben-Yossef cc_proc_header_desc(req, desc, seq_size); 1232ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1233ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size); 1234ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1235ff27e85aSGilad Ben-Yossef return; 1236ff27e85aSGilad Ben-Yossef } 1237ff27e85aSGilad Ben-Yossef 1238ff27e85aSGilad Ben-Yossef /** 1239ff27e85aSGilad Ben-Yossef * Double-pass flow 1240ff27e85aSGilad Ben-Yossef * Fallback for unsupported single-pass modes, 1241ff27e85aSGilad Ben-Yossef * i.e. using assoc. data of non-word-multiple 1242ff27e85aSGilad Ben-Yossef */ 1243ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1244ff27e85aSGilad Ben-Yossef /* encrypt first.. */ 1245ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1246ff27e85aSGilad Ben-Yossef /* authenc after..*/ 1247ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size); 1248ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1249ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size); 1250ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1251ff27e85aSGilad Ben-Yossef 1252ff27e85aSGilad Ben-Yossef } else { /*DECRYPT*/ 1253ff27e85aSGilad Ben-Yossef /* authenc first..*/ 1254ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size); 1255ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1256ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size); 1257ff27e85aSGilad Ben-Yossef /* decrypt after.. */ 1258ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1259ff27e85aSGilad Ben-Yossef /* read the digest result with setting the completion bit 1260ff27e85aSGilad Ben-Yossef * must be after the cipher operation 1261ff27e85aSGilad Ben-Yossef */ 1262ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1263ff27e85aSGilad Ben-Yossef } 1264ff27e85aSGilad Ben-Yossef } 1265ff27e85aSGilad Ben-Yossef 1266ff27e85aSGilad Ben-Yossef static void 1267ff27e85aSGilad Ben-Yossef cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1268ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1269ff27e85aSGilad Ben-Yossef { 1270ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1271ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1272ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1273ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 1274ff27e85aSGilad Ben-Yossef unsigned int data_flow_mode = 1275ff27e85aSGilad Ben-Yossef cc_get_data_flow(direct, ctx->flow_mode, 1276ff27e85aSGilad Ben-Yossef req_ctx->is_single_pass); 1277ff27e85aSGilad Ben-Yossef 1278ff27e85aSGilad Ben-Yossef if (req_ctx->is_single_pass) { 1279ff27e85aSGilad Ben-Yossef /** 1280ff27e85aSGilad Ben-Yossef * Single-pass flow 1281ff27e85aSGilad Ben-Yossef */ 1282ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size); 1283ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, seq_size); 1284ff27e85aSGilad Ben-Yossef cc_proc_header_desc(req, desc, seq_size); 1285ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1286ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1287ff27e85aSGilad Ben-Yossef return; 1288ff27e85aSGilad Ben-Yossef } 1289ff27e85aSGilad Ben-Yossef 1290ff27e85aSGilad Ben-Yossef /** 1291ff27e85aSGilad Ben-Yossef * Double-pass flow 1292ff27e85aSGilad Ben-Yossef * Fallback for unsupported single-pass modes, 1293ff27e85aSGilad Ben-Yossef * i.e. using assoc. data of non-word-multiple 1294ff27e85aSGilad Ben-Yossef */ 1295ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1296ff27e85aSGilad Ben-Yossef /* encrypt first.. */ 1297ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1298ff27e85aSGilad Ben-Yossef /* authenc after.. */ 1299ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size); 1300ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1301ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1302ff27e85aSGilad Ben-Yossef } else { /*DECRYPT*/ 1303ff27e85aSGilad Ben-Yossef /* authenc first.. */ 1304ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size); 1305ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1306ff27e85aSGilad Ben-Yossef /* decrypt after..*/ 1307ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1308ff27e85aSGilad Ben-Yossef /* read the digest result with setting the completion bit 1309ff27e85aSGilad Ben-Yossef * must be after the cipher operation 1310ff27e85aSGilad Ben-Yossef */ 1311ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1312ff27e85aSGilad Ben-Yossef } 1313ff27e85aSGilad Ben-Yossef } 1314ff27e85aSGilad Ben-Yossef 1315ff27e85aSGilad Ben-Yossef static int validate_data_size(struct cc_aead_ctx *ctx, 1316ff27e85aSGilad Ben-Yossef enum drv_crypto_direction direct, 1317ff27e85aSGilad Ben-Yossef struct aead_request *req) 1318ff27e85aSGilad Ben-Yossef { 1319ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1320ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1321da3cf67fSGilad Ben-Yossef unsigned int assoclen = areq_ctx->assoclen; 1322ff27e85aSGilad Ben-Yossef unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1323ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize) : req->cryptlen; 1324ff27e85aSGilad Ben-Yossef 1325ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1326ff27e85aSGilad Ben-Yossef req->cryptlen < ctx->authsize) 1327ff27e85aSGilad Ben-Yossef goto data_size_err; 1328ff27e85aSGilad Ben-Yossef 1329ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1330ff27e85aSGilad Ben-Yossef 1331ff27e85aSGilad Ben-Yossef switch (ctx->flow_mode) { 1332ff27e85aSGilad Ben-Yossef case S_DIN_to_AES: 1333ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CBC && 1334ff27e85aSGilad Ben-Yossef !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1335ff27e85aSGilad Ben-Yossef goto data_size_err; 1336ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) 1337ff27e85aSGilad Ben-Yossef break; 1338ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1339ff27e85aSGilad Ben-Yossef if (areq_ctx->plaintext_authenticate_only) 1340ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1341ff27e85aSGilad Ben-Yossef break; 1342ff27e85aSGilad Ben-Yossef } 1343ff27e85aSGilad Ben-Yossef 1344ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(assoclen, sizeof(u32))) 1345ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1346ff27e85aSGilad Ben-Yossef 1347ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR && 1348ff27e85aSGilad Ben-Yossef !IS_ALIGNED(cipherlen, sizeof(u32))) 1349ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1350ff27e85aSGilad Ben-Yossef 1351ff27e85aSGilad Ben-Yossef break; 1352ff27e85aSGilad Ben-Yossef case S_DIN_to_DES: 1353ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1354ff27e85aSGilad Ben-Yossef goto data_size_err; 1355ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1356ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1357ff27e85aSGilad Ben-Yossef break; 1358ff27e85aSGilad Ben-Yossef default: 1359ff27e85aSGilad Ben-Yossef dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1360ff27e85aSGilad Ben-Yossef goto data_size_err; 1361ff27e85aSGilad Ben-Yossef } 1362ff27e85aSGilad Ben-Yossef 1363ff27e85aSGilad Ben-Yossef return 0; 1364ff27e85aSGilad Ben-Yossef 1365ff27e85aSGilad Ben-Yossef data_size_err: 1366ff27e85aSGilad Ben-Yossef return -EINVAL; 1367ff27e85aSGilad Ben-Yossef } 1368ff27e85aSGilad Ben-Yossef 1369ff27e85aSGilad Ben-Yossef static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1370ff27e85aSGilad Ben-Yossef { 1371ff27e85aSGilad Ben-Yossef unsigned int len = 0; 1372ff27e85aSGilad Ben-Yossef 1373ff27e85aSGilad Ben-Yossef if (header_size == 0) 1374ff27e85aSGilad Ben-Yossef return 0; 1375ff27e85aSGilad Ben-Yossef 1376ff27e85aSGilad Ben-Yossef if (header_size < ((1UL << 16) - (1UL << 8))) { 1377ff27e85aSGilad Ben-Yossef len = 2; 1378ff27e85aSGilad Ben-Yossef 1379ff27e85aSGilad Ben-Yossef pa0_buff[0] = (header_size >> 8) & 0xFF; 1380ff27e85aSGilad Ben-Yossef pa0_buff[1] = header_size & 0xFF; 1381ff27e85aSGilad Ben-Yossef } else { 1382ff27e85aSGilad Ben-Yossef len = 6; 1383ff27e85aSGilad Ben-Yossef 1384ff27e85aSGilad Ben-Yossef pa0_buff[0] = 0xFF; 1385ff27e85aSGilad Ben-Yossef pa0_buff[1] = 0xFE; 1386ff27e85aSGilad Ben-Yossef pa0_buff[2] = (header_size >> 24) & 0xFF; 1387ff27e85aSGilad Ben-Yossef pa0_buff[3] = (header_size >> 16) & 0xFF; 1388ff27e85aSGilad Ben-Yossef pa0_buff[4] = (header_size >> 8) & 0xFF; 1389ff27e85aSGilad Ben-Yossef pa0_buff[5] = header_size & 0xFF; 1390ff27e85aSGilad Ben-Yossef } 1391ff27e85aSGilad Ben-Yossef 1392ff27e85aSGilad Ben-Yossef return len; 1393ff27e85aSGilad Ben-Yossef } 1394ff27e85aSGilad Ben-Yossef 1395ff27e85aSGilad Ben-Yossef static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1396ff27e85aSGilad Ben-Yossef { 1397ff27e85aSGilad Ben-Yossef __be32 data; 1398ff27e85aSGilad Ben-Yossef 1399ff27e85aSGilad Ben-Yossef memset(block, 0, csize); 1400ff27e85aSGilad Ben-Yossef block += csize; 1401ff27e85aSGilad Ben-Yossef 1402ff27e85aSGilad Ben-Yossef if (csize >= 4) 1403ff27e85aSGilad Ben-Yossef csize = 4; 1404ff27e85aSGilad Ben-Yossef else if (msglen > (1 << (8 * csize))) 1405ff27e85aSGilad Ben-Yossef return -EOVERFLOW; 1406ff27e85aSGilad Ben-Yossef 1407ff27e85aSGilad Ben-Yossef data = cpu_to_be32(msglen); 1408ff27e85aSGilad Ben-Yossef memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1409ff27e85aSGilad Ben-Yossef 1410ff27e85aSGilad Ben-Yossef return 0; 1411ff27e85aSGilad Ben-Yossef } 1412ff27e85aSGilad Ben-Yossef 1413ff27e85aSGilad Ben-Yossef static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1414ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1415ff27e85aSGilad Ben-Yossef { 1416ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1417ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1418ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1419ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1420ff27e85aSGilad Ben-Yossef unsigned int cipher_flow_mode; 1421ff27e85aSGilad Ben-Yossef dma_addr_t mac_result; 1422ff27e85aSGilad Ben-Yossef 1423ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1424ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_to_HASH_and_DOUT; 1425ff27e85aSGilad Ben-Yossef mac_result = req_ctx->mac_buf_dma_addr; 1426ff27e85aSGilad Ben-Yossef } else { /* Encrypt */ 1427ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_and_HASH; 1428ff27e85aSGilad Ben-Yossef mac_result = req_ctx->icv_dma_addr; 1429ff27e85aSGilad Ben-Yossef } 1430ff27e85aSGilad Ben-Yossef 1431ff27e85aSGilad Ben-Yossef /* load key */ 1432ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1433ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1434ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1435ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1436ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT); 1437ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1438ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1439ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1440ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1441ff27e85aSGilad Ben-Yossef idx++; 1442ff27e85aSGilad Ben-Yossef 1443ff27e85aSGilad Ben-Yossef /* load ctr state */ 1444ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1445ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1446ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1447ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1448ff27e85aSGilad Ben-Yossef req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1449ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1450ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1451ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1452ff27e85aSGilad Ben-Yossef idx++; 1453ff27e85aSGilad Ben-Yossef 1454ff27e85aSGilad Ben-Yossef /* load MAC key */ 1455ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1456ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1457ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1458ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1459ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT); 1460ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1461ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1462ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1463ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1464ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1465ff27e85aSGilad Ben-Yossef idx++; 1466ff27e85aSGilad Ben-Yossef 1467ff27e85aSGilad Ben-Yossef /* load MAC state */ 1468ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1469ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1470ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1471ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1472ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1473ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1474ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1475ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1476ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1477ff27e85aSGilad Ben-Yossef idx++; 1478ff27e85aSGilad Ben-Yossef 1479ff27e85aSGilad Ben-Yossef /* process assoc data */ 1480da3cf67fSGilad Ben-Yossef if (req_ctx->assoclen > 0) { 1481ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1482ff27e85aSGilad Ben-Yossef } else { 1483ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1484ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1485ff27e85aSGilad Ben-Yossef sg_dma_address(&req_ctx->ccm_adata_sg), 1486ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1487ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 1488ff27e85aSGilad Ben-Yossef idx++; 1489ff27e85aSGilad Ben-Yossef } 1490ff27e85aSGilad Ben-Yossef 1491ff27e85aSGilad Ben-Yossef /* process the cipher */ 1492ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen) 1493ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1494ff27e85aSGilad Ben-Yossef 1495ff27e85aSGilad Ben-Yossef /* Read temporal MAC */ 1496ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1497ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1498ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1499ff27e85aSGilad Ben-Yossef NS_BIT, 0); 1500ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1501ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1502ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1503ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1504ff27e85aSGilad Ben-Yossef idx++; 1505ff27e85aSGilad Ben-Yossef 1506ff27e85aSGilad Ben-Yossef /* load AES-CTR state (for last MAC calculation)*/ 1507ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1508ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1509ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1510ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1511ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1512ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1513ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1514ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1515ff27e85aSGilad Ben-Yossef idx++; 1516ff27e85aSGilad Ben-Yossef 1517ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1518ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1519ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1520ff27e85aSGilad Ben-Yossef idx++; 1521ff27e85aSGilad Ben-Yossef 1522ff27e85aSGilad Ben-Yossef /* encrypt the "T" value and store MAC in mac_state */ 1523ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1524ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1525ff27e85aSGilad Ben-Yossef ctx->authsize, NS_BIT); 1526ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 152727b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 1528ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT); 1529ff27e85aSGilad Ben-Yossef idx++; 1530ff27e85aSGilad Ben-Yossef 1531ff27e85aSGilad Ben-Yossef *seq_size = idx; 1532ff27e85aSGilad Ben-Yossef return 0; 1533ff27e85aSGilad Ben-Yossef } 1534ff27e85aSGilad Ben-Yossef 1535ff27e85aSGilad Ben-Yossef static int config_ccm_adata(struct aead_request *req) 1536ff27e85aSGilad Ben-Yossef { 1537ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1538ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1539ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1540ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1541ff27e85aSGilad Ben-Yossef //unsigned int size_of_a = 0, rem_a_size = 0; 1542ff27e85aSGilad Ben-Yossef unsigned int lp = req->iv[0]; 1543ff27e85aSGilad Ben-Yossef /* Note: The code assume that req->iv[0] already contains the value 1544ff27e85aSGilad Ben-Yossef * of L' of RFC3610 1545ff27e85aSGilad Ben-Yossef */ 1546ff27e85aSGilad Ben-Yossef unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1547ff27e85aSGilad Ben-Yossef unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1548ff27e85aSGilad Ben-Yossef u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1549ff27e85aSGilad Ben-Yossef u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1550ff27e85aSGilad Ben-Yossef u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1551ff27e85aSGilad Ben-Yossef unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1552ff27e85aSGilad Ben-Yossef DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1553ff27e85aSGilad Ben-Yossef req->cryptlen : 1554ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize); 1555ff27e85aSGilad Ben-Yossef int rc; 1556ff27e85aSGilad Ben-Yossef 1557ff27e85aSGilad Ben-Yossef memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1558ff27e85aSGilad Ben-Yossef memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1559ff27e85aSGilad Ben-Yossef 1560ff27e85aSGilad Ben-Yossef /* taken from crypto/ccm.c */ 1561ff27e85aSGilad Ben-Yossef /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1562ff27e85aSGilad Ben-Yossef if (l < 2 || l > 8) { 1563c7b31c88SGilad Ben-Yossef dev_dbg(dev, "illegal iv value %X\n", req->iv[0]); 1564ff27e85aSGilad Ben-Yossef return -EINVAL; 1565ff27e85aSGilad Ben-Yossef } 1566ff27e85aSGilad Ben-Yossef memcpy(b0, req->iv, AES_BLOCK_SIZE); 1567ff27e85aSGilad Ben-Yossef 1568ff27e85aSGilad Ben-Yossef /* format control info per RFC 3610 and 1569ff27e85aSGilad Ben-Yossef * NIST Special Publication 800-38C 1570ff27e85aSGilad Ben-Yossef */ 1571ff27e85aSGilad Ben-Yossef *b0 |= (8 * ((m - 2) / 2)); 1572da3cf67fSGilad Ben-Yossef if (req_ctx->assoclen > 0) 1573ff27e85aSGilad Ben-Yossef *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1574ff27e85aSGilad Ben-Yossef 1575ff27e85aSGilad Ben-Yossef rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1576ff27e85aSGilad Ben-Yossef if (rc) { 1577ff27e85aSGilad Ben-Yossef dev_err(dev, "message len overflow detected"); 1578ff27e85aSGilad Ben-Yossef return rc; 1579ff27e85aSGilad Ben-Yossef } 1580ff27e85aSGilad Ben-Yossef /* END of "taken from crypto/ccm.c" */ 1581ff27e85aSGilad Ben-Yossef 1582ff27e85aSGilad Ben-Yossef /* l(a) - size of associated data. */ 1583da3cf67fSGilad Ben-Yossef req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); 1584ff27e85aSGilad Ben-Yossef 1585ff27e85aSGilad Ben-Yossef memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1586ff27e85aSGilad Ben-Yossef req->iv[15] = 1; 1587ff27e85aSGilad Ben-Yossef 1588ff27e85aSGilad Ben-Yossef memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1589ff27e85aSGilad Ben-Yossef ctr_count_0[15] = 0; 1590ff27e85aSGilad Ben-Yossef 1591ff27e85aSGilad Ben-Yossef return 0; 1592ff27e85aSGilad Ben-Yossef } 1593ff27e85aSGilad Ben-Yossef 1594ff27e85aSGilad Ben-Yossef static void cc_proc_rfc4309_ccm(struct aead_request *req) 1595ff27e85aSGilad Ben-Yossef { 1596ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1597ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1598ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1599ff27e85aSGilad Ben-Yossef 1600ff27e85aSGilad Ben-Yossef /* L' */ 1601ff27e85aSGilad Ben-Yossef memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1602ff27e85aSGilad Ben-Yossef /* For RFC 4309, always use 4 bytes for message length 1603ff27e85aSGilad Ben-Yossef * (at most 2^32-1 bytes). 1604ff27e85aSGilad Ben-Yossef */ 1605ff27e85aSGilad Ben-Yossef areq_ctx->ctr_iv[0] = 3; 1606ff27e85aSGilad Ben-Yossef 1607ff27e85aSGilad Ben-Yossef /* In RFC 4309 there is an 11-bytes nonce+IV part, 1608ff27e85aSGilad Ben-Yossef * that we build here. 1609ff27e85aSGilad Ben-Yossef */ 1610ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1611ff27e85aSGilad Ben-Yossef CCM_BLOCK_NONCE_SIZE); 1612ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1613ff27e85aSGilad Ben-Yossef CCM_BLOCK_IV_SIZE); 1614ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1615da3cf67fSGilad Ben-Yossef areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE; 1616ff27e85aSGilad Ben-Yossef } 1617ff27e85aSGilad Ben-Yossef 1618ff27e85aSGilad Ben-Yossef static void cc_set_ghash_desc(struct aead_request *req, 1619ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 1620ff27e85aSGilad Ben-Yossef { 1621ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1622ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1623ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1624ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1625ff27e85aSGilad Ben-Yossef 1626ff27e85aSGilad Ben-Yossef /* load key to AES*/ 1627ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1628ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1629ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1630ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1631ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT); 1632ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1633ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1634ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1635ff27e85aSGilad Ben-Yossef idx++; 1636ff27e85aSGilad Ben-Yossef 1637ff27e85aSGilad Ben-Yossef /* process one zero block to generate hkey */ 1638ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1639ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1640ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1641ff27e85aSGilad Ben-Yossef NS_BIT, 0); 1642ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT); 1643ff27e85aSGilad Ben-Yossef idx++; 1644ff27e85aSGilad Ben-Yossef 1645ff27e85aSGilad Ben-Yossef /* Memory Barrier */ 1646ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1647ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1648ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1649ff27e85aSGilad Ben-Yossef idx++; 1650ff27e85aSGilad Ben-Yossef 1651ff27e85aSGilad Ben-Yossef /* Load GHASH subkey */ 1652ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1653ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1654ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1655ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1656ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1657ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1658ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1659ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1660ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1661ff27e85aSGilad Ben-Yossef idx++; 1662ff27e85aSGilad Ben-Yossef 1663ff27e85aSGilad Ben-Yossef /* Configure Hash Engine to work with GHASH. 1664ff27e85aSGilad Ben-Yossef * Since it was not possible to extend HASH submodes to add GHASH, 1665ff27e85aSGilad Ben-Yossef * The following command is necessary in order to 1666ff27e85aSGilad Ben-Yossef * select GHASH (according to HW designers) 1667ff27e85aSGilad Ben-Yossef */ 1668ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1669ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1670ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1671ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1672ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1673ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1674ff27e85aSGilad Ben-Yossef set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1675ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1676ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1677ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1678ff27e85aSGilad Ben-Yossef idx++; 1679ff27e85aSGilad Ben-Yossef 1680ff27e85aSGilad Ben-Yossef /* Load GHASH initial STATE (which is 0). (for any hash there is an 1681ff27e85aSGilad Ben-Yossef * initial state) 1682ff27e85aSGilad Ben-Yossef */ 1683ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1684ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1685ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1686ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1687ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1688ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1689ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1690ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1691ff27e85aSGilad Ben-Yossef idx++; 1692ff27e85aSGilad Ben-Yossef 1693ff27e85aSGilad Ben-Yossef *seq_size = idx; 1694ff27e85aSGilad Ben-Yossef } 1695ff27e85aSGilad Ben-Yossef 1696ff27e85aSGilad Ben-Yossef static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1697ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1698ff27e85aSGilad Ben-Yossef { 1699ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1700ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1701ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1702ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1703ff27e85aSGilad Ben-Yossef 1704ff27e85aSGilad Ben-Yossef /* load key to AES*/ 1705ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1706ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1707ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1708ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1709ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT); 1710ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1711ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1712ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1713ff27e85aSGilad Ben-Yossef idx++; 1714ff27e85aSGilad Ben-Yossef 1715ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1716ff27e85aSGilad Ben-Yossef /* load AES/CTR initial CTR value inc by 2*/ 1717ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1718ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1719ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1720ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1721ff27e85aSGilad Ben-Yossef req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1722ff27e85aSGilad Ben-Yossef NS_BIT); 1723ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1724ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1725ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1726ff27e85aSGilad Ben-Yossef idx++; 1727ff27e85aSGilad Ben-Yossef } 1728ff27e85aSGilad Ben-Yossef 1729ff27e85aSGilad Ben-Yossef *seq_size = idx; 1730ff27e85aSGilad Ben-Yossef } 1731ff27e85aSGilad Ben-Yossef 1732ff27e85aSGilad Ben-Yossef static void cc_proc_gcm_result(struct aead_request *req, 1733ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 1734ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1735ff27e85aSGilad Ben-Yossef { 1736ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1737ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1738ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1739ff27e85aSGilad Ben-Yossef dma_addr_t mac_result; 1740ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1741ff27e85aSGilad Ben-Yossef 1742ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1743ff27e85aSGilad Ben-Yossef mac_result = req_ctx->mac_buf_dma_addr; 1744ff27e85aSGilad Ben-Yossef } else { /* Encrypt */ 1745ff27e85aSGilad Ben-Yossef mac_result = req_ctx->icv_dma_addr; 1746ff27e85aSGilad Ben-Yossef } 1747ff27e85aSGilad Ben-Yossef 1748ff27e85aSGilad Ben-Yossef /* process(ghash) gcm_block_len */ 1749ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1750ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1751ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1752ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 1753ff27e85aSGilad Ben-Yossef idx++; 1754ff27e85aSGilad Ben-Yossef 1755ff27e85aSGilad Ben-Yossef /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1756ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1757ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1758ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1759ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1760ff27e85aSGilad Ben-Yossef NS_BIT, 0); 1761ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1762ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1763ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1764ff27e85aSGilad Ben-Yossef 1765ff27e85aSGilad Ben-Yossef idx++; 1766ff27e85aSGilad Ben-Yossef 1767ff27e85aSGilad Ben-Yossef /* load AES/CTR initial CTR value inc by 1*/ 1768ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1769ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1770ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1771ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1772ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1773ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1774ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1775ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1776ff27e85aSGilad Ben-Yossef idx++; 1777ff27e85aSGilad Ben-Yossef 1778ff27e85aSGilad Ben-Yossef /* Memory Barrier */ 1779ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1780ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1781ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1782ff27e85aSGilad Ben-Yossef idx++; 1783ff27e85aSGilad Ben-Yossef 1784ff27e85aSGilad Ben-Yossef /* process GCTR on stored GHASH and store MAC in mac_state*/ 1785ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1786ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1787ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1788ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1789ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 179027b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 1791ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT); 1792ff27e85aSGilad Ben-Yossef idx++; 1793ff27e85aSGilad Ben-Yossef 1794ff27e85aSGilad Ben-Yossef *seq_size = idx; 1795ff27e85aSGilad Ben-Yossef } 1796ff27e85aSGilad Ben-Yossef 1797ff27e85aSGilad Ben-Yossef static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1798ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1799ff27e85aSGilad Ben-Yossef { 1800ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1801ff27e85aSGilad Ben-Yossef unsigned int cipher_flow_mode; 1802ff27e85aSGilad Ben-Yossef 1803ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1804ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_and_HASH; 1805ff27e85aSGilad Ben-Yossef } else { /* Encrypt */ 1806ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_to_HASH_and_DOUT; 1807ff27e85aSGilad Ben-Yossef } 1808ff27e85aSGilad Ben-Yossef 1809ff27e85aSGilad Ben-Yossef //in RFC4543 no data to encrypt. just copy data from src to dest. 1810ff27e85aSGilad Ben-Yossef if (req_ctx->plaintext_authenticate_only) { 1811ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1812ff27e85aSGilad Ben-Yossef cc_set_ghash_desc(req, desc, seq_size); 1813ff27e85aSGilad Ben-Yossef /* process(ghash) assoc data */ 1814ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1815ff27e85aSGilad Ben-Yossef cc_set_gctr_desc(req, desc, seq_size); 1816ff27e85aSGilad Ben-Yossef cc_proc_gcm_result(req, desc, seq_size); 1817ff27e85aSGilad Ben-Yossef return 0; 1818ff27e85aSGilad Ben-Yossef } 1819ff27e85aSGilad Ben-Yossef 1820ff27e85aSGilad Ben-Yossef // for gcm and rfc4106. 1821ff27e85aSGilad Ben-Yossef cc_set_ghash_desc(req, desc, seq_size); 1822ff27e85aSGilad Ben-Yossef /* process(ghash) assoc data */ 1823da3cf67fSGilad Ben-Yossef if (req_ctx->assoclen > 0) 1824ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1825ff27e85aSGilad Ben-Yossef cc_set_gctr_desc(req, desc, seq_size); 1826ff27e85aSGilad Ben-Yossef /* process(gctr+ghash) */ 1827ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen) 1828ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1829ff27e85aSGilad Ben-Yossef cc_proc_gcm_result(req, desc, seq_size); 1830ff27e85aSGilad Ben-Yossef 1831ff27e85aSGilad Ben-Yossef return 0; 1832ff27e85aSGilad Ben-Yossef } 1833ff27e85aSGilad Ben-Yossef 1834ff27e85aSGilad Ben-Yossef static int config_gcm_context(struct aead_request *req) 1835ff27e85aSGilad Ben-Yossef { 1836ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1837ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1838ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1839ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1840ff27e85aSGilad Ben-Yossef 1841ff27e85aSGilad Ben-Yossef unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1842ff27e85aSGilad Ben-Yossef DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1843ff27e85aSGilad Ben-Yossef req->cryptlen : 1844ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize); 1845ff27e85aSGilad Ben-Yossef __be32 counter = cpu_to_be32(2); 1846ff27e85aSGilad Ben-Yossef 1847da3cf67fSGilad Ben-Yossef dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", 1848da3cf67fSGilad Ben-Yossef __func__, cryptlen, req_ctx->assoclen, ctx->authsize); 1849ff27e85aSGilad Ben-Yossef 1850ff27e85aSGilad Ben-Yossef memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1851ff27e85aSGilad Ben-Yossef 1852ff27e85aSGilad Ben-Yossef memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1853ff27e85aSGilad Ben-Yossef 1854ff27e85aSGilad Ben-Yossef memcpy(req->iv + 12, &counter, 4); 1855ff27e85aSGilad Ben-Yossef memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1856ff27e85aSGilad Ben-Yossef 1857ff27e85aSGilad Ben-Yossef counter = cpu_to_be32(1); 1858ff27e85aSGilad Ben-Yossef memcpy(req->iv + 12, &counter, 4); 1859ff27e85aSGilad Ben-Yossef memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1860ff27e85aSGilad Ben-Yossef 1861ff27e85aSGilad Ben-Yossef if (!req_ctx->plaintext_authenticate_only) { 1862ff27e85aSGilad Ben-Yossef __be64 temp64; 1863ff27e85aSGilad Ben-Yossef 1864da3cf67fSGilad Ben-Yossef temp64 = cpu_to_be64(req_ctx->assoclen * 8); 1865ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1866ff27e85aSGilad Ben-Yossef temp64 = cpu_to_be64(cryptlen * 8); 1867ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1868ff27e85aSGilad Ben-Yossef } else { 1869ff27e85aSGilad Ben-Yossef /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1870ff27e85aSGilad Ben-Yossef * data that is nothing is encrypted. 1871ff27e85aSGilad Ben-Yossef */ 1872ff27e85aSGilad Ben-Yossef __be64 temp64; 1873ff27e85aSGilad Ben-Yossef 1874da3cf67fSGilad Ben-Yossef temp64 = cpu_to_be64((req_ctx->assoclen + 1875da3cf67fSGilad Ben-Yossef GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8); 1876ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1877ff27e85aSGilad Ben-Yossef temp64 = 0; 1878ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1879ff27e85aSGilad Ben-Yossef } 1880ff27e85aSGilad Ben-Yossef 1881ff27e85aSGilad Ben-Yossef return 0; 1882ff27e85aSGilad Ben-Yossef } 1883ff27e85aSGilad Ben-Yossef 1884ff27e85aSGilad Ben-Yossef static void cc_proc_rfc4_gcm(struct aead_request *req) 1885ff27e85aSGilad Ben-Yossef { 1886ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1887ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1888ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1889ff27e85aSGilad Ben-Yossef 1890ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1891ff27e85aSGilad Ben-Yossef ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1892ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1893ff27e85aSGilad Ben-Yossef GCM_BLOCK_RFC4_IV_SIZE); 1894ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1895da3cf67fSGilad Ben-Yossef areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; 1896ff27e85aSGilad Ben-Yossef } 1897ff27e85aSGilad Ben-Yossef 1898ff27e85aSGilad Ben-Yossef static int cc_proc_aead(struct aead_request *req, 1899ff27e85aSGilad Ben-Yossef enum drv_crypto_direction direct) 1900ff27e85aSGilad Ben-Yossef { 1901ff27e85aSGilad Ben-Yossef int rc = 0; 1902ff27e85aSGilad Ben-Yossef int seq_len = 0; 1903ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1904ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1905ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1906ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1907ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1908ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 1909ff27e85aSGilad Ben-Yossef 1910ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1911ff27e85aSGilad Ben-Yossef ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1912ff27e85aSGilad Ben-Yossef ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1913ff27e85aSGilad Ben-Yossef sg_virt(req->dst), req->dst->offset, req->cryptlen); 1914ff27e85aSGilad Ben-Yossef 1915ff27e85aSGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 1916ff27e85aSGilad Ben-Yossef 1917ff27e85aSGilad Ben-Yossef /* Check data length according to mode */ 1918ff27e85aSGilad Ben-Yossef if (validate_data_size(ctx, direct, req)) { 1919ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1920da3cf67fSGilad Ben-Yossef req->cryptlen, areq_ctx->assoclen); 1921ff27e85aSGilad Ben-Yossef return -EINVAL; 1922ff27e85aSGilad Ben-Yossef } 1923ff27e85aSGilad Ben-Yossef 1924ff27e85aSGilad Ben-Yossef /* Setup request structure */ 1925f4274eecSGeert Uytterhoeven cc_req.user_cb = cc_aead_complete; 1926f4274eecSGeert Uytterhoeven cc_req.user_arg = req; 1927ff27e85aSGilad Ben-Yossef 1928ff27e85aSGilad Ben-Yossef /* Setup request context */ 1929ff27e85aSGilad Ben-Yossef areq_ctx->gen_ctx.op_type = direct; 1930ff27e85aSGilad Ben-Yossef areq_ctx->req_authsize = ctx->authsize; 1931ff27e85aSGilad Ben-Yossef areq_ctx->cipher_mode = ctx->cipher_mode; 1932ff27e85aSGilad Ben-Yossef 1933ff27e85aSGilad Ben-Yossef /* STAT_PHASE_1: Map buffers */ 1934ff27e85aSGilad Ben-Yossef 1935ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1936ff27e85aSGilad Ben-Yossef /* Build CTR IV - Copy nonce from last 4 bytes in 1937ff27e85aSGilad Ben-Yossef * CTR key to first 4 bytes in CTR IV 1938ff27e85aSGilad Ben-Yossef */ 1939ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1940ff27e85aSGilad Ben-Yossef CTR_RFC3686_NONCE_SIZE); 1941e6e6600cSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1942e6e6600cSGilad Ben-Yossef CTR_RFC3686_IV_SIZE); 1943ff27e85aSGilad Ben-Yossef /* Initialize counter portion of counter block */ 1944ff27e85aSGilad Ben-Yossef *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1945ff27e85aSGilad Ben-Yossef CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1946ff27e85aSGilad Ben-Yossef 1947ff27e85aSGilad Ben-Yossef /* Replace with counter iv */ 1948ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1949ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1950ff27e85aSGilad Ben-Yossef } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1951ff27e85aSGilad Ben-Yossef (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1952ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1953ff27e85aSGilad Ben-Yossef if (areq_ctx->ctr_iv != req->iv) { 1954ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv, req->iv, 1955ff27e85aSGilad Ben-Yossef crypto_aead_ivsize(tfm)); 1956ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1957ff27e85aSGilad Ben-Yossef } 1958ff27e85aSGilad Ben-Yossef } else { 1959ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1960ff27e85aSGilad Ben-Yossef } 1961ff27e85aSGilad Ben-Yossef 1962ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1963ff27e85aSGilad Ben-Yossef rc = config_ccm_adata(req); 1964ff27e85aSGilad Ben-Yossef if (rc) { 1965ff27e85aSGilad Ben-Yossef dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1966ff27e85aSGilad Ben-Yossef rc); 1967ff27e85aSGilad Ben-Yossef goto exit; 1968ff27e85aSGilad Ben-Yossef } 1969ff27e85aSGilad Ben-Yossef } else { 1970ff27e85aSGilad Ben-Yossef areq_ctx->ccm_hdr_size = ccm_header_size_null; 1971ff27e85aSGilad Ben-Yossef } 1972ff27e85aSGilad Ben-Yossef 1973ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1974ff27e85aSGilad Ben-Yossef rc = config_gcm_context(req); 1975ff27e85aSGilad Ben-Yossef if (rc) { 1976ff27e85aSGilad Ben-Yossef dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 1977ff27e85aSGilad Ben-Yossef rc); 1978ff27e85aSGilad Ben-Yossef goto exit; 1979ff27e85aSGilad Ben-Yossef } 1980ff27e85aSGilad Ben-Yossef } 1981ff27e85aSGilad Ben-Yossef 1982ff27e85aSGilad Ben-Yossef rc = cc_map_aead_request(ctx->drvdata, req); 1983ff27e85aSGilad Ben-Yossef if (rc) { 1984ff27e85aSGilad Ben-Yossef dev_err(dev, "map_request() failed\n"); 1985ff27e85aSGilad Ben-Yossef goto exit; 1986ff27e85aSGilad Ben-Yossef } 1987ff27e85aSGilad Ben-Yossef 1988ff27e85aSGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */ 1989ff27e85aSGilad Ben-Yossef 1990ff27e85aSGilad Ben-Yossef /* Load MLLI tables to SRAM if necessary */ 1991ff27e85aSGilad Ben-Yossef cc_mlli_to_sram(req, desc, &seq_len); 1992ff27e85aSGilad Ben-Yossef 1993ff27e85aSGilad Ben-Yossef /*TODO: move seq len by reference */ 1994ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { 1995ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 1996ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 1997ff27e85aSGilad Ben-Yossef cc_hmac_authenc(req, desc, &seq_len); 1998ff27e85aSGilad Ben-Yossef break; 1999ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC: 2000ff27e85aSGilad Ben-Yossef cc_xcbc_authenc(req, desc, &seq_len); 2001ff27e85aSGilad Ben-Yossef break; 2002ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: 2003ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) 2004ff27e85aSGilad Ben-Yossef cc_ccm(req, desc, &seq_len); 2005ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) 2006ff27e85aSGilad Ben-Yossef cc_gcm(req, desc, &seq_len); 2007ff27e85aSGilad Ben-Yossef break; 2008ff27e85aSGilad Ben-Yossef default: 2009ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2010ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, req); 2011ff27e85aSGilad Ben-Yossef rc = -ENOTSUPP; 2012ff27e85aSGilad Ben-Yossef goto exit; 2013ff27e85aSGilad Ben-Yossef } 2014ff27e85aSGilad Ben-Yossef 2015ff27e85aSGilad Ben-Yossef /* STAT_PHASE_3: Lock HW and push sequence */ 2016ff27e85aSGilad Ben-Yossef 2017ff27e85aSGilad Ben-Yossef rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2018ff27e85aSGilad Ben-Yossef 2019ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) { 2020ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2021ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, req); 2022ff27e85aSGilad Ben-Yossef } 2023ff27e85aSGilad Ben-Yossef 2024ff27e85aSGilad Ben-Yossef exit: 2025ff27e85aSGilad Ben-Yossef return rc; 2026ff27e85aSGilad Ben-Yossef } 2027ff27e85aSGilad Ben-Yossef 2028ff27e85aSGilad Ben-Yossef static int cc_aead_encrypt(struct aead_request *req) 2029ff27e85aSGilad Ben-Yossef { 2030ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2031ff27e85aSGilad Ben-Yossef int rc; 2032ff27e85aSGilad Ben-Yossef 20339f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 20349f31eb6eSGilad Ben-Yossef 2035ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2036ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2037da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2038ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = false; 2039ff27e85aSGilad Ben-Yossef 2040ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2041ff27e85aSGilad Ben-Yossef 2042ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2043ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2044ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2045ff27e85aSGilad Ben-Yossef 2046ff27e85aSGilad Ben-Yossef return rc; 2047ff27e85aSGilad Ben-Yossef } 2048ff27e85aSGilad Ben-Yossef 2049ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2050ff27e85aSGilad Ben-Yossef { 2051ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */ 2052ff27e85aSGilad Ben-Yossef 2053ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2054ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2055ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2056ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2057ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2058ff27e85aSGilad Ben-Yossef 2059ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2060c7b31c88SGilad Ben-Yossef dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2061ff27e85aSGilad Ben-Yossef goto out; 2062ff27e85aSGilad Ben-Yossef } 2063ff27e85aSGilad Ben-Yossef 20649f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 20659f31eb6eSGilad Ben-Yossef 2066ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2067ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2068da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2069ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2070ff27e85aSGilad Ben-Yossef 2071ff27e85aSGilad Ben-Yossef cc_proc_rfc4309_ccm(req); 2072ff27e85aSGilad Ben-Yossef 2073ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2074ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2075ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2076ff27e85aSGilad Ben-Yossef out: 2077ff27e85aSGilad Ben-Yossef return rc; 2078ff27e85aSGilad Ben-Yossef } 2079ff27e85aSGilad Ben-Yossef 2080ff27e85aSGilad Ben-Yossef static int cc_aead_decrypt(struct aead_request *req) 2081ff27e85aSGilad Ben-Yossef { 2082ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2083ff27e85aSGilad Ben-Yossef int rc; 2084ff27e85aSGilad Ben-Yossef 20859f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 20869f31eb6eSGilad Ben-Yossef 2087ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2088ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2089da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2090ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = false; 2091ff27e85aSGilad Ben-Yossef 2092ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2093ff27e85aSGilad Ben-Yossef 2094ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2095ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2096ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2097ff27e85aSGilad Ben-Yossef 2098ff27e85aSGilad Ben-Yossef return rc; 2099ff27e85aSGilad Ben-Yossef } 2100ff27e85aSGilad Ben-Yossef 2101ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2102ff27e85aSGilad Ben-Yossef { 2103ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2104ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2105ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2106ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2107ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2108ff27e85aSGilad Ben-Yossef 2109ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2110c7b31c88SGilad Ben-Yossef dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2111ff27e85aSGilad Ben-Yossef goto out; 2112ff27e85aSGilad Ben-Yossef } 2113ff27e85aSGilad Ben-Yossef 21149f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 21159f31eb6eSGilad Ben-Yossef 2116ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2117ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2118da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2119ff27e85aSGilad Ben-Yossef 2120ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2121ff27e85aSGilad Ben-Yossef cc_proc_rfc4309_ccm(req); 2122ff27e85aSGilad Ben-Yossef 2123ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2124ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2125ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2126ff27e85aSGilad Ben-Yossef 2127ff27e85aSGilad Ben-Yossef out: 2128ff27e85aSGilad Ben-Yossef return rc; 2129ff27e85aSGilad Ben-Yossef } 2130ff27e85aSGilad Ben-Yossef 2131ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2132ff27e85aSGilad Ben-Yossef unsigned int keylen) 2133ff27e85aSGilad Ben-Yossef { 2134ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2135ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2136ff27e85aSGilad Ben-Yossef 2137ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2138ff27e85aSGilad Ben-Yossef 2139ff27e85aSGilad Ben-Yossef if (keylen < 4) 2140ff27e85aSGilad Ben-Yossef return -EINVAL; 2141ff27e85aSGilad Ben-Yossef 2142ff27e85aSGilad Ben-Yossef keylen -= 4; 2143ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 4); 2144ff27e85aSGilad Ben-Yossef 2145ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen); 2146ff27e85aSGilad Ben-Yossef } 2147ff27e85aSGilad Ben-Yossef 2148ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2149ff27e85aSGilad Ben-Yossef unsigned int keylen) 2150ff27e85aSGilad Ben-Yossef { 2151ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2152ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2153ff27e85aSGilad Ben-Yossef 2154ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2155ff27e85aSGilad Ben-Yossef 2156ff27e85aSGilad Ben-Yossef if (keylen < 4) 2157ff27e85aSGilad Ben-Yossef return -EINVAL; 2158ff27e85aSGilad Ben-Yossef 2159ff27e85aSGilad Ben-Yossef keylen -= 4; 2160ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 4); 2161ff27e85aSGilad Ben-Yossef 2162ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen); 2163ff27e85aSGilad Ben-Yossef } 2164ff27e85aSGilad Ben-Yossef 2165ff27e85aSGilad Ben-Yossef static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2166ff27e85aSGilad Ben-Yossef unsigned int authsize) 2167ff27e85aSGilad Ben-Yossef { 2168ff27e85aSGilad Ben-Yossef switch (authsize) { 2169ff27e85aSGilad Ben-Yossef case 4: 2170ff27e85aSGilad Ben-Yossef case 8: 2171ff27e85aSGilad Ben-Yossef case 12: 2172ff27e85aSGilad Ben-Yossef case 13: 2173ff27e85aSGilad Ben-Yossef case 14: 2174ff27e85aSGilad Ben-Yossef case 15: 2175ff27e85aSGilad Ben-Yossef case 16: 2176ff27e85aSGilad Ben-Yossef break; 2177ff27e85aSGilad Ben-Yossef default: 2178ff27e85aSGilad Ben-Yossef return -EINVAL; 2179ff27e85aSGilad Ben-Yossef } 2180ff27e85aSGilad Ben-Yossef 2181ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 2182ff27e85aSGilad Ben-Yossef } 2183ff27e85aSGilad Ben-Yossef 2184ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2185ff27e85aSGilad Ben-Yossef unsigned int authsize) 2186ff27e85aSGilad Ben-Yossef { 2187ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2188ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2189ff27e85aSGilad Ben-Yossef 2190ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authsize %d\n", authsize); 2191ff27e85aSGilad Ben-Yossef 2192ff27e85aSGilad Ben-Yossef switch (authsize) { 2193ff27e85aSGilad Ben-Yossef case 8: 2194ff27e85aSGilad Ben-Yossef case 12: 2195ff27e85aSGilad Ben-Yossef case 16: 2196ff27e85aSGilad Ben-Yossef break; 2197ff27e85aSGilad Ben-Yossef default: 2198ff27e85aSGilad Ben-Yossef return -EINVAL; 2199ff27e85aSGilad Ben-Yossef } 2200ff27e85aSGilad Ben-Yossef 2201ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 2202ff27e85aSGilad Ben-Yossef } 2203ff27e85aSGilad Ben-Yossef 2204ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2205ff27e85aSGilad Ben-Yossef unsigned int authsize) 2206ff27e85aSGilad Ben-Yossef { 2207ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2208ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2209ff27e85aSGilad Ben-Yossef 2210ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authsize %d\n", authsize); 2211ff27e85aSGilad Ben-Yossef 2212ff27e85aSGilad Ben-Yossef if (authsize != 16) 2213ff27e85aSGilad Ben-Yossef return -EINVAL; 2214ff27e85aSGilad Ben-Yossef 2215ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 2216ff27e85aSGilad Ben-Yossef } 2217ff27e85aSGilad Ben-Yossef 2218ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2219ff27e85aSGilad Ben-Yossef { 2220ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */ 2221ff27e85aSGilad Ben-Yossef 2222ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2223ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2224ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2225ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2226ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2227ff27e85aSGilad Ben-Yossef 2228ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2229c7b31c88SGilad Ben-Yossef dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2230ff27e85aSGilad Ben-Yossef goto out; 2231ff27e85aSGilad Ben-Yossef } 2232ff27e85aSGilad Ben-Yossef 22339f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 22349f31eb6eSGilad Ben-Yossef 2235ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2236ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2237da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2238ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2239ff27e85aSGilad Ben-Yossef 2240ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2241ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2242ff27e85aSGilad Ben-Yossef 2243ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2244ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2245ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2246ff27e85aSGilad Ben-Yossef out: 2247ff27e85aSGilad Ben-Yossef return rc; 2248ff27e85aSGilad Ben-Yossef } 2249ff27e85aSGilad Ben-Yossef 2250ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2251ff27e85aSGilad Ben-Yossef { 2252ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */ 2253b93ecf42SIuliana Prodan struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2254b93ecf42SIuliana Prodan struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2255b93ecf42SIuliana Prodan struct device *dev = drvdata_to_dev(ctx->drvdata); 2256ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2257b93ecf42SIuliana Prodan int rc = -EINVAL; 2258b93ecf42SIuliana Prodan 2259b93ecf42SIuliana Prodan if (!valid_assoclen(req)) { 2260c7b31c88SGilad Ben-Yossef dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2261b93ecf42SIuliana Prodan goto out; 2262b93ecf42SIuliana Prodan } 2263ff27e85aSGilad Ben-Yossef 22649f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 22659f31eb6eSGilad Ben-Yossef 2266ff27e85aSGilad Ben-Yossef //plaintext is not encryped with rfc4543 2267ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = true; 2268ff27e85aSGilad Ben-Yossef 2269ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2270ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2271da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2272ff27e85aSGilad Ben-Yossef 2273ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2274ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2275ff27e85aSGilad Ben-Yossef 2276ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2277ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2278ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2279b93ecf42SIuliana Prodan out: 2280ff27e85aSGilad Ben-Yossef return rc; 2281ff27e85aSGilad Ben-Yossef } 2282ff27e85aSGilad Ben-Yossef 2283ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2284ff27e85aSGilad Ben-Yossef { 2285ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_decrypt() above. */ 2286ff27e85aSGilad Ben-Yossef 2287ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2288ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2289ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2290ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2291ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2292ff27e85aSGilad Ben-Yossef 2293ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2294c7b31c88SGilad Ben-Yossef dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2295ff27e85aSGilad Ben-Yossef goto out; 2296ff27e85aSGilad Ben-Yossef } 2297ff27e85aSGilad Ben-Yossef 22989f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 22999f31eb6eSGilad Ben-Yossef 2300ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2301ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2302da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2303ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2304ff27e85aSGilad Ben-Yossef 2305ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2306ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2307ff27e85aSGilad Ben-Yossef 2308ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2309ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2310ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2311ff27e85aSGilad Ben-Yossef out: 2312ff27e85aSGilad Ben-Yossef return rc; 2313ff27e85aSGilad Ben-Yossef } 2314ff27e85aSGilad Ben-Yossef 2315ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2316ff27e85aSGilad Ben-Yossef { 2317ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_decrypt() above. */ 2318b93ecf42SIuliana Prodan struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2319b93ecf42SIuliana Prodan struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2320b93ecf42SIuliana Prodan struct device *dev = drvdata_to_dev(ctx->drvdata); 2321ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2322b93ecf42SIuliana Prodan int rc = -EINVAL; 2323b93ecf42SIuliana Prodan 2324b93ecf42SIuliana Prodan if (!valid_assoclen(req)) { 2325c7b31c88SGilad Ben-Yossef dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2326b93ecf42SIuliana Prodan goto out; 2327b93ecf42SIuliana Prodan } 2328ff27e85aSGilad Ben-Yossef 23299f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 23309f31eb6eSGilad Ben-Yossef 2331ff27e85aSGilad Ben-Yossef //plaintext is not decryped with rfc4543 2332ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = true; 2333ff27e85aSGilad Ben-Yossef 2334ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2335ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2336da3cf67fSGilad Ben-Yossef areq_ctx->assoclen = req->assoclen; 2337ff27e85aSGilad Ben-Yossef 2338ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2339ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2340ff27e85aSGilad Ben-Yossef 2341ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2342ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2343ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2344b93ecf42SIuliana Prodan out: 2345ff27e85aSGilad Ben-Yossef return rc; 2346ff27e85aSGilad Ben-Yossef } 2347ff27e85aSGilad Ben-Yossef 2348ff27e85aSGilad Ben-Yossef /* aead alg */ 2349ff27e85aSGilad Ben-Yossef static struct cc_alg_template aead_algs[] = { 2350ff27e85aSGilad Ben-Yossef { 2351ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),cbc(aes))", 2352ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2353ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 2354ff27e85aSGilad Ben-Yossef .template_aead = { 2355ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2356ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2357ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2358ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2359ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2360ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2361ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2362ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE, 2363ff27e85aSGilad Ben-Yossef }, 2364ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2365ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2366ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1, 236727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 23681c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2369ff27e85aSGilad Ben-Yossef }, 2370ff27e85aSGilad Ben-Yossef { 2371ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),cbc(des3_ede))", 2372ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2373ff27e85aSGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE, 2374ff27e85aSGilad Ben-Yossef .template_aead = { 23759fbfcefcSHerbert Xu .setkey = cc_des3_aead_setkey, 2376ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2377ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2378ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2379ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2380ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2381ff27e85aSGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE, 2382ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE, 2383ff27e85aSGilad Ben-Yossef }, 2384ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2385ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 2386ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1, 238727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 23881c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2389ff27e85aSGilad Ben-Yossef }, 2390ff27e85aSGilad Ben-Yossef { 2391ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),cbc(aes))", 2392ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2393ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 2394ff27e85aSGilad Ben-Yossef .template_aead = { 2395ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2396ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2397ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2398ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2399ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2400ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2401ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2402ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE, 2403ff27e85aSGilad Ben-Yossef }, 2404ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2405ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2406ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256, 240727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24081c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2409ff27e85aSGilad Ben-Yossef }, 2410ff27e85aSGilad Ben-Yossef { 2411ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),cbc(des3_ede))", 2412ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2413ff27e85aSGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE, 2414ff27e85aSGilad Ben-Yossef .template_aead = { 24159fbfcefcSHerbert Xu .setkey = cc_des3_aead_setkey, 2416ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2417ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2418ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2419ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2420ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2421ff27e85aSGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE, 2422ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE, 2423ff27e85aSGilad Ben-Yossef }, 2424ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2425ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 2426ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256, 242727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24281c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2429ff27e85aSGilad Ben-Yossef }, 2430ff27e85aSGilad Ben-Yossef { 2431ff27e85aSGilad Ben-Yossef .name = "authenc(xcbc(aes),cbc(aes))", 2432ff27e85aSGilad Ben-Yossef .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2433ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 2434ff27e85aSGilad Ben-Yossef .template_aead = { 2435ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2436ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2437ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2438ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2439ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2440ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2441ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2442ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2443ff27e85aSGilad Ben-Yossef }, 2444ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2445ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2446ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_XCBC_MAC, 244727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24481c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2449ff27e85aSGilad Ben-Yossef }, 2450ff27e85aSGilad Ben-Yossef { 2451ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2452ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2453ff27e85aSGilad Ben-Yossef .blocksize = 1, 2454ff27e85aSGilad Ben-Yossef .template_aead = { 2455ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2456ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2457ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2458ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2459ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2460ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2461ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE, 2462ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE, 2463ff27e85aSGilad Ben-Yossef }, 2464ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 2465ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2466ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1, 246727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24681c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2469ff27e85aSGilad Ben-Yossef }, 2470ff27e85aSGilad Ben-Yossef { 2471ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2472ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2473ff27e85aSGilad Ben-Yossef .blocksize = 1, 2474ff27e85aSGilad Ben-Yossef .template_aead = { 2475ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2476ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2477ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2478ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2479ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2480ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2481ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE, 2482ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE, 2483ff27e85aSGilad Ben-Yossef }, 2484ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 2485ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2486ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256, 248727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24881c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2489ff27e85aSGilad Ben-Yossef }, 2490ff27e85aSGilad Ben-Yossef { 2491ff27e85aSGilad Ben-Yossef .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2492ff27e85aSGilad Ben-Yossef .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2493ff27e85aSGilad Ben-Yossef .blocksize = 1, 2494ff27e85aSGilad Ben-Yossef .template_aead = { 2495ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2496ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2497ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2498ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2499ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2500ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2501ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE, 2502ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2503ff27e85aSGilad Ben-Yossef }, 2504ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 2505ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2506ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_XCBC_MAC, 250727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25081c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2509ff27e85aSGilad Ben-Yossef }, 2510ff27e85aSGilad Ben-Yossef { 2511ff27e85aSGilad Ben-Yossef .name = "ccm(aes)", 2512ff27e85aSGilad Ben-Yossef .driver_name = "ccm-aes-ccree", 2513ff27e85aSGilad Ben-Yossef .blocksize = 1, 2514ff27e85aSGilad Ben-Yossef .template_aead = { 2515ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2516ff27e85aSGilad Ben-Yossef .setauthsize = cc_ccm_setauthsize, 2517ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2518ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2519ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2520ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2521ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2522ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2523ff27e85aSGilad Ben-Yossef }, 2524ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CCM, 2525ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2526ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 252727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25281c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2529ff27e85aSGilad Ben-Yossef }, 2530ff27e85aSGilad Ben-Yossef { 2531ff27e85aSGilad Ben-Yossef .name = "rfc4309(ccm(aes))", 2532ff27e85aSGilad Ben-Yossef .driver_name = "rfc4309-ccm-aes-ccree", 2533ff27e85aSGilad Ben-Yossef .blocksize = 1, 2534ff27e85aSGilad Ben-Yossef .template_aead = { 2535ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4309_ccm_setkey, 2536ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4309_ccm_setauthsize, 2537ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4309_ccm_encrypt, 2538ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4309_ccm_decrypt, 2539ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2540ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2541ff27e85aSGilad Ben-Yossef .ivsize = CCM_BLOCK_IV_SIZE, 2542ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2543ff27e85aSGilad Ben-Yossef }, 2544ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CCM, 2545ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2546ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 254727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25481c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2549ff27e85aSGilad Ben-Yossef }, 2550ff27e85aSGilad Ben-Yossef { 2551ff27e85aSGilad Ben-Yossef .name = "gcm(aes)", 2552ff27e85aSGilad Ben-Yossef .driver_name = "gcm-aes-ccree", 2553ff27e85aSGilad Ben-Yossef .blocksize = 1, 2554ff27e85aSGilad Ben-Yossef .template_aead = { 2555ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2556ff27e85aSGilad Ben-Yossef .setauthsize = cc_gcm_setauthsize, 2557ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2558ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2559ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2560ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2561ff27e85aSGilad Ben-Yossef .ivsize = 12, 2562ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2563ff27e85aSGilad Ben-Yossef }, 2564ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR, 2565ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2566ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 256727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25681c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2569ff27e85aSGilad Ben-Yossef }, 2570ff27e85aSGilad Ben-Yossef { 2571ff27e85aSGilad Ben-Yossef .name = "rfc4106(gcm(aes))", 2572ff27e85aSGilad Ben-Yossef .driver_name = "rfc4106-gcm-aes-ccree", 2573ff27e85aSGilad Ben-Yossef .blocksize = 1, 2574ff27e85aSGilad Ben-Yossef .template_aead = { 2575ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4106_gcm_setkey, 2576ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4106_gcm_setauthsize, 2577ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4106_gcm_encrypt, 2578ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4106_gcm_decrypt, 2579ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2580ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2581ff27e85aSGilad Ben-Yossef .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2582ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2583ff27e85aSGilad Ben-Yossef }, 2584ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR, 2585ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2586ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 258727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25881c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2589ff27e85aSGilad Ben-Yossef }, 2590ff27e85aSGilad Ben-Yossef { 2591ff27e85aSGilad Ben-Yossef .name = "rfc4543(gcm(aes))", 2592ff27e85aSGilad Ben-Yossef .driver_name = "rfc4543-gcm-aes-ccree", 2593ff27e85aSGilad Ben-Yossef .blocksize = 1, 2594ff27e85aSGilad Ben-Yossef .template_aead = { 2595ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4543_gcm_setkey, 2596ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4543_gcm_setauthsize, 2597ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4543_gcm_encrypt, 2598ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4543_gcm_decrypt, 2599ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2600ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2601ff27e85aSGilad Ben-Yossef .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2602ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2603ff27e85aSGilad Ben-Yossef }, 2604ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR, 2605ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2606ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 260727b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 26081c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2609ff27e85aSGilad Ben-Yossef }, 2610ff27e85aSGilad Ben-Yossef }; 2611ff27e85aSGilad Ben-Yossef 2612ff27e85aSGilad Ben-Yossef static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2613ff27e85aSGilad Ben-Yossef struct device *dev) 2614ff27e85aSGilad Ben-Yossef { 2615ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg; 2616ff27e85aSGilad Ben-Yossef struct aead_alg *alg; 2617ff27e85aSGilad Ben-Yossef 2618ff27e85aSGilad Ben-Yossef t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2619ff27e85aSGilad Ben-Yossef if (!t_alg) 2620ff27e85aSGilad Ben-Yossef return ERR_PTR(-ENOMEM); 2621ff27e85aSGilad Ben-Yossef 2622ff27e85aSGilad Ben-Yossef alg = &tmpl->template_aead; 2623ff27e85aSGilad Ben-Yossef 2624ff27e85aSGilad Ben-Yossef snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 2625ff27e85aSGilad Ben-Yossef snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2626ff27e85aSGilad Ben-Yossef tmpl->driver_name); 2627ff27e85aSGilad Ben-Yossef alg->base.cra_module = THIS_MODULE; 2628ff27e85aSGilad Ben-Yossef alg->base.cra_priority = CC_CRA_PRIO; 2629ff27e85aSGilad Ben-Yossef 2630ff27e85aSGilad Ben-Yossef alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 263176c9e53eSGilad Ben-Yossef alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 263221f802ccSGilad Ben-Yossef alg->base.cra_blocksize = tmpl->blocksize; 2633ff27e85aSGilad Ben-Yossef alg->init = cc_aead_init; 2634ff27e85aSGilad Ben-Yossef alg->exit = cc_aead_exit; 2635ff27e85aSGilad Ben-Yossef 2636ff27e85aSGilad Ben-Yossef t_alg->aead_alg = *alg; 2637ff27e85aSGilad Ben-Yossef 2638ff27e85aSGilad Ben-Yossef t_alg->cipher_mode = tmpl->cipher_mode; 2639ff27e85aSGilad Ben-Yossef t_alg->flow_mode = tmpl->flow_mode; 2640ff27e85aSGilad Ben-Yossef t_alg->auth_mode = tmpl->auth_mode; 2641ff27e85aSGilad Ben-Yossef 2642ff27e85aSGilad Ben-Yossef return t_alg; 2643ff27e85aSGilad Ben-Yossef } 2644ff27e85aSGilad Ben-Yossef 2645ff27e85aSGilad Ben-Yossef int cc_aead_free(struct cc_drvdata *drvdata) 2646ff27e85aSGilad Ben-Yossef { 2647ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg, *n; 2648ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle = 2649ff27e85aSGilad Ben-Yossef (struct cc_aead_handle *)drvdata->aead_handle; 2650ff27e85aSGilad Ben-Yossef 2651ff27e85aSGilad Ben-Yossef if (aead_handle) { 2652ff27e85aSGilad Ben-Yossef /* Remove registered algs */ 2653ff27e85aSGilad Ben-Yossef list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, 2654ff27e85aSGilad Ben-Yossef entry) { 2655ff27e85aSGilad Ben-Yossef crypto_unregister_aead(&t_alg->aead_alg); 2656ff27e85aSGilad Ben-Yossef list_del(&t_alg->entry); 2657ff27e85aSGilad Ben-Yossef kfree(t_alg); 2658ff27e85aSGilad Ben-Yossef } 2659ff27e85aSGilad Ben-Yossef kfree(aead_handle); 2660ff27e85aSGilad Ben-Yossef drvdata->aead_handle = NULL; 2661ff27e85aSGilad Ben-Yossef } 2662ff27e85aSGilad Ben-Yossef 2663ff27e85aSGilad Ben-Yossef return 0; 2664ff27e85aSGilad Ben-Yossef } 2665ff27e85aSGilad Ben-Yossef 2666ff27e85aSGilad Ben-Yossef int cc_aead_alloc(struct cc_drvdata *drvdata) 2667ff27e85aSGilad Ben-Yossef { 2668ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle; 2669ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg; 2670ff27e85aSGilad Ben-Yossef int rc = -ENOMEM; 2671ff27e85aSGilad Ben-Yossef int alg; 2672ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(drvdata); 2673ff27e85aSGilad Ben-Yossef 2674ff27e85aSGilad Ben-Yossef aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL); 2675ff27e85aSGilad Ben-Yossef if (!aead_handle) { 2676ff27e85aSGilad Ben-Yossef rc = -ENOMEM; 2677ff27e85aSGilad Ben-Yossef goto fail0; 2678ff27e85aSGilad Ben-Yossef } 2679ff27e85aSGilad Ben-Yossef 2680ff27e85aSGilad Ben-Yossef INIT_LIST_HEAD(&aead_handle->aead_list); 2681ff27e85aSGilad Ben-Yossef drvdata->aead_handle = aead_handle; 2682ff27e85aSGilad Ben-Yossef 2683ff27e85aSGilad Ben-Yossef aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2684ff27e85aSGilad Ben-Yossef MAX_HMAC_DIGEST_SIZE); 2685ff27e85aSGilad Ben-Yossef 2686ff27e85aSGilad Ben-Yossef if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2687ff27e85aSGilad Ben-Yossef rc = -ENOMEM; 2688ff27e85aSGilad Ben-Yossef goto fail1; 2689ff27e85aSGilad Ben-Yossef } 2690ff27e85aSGilad Ben-Yossef 2691ff27e85aSGilad Ben-Yossef /* Linux crypto */ 2692ff27e85aSGilad Ben-Yossef for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 26931c876a90SGilad Ben-Yossef if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 26941c876a90SGilad Ben-Yossef !(drvdata->std_bodies & aead_algs[alg].std_body)) 269527b3b22dSGilad Ben-Yossef continue; 269627b3b22dSGilad Ben-Yossef 2697ff27e85aSGilad Ben-Yossef t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2698ff27e85aSGilad Ben-Yossef if (IS_ERR(t_alg)) { 2699ff27e85aSGilad Ben-Yossef rc = PTR_ERR(t_alg); 2700ff27e85aSGilad Ben-Yossef dev_err(dev, "%s alg allocation failed\n", 2701ff27e85aSGilad Ben-Yossef aead_algs[alg].driver_name); 2702ff27e85aSGilad Ben-Yossef goto fail1; 2703ff27e85aSGilad Ben-Yossef } 2704ff27e85aSGilad Ben-Yossef t_alg->drvdata = drvdata; 2705ff27e85aSGilad Ben-Yossef rc = crypto_register_aead(&t_alg->aead_alg); 2706ff27e85aSGilad Ben-Yossef if (rc) { 2707ff27e85aSGilad Ben-Yossef dev_err(dev, "%s alg registration failed\n", 2708ff27e85aSGilad Ben-Yossef t_alg->aead_alg.base.cra_driver_name); 2709ff27e85aSGilad Ben-Yossef goto fail2; 2710ff27e85aSGilad Ben-Yossef } else { 2711ff27e85aSGilad Ben-Yossef list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2712ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Registered %s\n", 2713ff27e85aSGilad Ben-Yossef t_alg->aead_alg.base.cra_driver_name); 2714ff27e85aSGilad Ben-Yossef } 2715ff27e85aSGilad Ben-Yossef } 2716ff27e85aSGilad Ben-Yossef 2717ff27e85aSGilad Ben-Yossef return 0; 2718ff27e85aSGilad Ben-Yossef 2719ff27e85aSGilad Ben-Yossef fail2: 2720ff27e85aSGilad Ben-Yossef kfree(t_alg); 2721ff27e85aSGilad Ben-Yossef fail1: 2722ff27e85aSGilad Ben-Yossef cc_aead_free(drvdata); 2723ff27e85aSGilad Ben-Yossef fail0: 2724ff27e85aSGilad Ben-Yossef return rc; 2725ff27e85aSGilad Ben-Yossef } 2726