1ff27e85aSGilad Ben-Yossef // SPDX-License-Identifier: GPL-2.0 203963caeSGilad Ben-Yossef /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3ff27e85aSGilad Ben-Yossef 4ff27e85aSGilad Ben-Yossef #include <linux/kernel.h> 5ff27e85aSGilad Ben-Yossef #include <linux/module.h> 6ff27e85aSGilad Ben-Yossef #include <crypto/algapi.h> 7ff27e85aSGilad Ben-Yossef #include <crypto/internal/aead.h> 8ff27e85aSGilad Ben-Yossef #include <crypto/authenc.h> 9ff27e85aSGilad Ben-Yossef #include <crypto/des.h> 10ff27e85aSGilad Ben-Yossef #include <linux/rtnetlink.h> 11ff27e85aSGilad Ben-Yossef #include "cc_driver.h" 12ff27e85aSGilad Ben-Yossef #include "cc_buffer_mgr.h" 13ff27e85aSGilad Ben-Yossef #include "cc_aead.h" 14ff27e85aSGilad Ben-Yossef #include "cc_request_mgr.h" 15ff27e85aSGilad Ben-Yossef #include "cc_hash.h" 16ff27e85aSGilad Ben-Yossef #include "cc_sram_mgr.h" 17ff27e85aSGilad Ben-Yossef 18ff27e85aSGilad Ben-Yossef #define template_aead template_u.aead 19ff27e85aSGilad Ben-Yossef 20ff27e85aSGilad Ben-Yossef #define MAX_AEAD_SETKEY_SEQ 12 21ff27e85aSGilad Ben-Yossef #define MAX_AEAD_PROCESS_SEQ 23 22ff27e85aSGilad Ben-Yossef 23ff27e85aSGilad Ben-Yossef #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 24ff27e85aSGilad Ben-Yossef #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 25ff27e85aSGilad Ben-Yossef 26ff27e85aSGilad Ben-Yossef #define AES_CCM_RFC4309_NONCE_SIZE 3 27ff27e85aSGilad Ben-Yossef #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 28ff27e85aSGilad Ben-Yossef 29ff27e85aSGilad Ben-Yossef /* Value of each ICV_CMP byte (of 8) in case of success */ 30ff27e85aSGilad Ben-Yossef #define ICV_VERIF_OK 0x01 31ff27e85aSGilad Ben-Yossef 32ff27e85aSGilad Ben-Yossef struct cc_aead_handle { 33ff27e85aSGilad Ben-Yossef cc_sram_addr_t sram_workspace_addr; 34ff27e85aSGilad Ben-Yossef struct list_head aead_list; 35ff27e85aSGilad Ben-Yossef }; 36ff27e85aSGilad Ben-Yossef 37ff27e85aSGilad Ben-Yossef struct cc_hmac_s { 38ff27e85aSGilad Ben-Yossef u8 *padded_authkey; 39ff27e85aSGilad Ben-Yossef u8 *ipad_opad; /* IPAD, OPAD*/ 40ff27e85aSGilad Ben-Yossef dma_addr_t padded_authkey_dma_addr; 41ff27e85aSGilad Ben-Yossef dma_addr_t ipad_opad_dma_addr; 42ff27e85aSGilad Ben-Yossef }; 43ff27e85aSGilad Ben-Yossef 44ff27e85aSGilad Ben-Yossef struct cc_xcbc_s { 45ff27e85aSGilad Ben-Yossef u8 *xcbc_keys; /* K1,K2,K3 */ 46ff27e85aSGilad Ben-Yossef dma_addr_t xcbc_keys_dma_addr; 47ff27e85aSGilad Ben-Yossef }; 48ff27e85aSGilad Ben-Yossef 49ff27e85aSGilad Ben-Yossef struct cc_aead_ctx { 50ff27e85aSGilad Ben-Yossef struct cc_drvdata *drvdata; 51ff27e85aSGilad Ben-Yossef u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 52ff27e85aSGilad Ben-Yossef u8 *enckey; 53ff27e85aSGilad Ben-Yossef dma_addr_t enckey_dma_addr; 54ff27e85aSGilad Ben-Yossef union { 55ff27e85aSGilad Ben-Yossef struct cc_hmac_s hmac; 56ff27e85aSGilad Ben-Yossef struct cc_xcbc_s xcbc; 57ff27e85aSGilad Ben-Yossef } auth_state; 58ff27e85aSGilad Ben-Yossef unsigned int enc_keylen; 59ff27e85aSGilad Ben-Yossef unsigned int auth_keylen; 60ff27e85aSGilad Ben-Yossef unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 61f1e52fd0SYael Chemla unsigned int hash_len; 62ff27e85aSGilad Ben-Yossef enum drv_cipher_mode cipher_mode; 63ff27e85aSGilad Ben-Yossef enum cc_flow_mode flow_mode; 64ff27e85aSGilad Ben-Yossef enum drv_hash_mode auth_mode; 65ff27e85aSGilad Ben-Yossef }; 66ff27e85aSGilad Ben-Yossef 67ff27e85aSGilad Ben-Yossef static inline bool valid_assoclen(struct aead_request *req) 68ff27e85aSGilad Ben-Yossef { 69ff27e85aSGilad Ben-Yossef return ((req->assoclen == 16) || (req->assoclen == 20)); 70ff27e85aSGilad Ben-Yossef } 71ff27e85aSGilad Ben-Yossef 72ff27e85aSGilad Ben-Yossef static void cc_aead_exit(struct crypto_aead *tfm) 73ff27e85aSGilad Ben-Yossef { 74ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 75ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 76ff27e85aSGilad Ben-Yossef 77ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 78ff27e85aSGilad Ben-Yossef crypto_tfm_alg_name(&tfm->base)); 79ff27e85aSGilad Ben-Yossef 80ff27e85aSGilad Ben-Yossef /* Unmap enckey buffer */ 81ff27e85aSGilad Ben-Yossef if (ctx->enckey) { 82ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 83ff27e85aSGilad Ben-Yossef ctx->enckey_dma_addr); 84ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 85ff27e85aSGilad Ben-Yossef &ctx->enckey_dma_addr); 86ff27e85aSGilad Ben-Yossef ctx->enckey_dma_addr = 0; 87ff27e85aSGilad Ben-Yossef ctx->enckey = NULL; 88ff27e85aSGilad Ben-Yossef } 89ff27e85aSGilad Ben-Yossef 90ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 91ff27e85aSGilad Ben-Yossef struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 92ff27e85aSGilad Ben-Yossef 93ff27e85aSGilad Ben-Yossef if (xcbc->xcbc_keys) { 94ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 95ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys, 96ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys_dma_addr); 97ff27e85aSGilad Ben-Yossef } 98ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 99ff27e85aSGilad Ben-Yossef &xcbc->xcbc_keys_dma_addr); 100ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys_dma_addr = 0; 101ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys = NULL; 102ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 103ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 104ff27e85aSGilad Ben-Yossef 105ff27e85aSGilad Ben-Yossef if (hmac->ipad_opad) { 106ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 107ff27e85aSGilad Ben-Yossef hmac->ipad_opad, 108ff27e85aSGilad Ben-Yossef hmac->ipad_opad_dma_addr); 109ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 110ff27e85aSGilad Ben-Yossef &hmac->ipad_opad_dma_addr); 111ff27e85aSGilad Ben-Yossef hmac->ipad_opad_dma_addr = 0; 112ff27e85aSGilad Ben-Yossef hmac->ipad_opad = NULL; 113ff27e85aSGilad Ben-Yossef } 114ff27e85aSGilad Ben-Yossef if (hmac->padded_authkey) { 115ff27e85aSGilad Ben-Yossef dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 116ff27e85aSGilad Ben-Yossef hmac->padded_authkey, 117ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr); 118ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 119ff27e85aSGilad Ben-Yossef &hmac->padded_authkey_dma_addr); 120ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr = 0; 121ff27e85aSGilad Ben-Yossef hmac->padded_authkey = NULL; 122ff27e85aSGilad Ben-Yossef } 123ff27e85aSGilad Ben-Yossef } 124ff27e85aSGilad Ben-Yossef } 125ff27e85aSGilad Ben-Yossef 126f1e52fd0SYael Chemla static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 127f1e52fd0SYael Chemla { 128f1e52fd0SYael Chemla struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 129f1e52fd0SYael Chemla 130f1e52fd0SYael Chemla return cc_get_default_hash_len(ctx->drvdata); 131f1e52fd0SYael Chemla } 132f1e52fd0SYael Chemla 133ff27e85aSGilad Ben-Yossef static int cc_aead_init(struct crypto_aead *tfm) 134ff27e85aSGilad Ben-Yossef { 135ff27e85aSGilad Ben-Yossef struct aead_alg *alg = crypto_aead_alg(tfm); 136ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 137ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *cc_alg = 138ff27e85aSGilad Ben-Yossef container_of(alg, struct cc_crypto_alg, aead_alg); 139ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(cc_alg->drvdata); 140ff27e85aSGilad Ben-Yossef 141ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 142ff27e85aSGilad Ben-Yossef crypto_tfm_alg_name(&tfm->base)); 143ff27e85aSGilad Ben-Yossef 144ff27e85aSGilad Ben-Yossef /* Initialize modes in instance */ 145ff27e85aSGilad Ben-Yossef ctx->cipher_mode = cc_alg->cipher_mode; 146ff27e85aSGilad Ben-Yossef ctx->flow_mode = cc_alg->flow_mode; 147ff27e85aSGilad Ben-Yossef ctx->auth_mode = cc_alg->auth_mode; 148ff27e85aSGilad Ben-Yossef ctx->drvdata = cc_alg->drvdata; 149ff27e85aSGilad Ben-Yossef crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx)); 150ff27e85aSGilad Ben-Yossef 151ff27e85aSGilad Ben-Yossef /* Allocate key buffer, cache line aligned */ 152ff27e85aSGilad Ben-Yossef ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 153ff27e85aSGilad Ben-Yossef &ctx->enckey_dma_addr, GFP_KERNEL); 154ff27e85aSGilad Ben-Yossef if (!ctx->enckey) { 155ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating key buffer\n"); 156ff27e85aSGilad Ben-Yossef goto init_failed; 157ff27e85aSGilad Ben-Yossef } 158ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 159ff27e85aSGilad Ben-Yossef ctx->enckey); 160ff27e85aSGilad Ben-Yossef 161ff27e85aSGilad Ben-Yossef /* Set default authlen value */ 162ff27e85aSGilad Ben-Yossef 163ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 164ff27e85aSGilad Ben-Yossef struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 165ff27e85aSGilad Ben-Yossef const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 166ff27e85aSGilad Ben-Yossef 167ff27e85aSGilad Ben-Yossef /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 168ff27e85aSGilad Ben-Yossef /* (and temporary for user key - up to 256b) */ 169ff27e85aSGilad Ben-Yossef xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 170ff27e85aSGilad Ben-Yossef &xcbc->xcbc_keys_dma_addr, 171ff27e85aSGilad Ben-Yossef GFP_KERNEL); 172ff27e85aSGilad Ben-Yossef if (!xcbc->xcbc_keys) { 173ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 174ff27e85aSGilad Ben-Yossef goto init_failed; 175ff27e85aSGilad Ben-Yossef } 176ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 177ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 178ff27e85aSGilad Ben-Yossef const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 179ff27e85aSGilad Ben-Yossef dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 180ff27e85aSGilad Ben-Yossef 181ff27e85aSGilad Ben-Yossef /* Allocate dma-coherent buffer for IPAD + OPAD */ 182ff27e85aSGilad Ben-Yossef hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 183ff27e85aSGilad Ben-Yossef &hmac->ipad_opad_dma_addr, 184ff27e85aSGilad Ben-Yossef GFP_KERNEL); 185ff27e85aSGilad Ben-Yossef 186ff27e85aSGilad Ben-Yossef if (!hmac->ipad_opad) { 187ff27e85aSGilad Ben-Yossef dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 188ff27e85aSGilad Ben-Yossef goto init_failed; 189ff27e85aSGilad Ben-Yossef } 190ff27e85aSGilad Ben-Yossef 191ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 192ff27e85aSGilad Ben-Yossef hmac->ipad_opad); 193ff27e85aSGilad Ben-Yossef 194ff27e85aSGilad Ben-Yossef hmac->padded_authkey = dma_alloc_coherent(dev, 195ff27e85aSGilad Ben-Yossef MAX_HMAC_BLOCK_SIZE, 196ff27e85aSGilad Ben-Yossef pkey_dma, 197ff27e85aSGilad Ben-Yossef GFP_KERNEL); 198ff27e85aSGilad Ben-Yossef 199ff27e85aSGilad Ben-Yossef if (!hmac->padded_authkey) { 200ff27e85aSGilad Ben-Yossef dev_err(dev, "failed to allocate padded_authkey\n"); 201ff27e85aSGilad Ben-Yossef goto init_failed; 202ff27e85aSGilad Ben-Yossef } 203ff27e85aSGilad Ben-Yossef } else { 204ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.ipad_opad = NULL; 205ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.padded_authkey = NULL; 206ff27e85aSGilad Ben-Yossef } 207f1e52fd0SYael Chemla ctx->hash_len = cc_get_aead_hash_len(tfm); 208ff27e85aSGilad Ben-Yossef 209ff27e85aSGilad Ben-Yossef return 0; 210ff27e85aSGilad Ben-Yossef 211ff27e85aSGilad Ben-Yossef init_failed: 212ff27e85aSGilad Ben-Yossef cc_aead_exit(tfm); 213ff27e85aSGilad Ben-Yossef return -ENOMEM; 214ff27e85aSGilad Ben-Yossef } 215ff27e85aSGilad Ben-Yossef 216ff27e85aSGilad Ben-Yossef static void cc_aead_complete(struct device *dev, void *cc_req, int err) 217ff27e85aSGilad Ben-Yossef { 218ff27e85aSGilad Ben-Yossef struct aead_request *areq = (struct aead_request *)cc_req; 219ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 220ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 221ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 222ff27e85aSGilad Ben-Yossef 223a108f931SGilad Ben-Yossef /* BACKLOG notification */ 224a108f931SGilad Ben-Yossef if (err == -EINPROGRESS) 225a108f931SGilad Ben-Yossef goto done; 226a108f931SGilad Ben-Yossef 227ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, areq); 228ff27e85aSGilad Ben-Yossef 229ff27e85aSGilad Ben-Yossef /* Restore ordinary iv pointer */ 230ff27e85aSGilad Ben-Yossef areq->iv = areq_ctx->backup_iv; 231ff27e85aSGilad Ben-Yossef 232ff27e85aSGilad Ben-Yossef if (err) 233ff27e85aSGilad Ben-Yossef goto done; 234ff27e85aSGilad Ben-Yossef 235ff27e85aSGilad Ben-Yossef if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 236ff27e85aSGilad Ben-Yossef if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 237ff27e85aSGilad Ben-Yossef ctx->authsize) != 0) { 238ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 239ff27e85aSGilad Ben-Yossef ctx->authsize, ctx->cipher_mode); 240ff27e85aSGilad Ben-Yossef /* In case of payload authentication failure, MUST NOT 241ff27e85aSGilad Ben-Yossef * revealed the decrypted message --> zero its memory. 242ff27e85aSGilad Ben-Yossef */ 243ff27e85aSGilad Ben-Yossef cc_zero_sgl(areq->dst, areq_ctx->cryptlen); 244ff27e85aSGilad Ben-Yossef err = -EBADMSG; 245ff27e85aSGilad Ben-Yossef } 246ff27e85aSGilad Ben-Yossef } else { /*ENCRYPT*/ 247ff27e85aSGilad Ben-Yossef if (areq_ctx->is_icv_fragmented) { 248ff27e85aSGilad Ben-Yossef u32 skip = areq->cryptlen + areq_ctx->dst_offset; 249ff27e85aSGilad Ben-Yossef 250ff27e85aSGilad Ben-Yossef cc_copy_sg_portion(dev, areq_ctx->mac_buf, 251ff27e85aSGilad Ben-Yossef areq_ctx->dst_sgl, skip, 252ff27e85aSGilad Ben-Yossef (skip + ctx->authsize), 253ff27e85aSGilad Ben-Yossef CC_SG_FROM_BUF); 254ff27e85aSGilad Ben-Yossef } 255ff27e85aSGilad Ben-Yossef 256ff27e85aSGilad Ben-Yossef /* If an IV was generated, copy it back to the user provided 257ff27e85aSGilad Ben-Yossef * buffer. 258ff27e85aSGilad Ben-Yossef */ 259ff27e85aSGilad Ben-Yossef if (areq_ctx->backup_giv) { 260ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) 261ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv + 262ff27e85aSGilad Ben-Yossef CTR_RFC3686_NONCE_SIZE, 263ff27e85aSGilad Ben-Yossef CTR_RFC3686_IV_SIZE); 264ff27e85aSGilad Ben-Yossef else if (ctx->cipher_mode == DRV_CIPHER_CCM) 265ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv + 266ff27e85aSGilad Ben-Yossef CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE); 267ff27e85aSGilad Ben-Yossef } 268ff27e85aSGilad Ben-Yossef } 269ff27e85aSGilad Ben-Yossef done: 270ff27e85aSGilad Ben-Yossef aead_request_complete(areq, err); 271ff27e85aSGilad Ben-Yossef } 272ff27e85aSGilad Ben-Yossef 273ff27e85aSGilad Ben-Yossef static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 274ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx) 275ff27e85aSGilad Ben-Yossef { 276ff27e85aSGilad Ben-Yossef /* Load the AES key */ 277ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[0]); 278ff27e85aSGilad Ben-Yossef /* We are using for the source/user key the same buffer 279ff27e85aSGilad Ben-Yossef * as for the output keys, * because after this key loading it 280ff27e85aSGilad Ben-Yossef * is not needed anymore 281ff27e85aSGilad Ben-Yossef */ 282ff27e85aSGilad Ben-Yossef set_din_type(&desc[0], DMA_DLLI, 283ff27e85aSGilad Ben-Yossef ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 284ff27e85aSGilad Ben-Yossef NS_BIT); 285ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 286ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 287ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[0], ctx->auth_keylen); 288ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[0], S_DIN_to_AES); 289ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 290ff27e85aSGilad Ben-Yossef 291ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[1]); 292ff27e85aSGilad Ben-Yossef set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 293ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[1], DIN_AES_DOUT); 294ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 295ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0); 296ff27e85aSGilad Ben-Yossef 297ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[2]); 298ff27e85aSGilad Ben-Yossef set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 299ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[2], DIN_AES_DOUT); 300ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 301ff27e85aSGilad Ben-Yossef + AES_KEYSIZE_128), 302ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0); 303ff27e85aSGilad Ben-Yossef 304ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[3]); 305ff27e85aSGilad Ben-Yossef set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 306ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[3], DIN_AES_DOUT); 307ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 308ff27e85aSGilad Ben-Yossef + 2 * AES_KEYSIZE_128), 309ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT, 0); 310ff27e85aSGilad Ben-Yossef 311ff27e85aSGilad Ben-Yossef return 4; 312ff27e85aSGilad Ben-Yossef } 313ff27e85aSGilad Ben-Yossef 314ff27e85aSGilad Ben-Yossef static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx) 315ff27e85aSGilad Ben-Yossef { 316ff27e85aSGilad Ben-Yossef unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 317ff27e85aSGilad Ben-Yossef unsigned int digest_ofs = 0; 318ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 319ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 320ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 321ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 322ff27e85aSGilad Ben-Yossef struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 323ff27e85aSGilad Ben-Yossef 324ff27e85aSGilad Ben-Yossef unsigned int idx = 0; 325ff27e85aSGilad Ben-Yossef int i; 326ff27e85aSGilad Ben-Yossef 327ff27e85aSGilad Ben-Yossef /* calc derived HMAC key */ 328ff27e85aSGilad Ben-Yossef for (i = 0; i < 2; i++) { 329ff27e85aSGilad Ben-Yossef /* Load hash initial state */ 330ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 331ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 332ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], 333ff27e85aSGilad Ben-Yossef cc_larval_digest_addr(ctx->drvdata, 334ff27e85aSGilad Ben-Yossef ctx->auth_mode), 335ff27e85aSGilad Ben-Yossef digest_size); 336ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 337ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 338ff27e85aSGilad Ben-Yossef idx++; 339ff27e85aSGilad Ben-Yossef 340ff27e85aSGilad Ben-Yossef /* Load the hash current length*/ 341ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 342ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 343f1e52fd0SYael Chemla set_din_const(&desc[idx], 0, ctx->hash_len); 344ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 345ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 346ff27e85aSGilad Ben-Yossef idx++; 347ff27e85aSGilad Ben-Yossef 348ff27e85aSGilad Ben-Yossef /* Prepare ipad key */ 349ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 350ff27e85aSGilad Ben-Yossef set_xor_val(&desc[idx], hmac_pad_const[i]); 351ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 352ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 353ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 354ff27e85aSGilad Ben-Yossef idx++; 355ff27e85aSGilad Ben-Yossef 356ff27e85aSGilad Ben-Yossef /* Perform HASH update */ 357ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 358ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 359ff27e85aSGilad Ben-Yossef hmac->padded_authkey_dma_addr, 360ff27e85aSGilad Ben-Yossef SHA256_BLOCK_SIZE, NS_BIT); 361ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 362ff27e85aSGilad Ben-Yossef set_xor_active(&desc[idx]); 363ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 364ff27e85aSGilad Ben-Yossef idx++; 365ff27e85aSGilad Ben-Yossef 366ff27e85aSGilad Ben-Yossef /* Get the digset */ 367ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 368ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 369ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], 370ff27e85aSGilad Ben-Yossef (hmac->ipad_opad_dma_addr + digest_ofs), 371ff27e85aSGilad Ben-Yossef digest_size, NS_BIT, 0); 372ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 373ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 374ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 375ff27e85aSGilad Ben-Yossef idx++; 376ff27e85aSGilad Ben-Yossef 377ff27e85aSGilad Ben-Yossef digest_ofs += digest_size; 378ff27e85aSGilad Ben-Yossef } 379ff27e85aSGilad Ben-Yossef 380ff27e85aSGilad Ben-Yossef return idx; 381ff27e85aSGilad Ben-Yossef } 382ff27e85aSGilad Ben-Yossef 383ff27e85aSGilad Ben-Yossef static int validate_keys_sizes(struct cc_aead_ctx *ctx) 384ff27e85aSGilad Ben-Yossef { 385ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 386ff27e85aSGilad Ben-Yossef 387ff27e85aSGilad Ben-Yossef dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 388ff27e85aSGilad Ben-Yossef ctx->enc_keylen, ctx->auth_keylen); 389ff27e85aSGilad Ben-Yossef 390ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { 391ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 392ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 393ff27e85aSGilad Ben-Yossef break; 394ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC: 395ff27e85aSGilad Ben-Yossef if (ctx->auth_keylen != AES_KEYSIZE_128 && 396ff27e85aSGilad Ben-Yossef ctx->auth_keylen != AES_KEYSIZE_192 && 397ff27e85aSGilad Ben-Yossef ctx->auth_keylen != AES_KEYSIZE_256) 398ff27e85aSGilad Ben-Yossef return -ENOTSUPP; 399ff27e85aSGilad Ben-Yossef break; 400ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 401ff27e85aSGilad Ben-Yossef if (ctx->auth_keylen > 0) 402ff27e85aSGilad Ben-Yossef return -EINVAL; 403ff27e85aSGilad Ben-Yossef break; 404ff27e85aSGilad Ben-Yossef default: 405ff27e85aSGilad Ben-Yossef dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 406ff27e85aSGilad Ben-Yossef return -EINVAL; 407ff27e85aSGilad Ben-Yossef } 408ff27e85aSGilad Ben-Yossef /* Check cipher key size */ 409ff27e85aSGilad Ben-Yossef if (ctx->flow_mode == S_DIN_to_DES) { 410ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 411ff27e85aSGilad Ben-Yossef dev_err(dev, "Invalid cipher(3DES) key size: %u\n", 412ff27e85aSGilad Ben-Yossef ctx->enc_keylen); 413ff27e85aSGilad Ben-Yossef return -EINVAL; 414ff27e85aSGilad Ben-Yossef } 415ff27e85aSGilad Ben-Yossef } else { /* Default assumed to be AES ciphers */ 416ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen != AES_KEYSIZE_128 && 417ff27e85aSGilad Ben-Yossef ctx->enc_keylen != AES_KEYSIZE_192 && 418ff27e85aSGilad Ben-Yossef ctx->enc_keylen != AES_KEYSIZE_256) { 419ff27e85aSGilad Ben-Yossef dev_err(dev, "Invalid cipher(AES) key size: %u\n", 420ff27e85aSGilad Ben-Yossef ctx->enc_keylen); 421ff27e85aSGilad Ben-Yossef return -EINVAL; 422ff27e85aSGilad Ben-Yossef } 423ff27e85aSGilad Ben-Yossef } 424ff27e85aSGilad Ben-Yossef 425ff27e85aSGilad Ben-Yossef return 0; /* All tests of keys sizes passed */ 426ff27e85aSGilad Ben-Yossef } 427ff27e85aSGilad Ben-Yossef 428ff27e85aSGilad Ben-Yossef /* This function prepers the user key so it can pass to the hmac processing 429ff27e85aSGilad Ben-Yossef * (copy to intenral buffer or hash in case of key longer than block 430ff27e85aSGilad Ben-Yossef */ 431ff27e85aSGilad Ben-Yossef static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key, 432ff27e85aSGilad Ben-Yossef unsigned int keylen) 433ff27e85aSGilad Ben-Yossef { 434ff27e85aSGilad Ben-Yossef dma_addr_t key_dma_addr = 0; 435ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 436ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 437ff27e85aSGilad Ben-Yossef u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode); 438ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 439ff27e85aSGilad Ben-Yossef unsigned int blocksize; 440ff27e85aSGilad Ben-Yossef unsigned int digestsize; 441ff27e85aSGilad Ben-Yossef unsigned int hashmode; 442ff27e85aSGilad Ben-Yossef unsigned int idx = 0; 443ff27e85aSGilad Ben-Yossef int rc = 0; 444ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 445ff27e85aSGilad Ben-Yossef dma_addr_t padded_authkey_dma_addr = 446ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.padded_authkey_dma_addr; 447ff27e85aSGilad Ben-Yossef 448ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { /* auth_key required and >0 */ 449ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 450ff27e85aSGilad Ben-Yossef blocksize = SHA1_BLOCK_SIZE; 451ff27e85aSGilad Ben-Yossef digestsize = SHA1_DIGEST_SIZE; 452ff27e85aSGilad Ben-Yossef hashmode = DRV_HASH_HW_SHA1; 453ff27e85aSGilad Ben-Yossef break; 454ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 455ff27e85aSGilad Ben-Yossef default: 456ff27e85aSGilad Ben-Yossef blocksize = SHA256_BLOCK_SIZE; 457ff27e85aSGilad Ben-Yossef digestsize = SHA256_DIGEST_SIZE; 458ff27e85aSGilad Ben-Yossef hashmode = DRV_HASH_HW_SHA256; 459ff27e85aSGilad Ben-Yossef } 460ff27e85aSGilad Ben-Yossef 461ff27e85aSGilad Ben-Yossef if (keylen != 0) { 462ff27e85aSGilad Ben-Yossef key_dma_addr = dma_map_single(dev, (void *)key, keylen, 463ff27e85aSGilad Ben-Yossef DMA_TO_DEVICE); 464ff27e85aSGilad Ben-Yossef if (dma_mapping_error(dev, key_dma_addr)) { 465ff27e85aSGilad Ben-Yossef dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 466ff27e85aSGilad Ben-Yossef key, keylen); 467ff27e85aSGilad Ben-Yossef return -ENOMEM; 468ff27e85aSGilad Ben-Yossef } 469ff27e85aSGilad Ben-Yossef if (keylen > blocksize) { 470ff27e85aSGilad Ben-Yossef /* Load hash initial state */ 471ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 472ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode); 473ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], larval_addr, digestsize); 474ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 475ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 476ff27e85aSGilad Ben-Yossef idx++; 477ff27e85aSGilad Ben-Yossef 478ff27e85aSGilad Ben-Yossef /* Load the hash current length*/ 479ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 480ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode); 481f1e52fd0SYael Chemla set_din_const(&desc[idx], 0, ctx->hash_len); 482ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 483ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 484ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 485ff27e85aSGilad Ben-Yossef idx++; 486ff27e85aSGilad Ben-Yossef 487ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 488ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 489ff27e85aSGilad Ben-Yossef key_dma_addr, keylen, NS_BIT); 490ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 491ff27e85aSGilad Ben-Yossef idx++; 492ff27e85aSGilad Ben-Yossef 493ff27e85aSGilad Ben-Yossef /* Get hashed key */ 494ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 495ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hashmode); 496ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 497ff27e85aSGilad Ben-Yossef digestsize, NS_BIT, 0); 498ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 499ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 500ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 501ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], 502ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN); 503ff27e85aSGilad Ben-Yossef idx++; 504ff27e85aSGilad Ben-Yossef 505ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 506ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, (blocksize - digestsize)); 507ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 508ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 509ff27e85aSGilad Ben-Yossef digestsize), (blocksize - digestsize), 510ff27e85aSGilad Ben-Yossef NS_BIT, 0); 511ff27e85aSGilad Ben-Yossef idx++; 512ff27e85aSGilad Ben-Yossef } else { 513ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 514ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 515ff27e85aSGilad Ben-Yossef keylen, NS_BIT); 516ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 517ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 518ff27e85aSGilad Ben-Yossef keylen, NS_BIT, 0); 519ff27e85aSGilad Ben-Yossef idx++; 520ff27e85aSGilad Ben-Yossef 521ff27e85aSGilad Ben-Yossef if ((blocksize - keylen) != 0) { 522ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 523ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, 524ff27e85aSGilad Ben-Yossef (blocksize - keylen)); 525ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 526ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], 527ff27e85aSGilad Ben-Yossef (padded_authkey_dma_addr + 528ff27e85aSGilad Ben-Yossef keylen), 529ff27e85aSGilad Ben-Yossef (blocksize - keylen), NS_BIT, 0); 530ff27e85aSGilad Ben-Yossef idx++; 531ff27e85aSGilad Ben-Yossef } 532ff27e85aSGilad Ben-Yossef } 533ff27e85aSGilad Ben-Yossef } else { 534ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 535ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, (blocksize - keylen)); 536ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], BYPASS); 537ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 538ff27e85aSGilad Ben-Yossef blocksize, NS_BIT, 0); 539ff27e85aSGilad Ben-Yossef idx++; 540ff27e85aSGilad Ben-Yossef } 541ff27e85aSGilad Ben-Yossef 542ff27e85aSGilad Ben-Yossef rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 543ff27e85aSGilad Ben-Yossef if (rc) 544ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc); 545ff27e85aSGilad Ben-Yossef 546ff27e85aSGilad Ben-Yossef if (key_dma_addr) 547ff27e85aSGilad Ben-Yossef dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 548ff27e85aSGilad Ben-Yossef 549ff27e85aSGilad Ben-Yossef return rc; 550ff27e85aSGilad Ben-Yossef } 551ff27e85aSGilad Ben-Yossef 552ff27e85aSGilad Ben-Yossef static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 553ff27e85aSGilad Ben-Yossef unsigned int keylen) 554ff27e85aSGilad Ben-Yossef { 555ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 556ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 557ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 558ff27e85aSGilad Ben-Yossef unsigned int seq_len = 0; 559ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 560dc95b535SEric Biggers const u8 *enckey, *authkey; 561dc95b535SEric Biggers int rc; 562ff27e85aSGilad Ben-Yossef 563ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 564ff27e85aSGilad Ben-Yossef ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 565ff27e85aSGilad Ben-Yossef 566ff27e85aSGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 567ff27e85aSGilad Ben-Yossef 568ff27e85aSGilad Ben-Yossef if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 569dc95b535SEric Biggers struct crypto_authenc_keys keys; 570dc95b535SEric Biggers 571dc95b535SEric Biggers rc = crypto_authenc_extractkeys(&keys, key, keylen); 572dc95b535SEric Biggers if (rc) 573ff27e85aSGilad Ben-Yossef goto badkey; 574dc95b535SEric Biggers enckey = keys.enckey; 575dc95b535SEric Biggers authkey = keys.authkey; 576dc95b535SEric Biggers ctx->enc_keylen = keys.enckeylen; 577dc95b535SEric Biggers ctx->auth_keylen = keys.authkeylen; 578ff27e85aSGilad Ben-Yossef 579ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) { 580ff27e85aSGilad Ben-Yossef /* the nonce is stored in bytes at end of key */ 581dc95b535SEric Biggers rc = -EINVAL; 582ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen < 583ff27e85aSGilad Ben-Yossef (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 584ff27e85aSGilad Ben-Yossef goto badkey; 585ff27e85aSGilad Ben-Yossef /* Copy nonce from last 4 bytes in CTR key to 586ff27e85aSGilad Ben-Yossef * first 4 bytes in CTR IV 587ff27e85aSGilad Ben-Yossef */ 588dc95b535SEric Biggers memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - 589dc95b535SEric Biggers CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); 590ff27e85aSGilad Ben-Yossef /* Set CTR key size */ 591ff27e85aSGilad Ben-Yossef ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 592ff27e85aSGilad Ben-Yossef } 593ff27e85aSGilad Ben-Yossef } else { /* non-authenc - has just one key */ 594dc95b535SEric Biggers enckey = key; 595dc95b535SEric Biggers authkey = NULL; 596ff27e85aSGilad Ben-Yossef ctx->enc_keylen = keylen; 597ff27e85aSGilad Ben-Yossef ctx->auth_keylen = 0; 598ff27e85aSGilad Ben-Yossef } 599ff27e85aSGilad Ben-Yossef 600ff27e85aSGilad Ben-Yossef rc = validate_keys_sizes(ctx); 601ff27e85aSGilad Ben-Yossef if (rc) 602ff27e85aSGilad Ben-Yossef goto badkey; 603ff27e85aSGilad Ben-Yossef 604ff27e85aSGilad Ben-Yossef /* STAT_PHASE_1: Copy key to ctx */ 605ff27e85aSGilad Ben-Yossef 606ff27e85aSGilad Ben-Yossef /* Get key material */ 607dc95b535SEric Biggers memcpy(ctx->enckey, enckey, ctx->enc_keylen); 608ff27e85aSGilad Ben-Yossef if (ctx->enc_keylen == 24) 609ff27e85aSGilad Ben-Yossef memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 610ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 611dc95b535SEric Biggers memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, 612dc95b535SEric Biggers ctx->auth_keylen); 613ff27e85aSGilad Ben-Yossef } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 614dc95b535SEric Biggers rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); 615ff27e85aSGilad Ben-Yossef if (rc) 616ff27e85aSGilad Ben-Yossef goto badkey; 617ff27e85aSGilad Ben-Yossef } 618ff27e85aSGilad Ben-Yossef 619ff27e85aSGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */ 620ff27e85aSGilad Ben-Yossef 621ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { 622ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 623ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 624ff27e85aSGilad Ben-Yossef seq_len = hmac_setkey(desc, ctx); 625ff27e85aSGilad Ben-Yossef break; 626ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC: 627ff27e85aSGilad Ben-Yossef seq_len = xcbc_setkey(desc, ctx); 628ff27e85aSGilad Ben-Yossef break; 629ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 630ff27e85aSGilad Ben-Yossef break; /* No auth. key setup */ 631ff27e85aSGilad Ben-Yossef default: 632ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 633ff27e85aSGilad Ben-Yossef rc = -ENOTSUPP; 634ff27e85aSGilad Ben-Yossef goto badkey; 635ff27e85aSGilad Ben-Yossef } 636ff27e85aSGilad Ben-Yossef 637ff27e85aSGilad Ben-Yossef /* STAT_PHASE_3: Submit sequence to HW */ 638ff27e85aSGilad Ben-Yossef 639ff27e85aSGilad Ben-Yossef if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 640ff27e85aSGilad Ben-Yossef rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 641ff27e85aSGilad Ben-Yossef if (rc) { 642ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc); 643ff27e85aSGilad Ben-Yossef goto setkey_error; 644ff27e85aSGilad Ben-Yossef } 645ff27e85aSGilad Ben-Yossef } 646ff27e85aSGilad Ben-Yossef 647ff27e85aSGilad Ben-Yossef /* Update STAT_PHASE_3 */ 648ff27e85aSGilad Ben-Yossef return rc; 649ff27e85aSGilad Ben-Yossef 650ff27e85aSGilad Ben-Yossef badkey: 651ff27e85aSGilad Ben-Yossef crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 652ff27e85aSGilad Ben-Yossef 653ff27e85aSGilad Ben-Yossef setkey_error: 654ff27e85aSGilad Ben-Yossef return rc; 655ff27e85aSGilad Ben-Yossef } 656ff27e85aSGilad Ben-Yossef 6579fbfcefcSHerbert Xu static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 6589fbfcefcSHerbert Xu unsigned int keylen) 6599fbfcefcSHerbert Xu { 6609fbfcefcSHerbert Xu struct crypto_authenc_keys keys; 6619fbfcefcSHerbert Xu u32 flags; 6629fbfcefcSHerbert Xu int err; 6639fbfcefcSHerbert Xu 6649fbfcefcSHerbert Xu err = crypto_authenc_extractkeys(&keys, key, keylen); 6659fbfcefcSHerbert Xu if (unlikely(err)) 6669fbfcefcSHerbert Xu goto badkey; 6679fbfcefcSHerbert Xu 6689fbfcefcSHerbert Xu err = -EINVAL; 6699fbfcefcSHerbert Xu if (keys.enckeylen != DES3_EDE_KEY_SIZE) 6709fbfcefcSHerbert Xu goto badkey; 6719fbfcefcSHerbert Xu 6729fbfcefcSHerbert Xu flags = crypto_aead_get_flags(aead); 6739fbfcefcSHerbert Xu err = __des3_verify_key(&flags, keys.enckey); 6749fbfcefcSHerbert Xu if (unlikely(err)) { 6759fbfcefcSHerbert Xu crypto_aead_set_flags(aead, flags); 6769fbfcefcSHerbert Xu goto out; 6779fbfcefcSHerbert Xu } 6789fbfcefcSHerbert Xu 6799fbfcefcSHerbert Xu err = cc_aead_setkey(aead, key, keylen); 6809fbfcefcSHerbert Xu 6819fbfcefcSHerbert Xu out: 6829fbfcefcSHerbert Xu memzero_explicit(&keys, sizeof(keys)); 6839fbfcefcSHerbert Xu return err; 6849fbfcefcSHerbert Xu 6859fbfcefcSHerbert Xu badkey: 6869fbfcefcSHerbert Xu crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 6879fbfcefcSHerbert Xu goto out; 6889fbfcefcSHerbert Xu } 6899fbfcefcSHerbert Xu 690ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 691ff27e85aSGilad Ben-Yossef unsigned int keylen) 692ff27e85aSGilad Ben-Yossef { 693ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 694ff27e85aSGilad Ben-Yossef 695ff27e85aSGilad Ben-Yossef if (keylen < 3) 696ff27e85aSGilad Ben-Yossef return -EINVAL; 697ff27e85aSGilad Ben-Yossef 698ff27e85aSGilad Ben-Yossef keylen -= 3; 699ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 3); 700ff27e85aSGilad Ben-Yossef 701ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen); 702ff27e85aSGilad Ben-Yossef } 703ff27e85aSGilad Ben-Yossef 704ff27e85aSGilad Ben-Yossef static int cc_aead_setauthsize(struct crypto_aead *authenc, 705ff27e85aSGilad Ben-Yossef unsigned int authsize) 706ff27e85aSGilad Ben-Yossef { 707ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 708ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 709ff27e85aSGilad Ben-Yossef 710ff27e85aSGilad Ben-Yossef /* Unsupported auth. sizes */ 711ff27e85aSGilad Ben-Yossef if (authsize == 0 || 712ff27e85aSGilad Ben-Yossef authsize > crypto_aead_maxauthsize(authenc)) { 713ff27e85aSGilad Ben-Yossef return -ENOTSUPP; 714ff27e85aSGilad Ben-Yossef } 715ff27e85aSGilad Ben-Yossef 716ff27e85aSGilad Ben-Yossef ctx->authsize = authsize; 717ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authlen=%d\n", ctx->authsize); 718ff27e85aSGilad Ben-Yossef 719ff27e85aSGilad Ben-Yossef return 0; 720ff27e85aSGilad Ben-Yossef } 721ff27e85aSGilad Ben-Yossef 722ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 723ff27e85aSGilad Ben-Yossef unsigned int authsize) 724ff27e85aSGilad Ben-Yossef { 725ff27e85aSGilad Ben-Yossef switch (authsize) { 726ff27e85aSGilad Ben-Yossef case 8: 727ff27e85aSGilad Ben-Yossef case 12: 728ff27e85aSGilad Ben-Yossef case 16: 729ff27e85aSGilad Ben-Yossef break; 730ff27e85aSGilad Ben-Yossef default: 731ff27e85aSGilad Ben-Yossef return -EINVAL; 732ff27e85aSGilad Ben-Yossef } 733ff27e85aSGilad Ben-Yossef 734ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 735ff27e85aSGilad Ben-Yossef } 736ff27e85aSGilad Ben-Yossef 737ff27e85aSGilad Ben-Yossef static int cc_ccm_setauthsize(struct crypto_aead *authenc, 738ff27e85aSGilad Ben-Yossef unsigned int authsize) 739ff27e85aSGilad Ben-Yossef { 740ff27e85aSGilad Ben-Yossef switch (authsize) { 741ff27e85aSGilad Ben-Yossef case 4: 742ff27e85aSGilad Ben-Yossef case 6: 743ff27e85aSGilad Ben-Yossef case 8: 744ff27e85aSGilad Ben-Yossef case 10: 745ff27e85aSGilad Ben-Yossef case 12: 746ff27e85aSGilad Ben-Yossef case 14: 747ff27e85aSGilad Ben-Yossef case 16: 748ff27e85aSGilad Ben-Yossef break; 749ff27e85aSGilad Ben-Yossef default: 750ff27e85aSGilad Ben-Yossef return -EINVAL; 751ff27e85aSGilad Ben-Yossef } 752ff27e85aSGilad Ben-Yossef 753ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 754ff27e85aSGilad Ben-Yossef } 755ff27e85aSGilad Ben-Yossef 756ff27e85aSGilad Ben-Yossef static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 757ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 758ff27e85aSGilad Ben-Yossef { 759ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 760ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 761ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 762ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 763ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 764ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 765ff27e85aSGilad Ben-Yossef 766ff27e85aSGilad Ben-Yossef switch (assoc_dma_type) { 767ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI: 768ff27e85aSGilad Ben-Yossef dev_dbg(dev, "ASSOC buffer type DLLI\n"); 769ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 770ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 771ff27e85aSGilad Ben-Yossef areq->assoclen, NS_BIT); 772ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 773ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 774ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen > 0) 775ff27e85aSGilad Ben-Yossef set_din_not_last_indication(&desc[idx]); 776ff27e85aSGilad Ben-Yossef break; 777ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI: 778ff27e85aSGilad Ben-Yossef dev_dbg(dev, "ASSOC buffer type MLLI\n"); 779ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 780ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 781ff27e85aSGilad Ben-Yossef areq_ctx->assoc.mlli_nents, NS_BIT); 782ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 783ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 784ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen > 0) 785ff27e85aSGilad Ben-Yossef set_din_not_last_indication(&desc[idx]); 786ff27e85aSGilad Ben-Yossef break; 787ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL: 788ff27e85aSGilad Ben-Yossef default: 789ff27e85aSGilad Ben-Yossef dev_err(dev, "Invalid ASSOC buffer type\n"); 790ff27e85aSGilad Ben-Yossef } 791ff27e85aSGilad Ben-Yossef 792ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 793ff27e85aSGilad Ben-Yossef } 794ff27e85aSGilad Ben-Yossef 795ff27e85aSGilad Ben-Yossef static void cc_proc_authen_desc(struct aead_request *areq, 796ff27e85aSGilad Ben-Yossef unsigned int flow_mode, 797ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 798ff27e85aSGilad Ben-Yossef unsigned int *seq_size, int direct) 799ff27e85aSGilad Ben-Yossef { 800ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 801ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 802ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 803ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 804ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 805ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 806ff27e85aSGilad Ben-Yossef 807ff27e85aSGilad Ben-Yossef switch (data_dma_type) { 808ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI: 809ff27e85aSGilad Ben-Yossef { 810ff27e85aSGilad Ben-Yossef struct scatterlist *cipher = 811ff27e85aSGilad Ben-Yossef (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 812ff27e85aSGilad Ben-Yossef areq_ctx->dst_sgl : areq_ctx->src_sgl; 813ff27e85aSGilad Ben-Yossef 814ff27e85aSGilad Ben-Yossef unsigned int offset = 815ff27e85aSGilad Ben-Yossef (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 816ff27e85aSGilad Ben-Yossef areq_ctx->dst_offset : areq_ctx->src_offset; 817ff27e85aSGilad Ben-Yossef dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 818ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 819ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 820ff27e85aSGilad Ben-Yossef (sg_dma_address(cipher) + offset), 821ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen, NS_BIT); 822ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 823ff27e85aSGilad Ben-Yossef break; 824ff27e85aSGilad Ben-Yossef } 825ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI: 826ff27e85aSGilad Ben-Yossef { 827ff27e85aSGilad Ben-Yossef /* DOUBLE-PASS flow (as default) 828ff27e85aSGilad Ben-Yossef * assoc. + iv + data -compact in one table 829ff27e85aSGilad Ben-Yossef * if assoclen is ZERO only IV perform 830ff27e85aSGilad Ben-Yossef */ 831ff27e85aSGilad Ben-Yossef cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr; 832ff27e85aSGilad Ben-Yossef u32 mlli_nents = areq_ctx->assoc.mlli_nents; 833ff27e85aSGilad Ben-Yossef 834ff27e85aSGilad Ben-Yossef if (areq_ctx->is_single_pass) { 835ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 836ff27e85aSGilad Ben-Yossef mlli_addr = areq_ctx->dst.sram_addr; 837ff27e85aSGilad Ben-Yossef mlli_nents = areq_ctx->dst.mlli_nents; 838ff27e85aSGilad Ben-Yossef } else { 839ff27e85aSGilad Ben-Yossef mlli_addr = areq_ctx->src.sram_addr; 840ff27e85aSGilad Ben-Yossef mlli_nents = areq_ctx->src.mlli_nents; 841ff27e85aSGilad Ben-Yossef } 842ff27e85aSGilad Ben-Yossef } 843ff27e85aSGilad Ben-Yossef 844ff27e85aSGilad Ben-Yossef dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 845ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 846ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 847ff27e85aSGilad Ben-Yossef NS_BIT); 848ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 849ff27e85aSGilad Ben-Yossef break; 850ff27e85aSGilad Ben-Yossef } 851ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL: 852ff27e85aSGilad Ben-Yossef default: 853ff27e85aSGilad Ben-Yossef dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 854ff27e85aSGilad Ben-Yossef } 855ff27e85aSGilad Ben-Yossef 856ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 857ff27e85aSGilad Ben-Yossef } 858ff27e85aSGilad Ben-Yossef 859ff27e85aSGilad Ben-Yossef static void cc_proc_cipher_desc(struct aead_request *areq, 860ff27e85aSGilad Ben-Yossef unsigned int flow_mode, 861ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 862ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 863ff27e85aSGilad Ben-Yossef { 864ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 865ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 866ff27e85aSGilad Ben-Yossef enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 867ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 868ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 869ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 870ff27e85aSGilad Ben-Yossef 871ff27e85aSGilad Ben-Yossef if (areq_ctx->cryptlen == 0) 872ff27e85aSGilad Ben-Yossef return; /*null processing*/ 873ff27e85aSGilad Ben-Yossef 874ff27e85aSGilad Ben-Yossef switch (data_dma_type) { 875ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_DLLI: 876ff27e85aSGilad Ben-Yossef dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 877ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 878ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 879ff27e85aSGilad Ben-Yossef (sg_dma_address(areq_ctx->src_sgl) + 880ff27e85aSGilad Ben-Yossef areq_ctx->src_offset), areq_ctx->cryptlen, 881ff27e85aSGilad Ben-Yossef NS_BIT); 882ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], 883ff27e85aSGilad Ben-Yossef (sg_dma_address(areq_ctx->dst_sgl) + 884ff27e85aSGilad Ben-Yossef areq_ctx->dst_offset), 885ff27e85aSGilad Ben-Yossef areq_ctx->cryptlen, NS_BIT, 0); 886ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 887ff27e85aSGilad Ben-Yossef break; 888ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_MLLI: 889ff27e85aSGilad Ben-Yossef dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 890ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 891ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 892ff27e85aSGilad Ben-Yossef areq_ctx->src.mlli_nents, NS_BIT); 893ff27e85aSGilad Ben-Yossef set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 894ff27e85aSGilad Ben-Yossef areq_ctx->dst.mlli_nents, NS_BIT, 0); 895ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], flow_mode); 896ff27e85aSGilad Ben-Yossef break; 897ff27e85aSGilad Ben-Yossef case CC_DMA_BUF_NULL: 898ff27e85aSGilad Ben-Yossef default: 899ff27e85aSGilad Ben-Yossef dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 900ff27e85aSGilad Ben-Yossef } 901ff27e85aSGilad Ben-Yossef 902ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 903ff27e85aSGilad Ben-Yossef } 904ff27e85aSGilad Ben-Yossef 905ff27e85aSGilad Ben-Yossef static void cc_proc_digest_desc(struct aead_request *req, 906ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 907ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 908ff27e85aSGilad Ben-Yossef { 909ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 910ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 911ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 912ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 913ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 914ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 915ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 916ff27e85aSGilad Ben-Yossef 917ff27e85aSGilad Ben-Yossef /* Get final ICV result */ 918ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 919ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 920ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 921ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 922ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 923ff27e85aSGilad Ben-Yossef NS_BIT, 1); 92427b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 925ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 926ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 927ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 928ff27e85aSGilad Ben-Yossef } else { 929ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], 930ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN); 931ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 932ff27e85aSGilad Ben-Yossef } 933ff27e85aSGilad Ben-Yossef } else { /*Decrypt*/ 934ff27e85aSGilad Ben-Yossef /* Get ICV out from hardware */ 935ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 936ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 937ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 938ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 939ff27e85aSGilad Ben-Yossef ctx->authsize, NS_BIT, 1); 94027b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 941ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], 942ff27e85aSGilad Ben-Yossef HASH_DIGEST_RESULT_LITTLE_ENDIAN); 943ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 944ff27e85aSGilad Ben-Yossef if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 945ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 946ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 947ff27e85aSGilad Ben-Yossef } else { 948ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 949ff27e85aSGilad Ben-Yossef } 950ff27e85aSGilad Ben-Yossef } 951ff27e85aSGilad Ben-Yossef 952ff27e85aSGilad Ben-Yossef *seq_size = (++idx); 953ff27e85aSGilad Ben-Yossef } 954ff27e85aSGilad Ben-Yossef 955ff27e85aSGilad Ben-Yossef static void cc_set_cipher_desc(struct aead_request *req, 956ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 957ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 958ff27e85aSGilad Ben-Yossef { 959ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 960ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 961ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 962ff27e85aSGilad Ben-Yossef unsigned int hw_iv_size = req_ctx->hw_iv_size; 963ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 964ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 965ff27e85aSGilad Ben-Yossef 966ff27e85aSGilad Ben-Yossef /* Setup cipher state */ 967ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 968ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], direct); 969ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], ctx->flow_mode); 970ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 971ff27e85aSGilad Ben-Yossef hw_iv_size, NS_BIT); 972ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) 973ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 974ff27e85aSGilad Ben-Yossef else 975ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 976ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], ctx->cipher_mode); 977ff27e85aSGilad Ben-Yossef idx++; 978ff27e85aSGilad Ben-Yossef 979ff27e85aSGilad Ben-Yossef /* Setup enc. key */ 980ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 981ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], direct); 982ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 983ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], ctx->flow_mode); 984ff27e85aSGilad Ben-Yossef if (ctx->flow_mode == S_DIN_to_AES) { 985ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 986ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 987ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT); 988ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 989ff27e85aSGilad Ben-Yossef } else { 990ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 991ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT); 992ff27e85aSGilad Ben-Yossef set_key_size_des(&desc[idx], ctx->enc_keylen); 993ff27e85aSGilad Ben-Yossef } 994ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], ctx->cipher_mode); 995ff27e85aSGilad Ben-Yossef idx++; 996ff27e85aSGilad Ben-Yossef 997ff27e85aSGilad Ben-Yossef *seq_size = idx; 998ff27e85aSGilad Ben-Yossef } 999ff27e85aSGilad Ben-Yossef 1000ff27e85aSGilad Ben-Yossef static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 1001ff27e85aSGilad Ben-Yossef unsigned int *seq_size, unsigned int data_flow_mode) 1002ff27e85aSGilad Ben-Yossef { 1003ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1004ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 1005ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1006ff27e85aSGilad Ben-Yossef 1007ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen == 0) 1008ff27e85aSGilad Ben-Yossef return; /*null processing*/ 1009ff27e85aSGilad Ben-Yossef 1010ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, &idx); 1011ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 1012ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1013ff27e85aSGilad Ben-Yossef /* We must wait for DMA to write all cipher */ 1014ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1015ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1016ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1017ff27e85aSGilad Ben-Yossef idx++; 1018ff27e85aSGilad Ben-Yossef } 1019ff27e85aSGilad Ben-Yossef 1020ff27e85aSGilad Ben-Yossef *seq_size = idx; 1021ff27e85aSGilad Ben-Yossef } 1022ff27e85aSGilad Ben-Yossef 1023ff27e85aSGilad Ben-Yossef static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 1024ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1025ff27e85aSGilad Ben-Yossef { 1026ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1027ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1028ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1029ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1030ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1031ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1032ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1033ff27e85aSGilad Ben-Yossef 1034ff27e85aSGilad Ben-Yossef /* Loading hash ipad xor key state */ 1035ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1036ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1037ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1038ff27e85aSGilad Ben-Yossef ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1039ff27e85aSGilad Ben-Yossef NS_BIT); 1040ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1041ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1042ff27e85aSGilad Ben-Yossef idx++; 1043ff27e85aSGilad Ben-Yossef 1044ff27e85aSGilad Ben-Yossef /* Load init. digest len (64 bytes) */ 1045ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1046ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1047ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1048f1e52fd0SYael Chemla ctx->hash_len); 1049ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1050ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1051ff27e85aSGilad Ben-Yossef idx++; 1052ff27e85aSGilad Ben-Yossef 1053ff27e85aSGilad Ben-Yossef *seq_size = idx; 1054ff27e85aSGilad Ben-Yossef } 1055ff27e85aSGilad Ben-Yossef 1056ff27e85aSGilad Ben-Yossef static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1057ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1058ff27e85aSGilad Ben-Yossef { 1059ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1060ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1061ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1062ff27e85aSGilad Ben-Yossef 1063ff27e85aSGilad Ben-Yossef /* Loading MAC state */ 1064ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1065ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1066ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1067ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1068ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1069ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1070ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1071ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1072ff27e85aSGilad Ben-Yossef idx++; 1073ff27e85aSGilad Ben-Yossef 1074ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K1 */ 1075ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1076ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1077ff27e85aSGilad Ben-Yossef ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1078ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128, NS_BIT); 1079ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1080ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1081ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1082ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1083ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1084ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1085ff27e85aSGilad Ben-Yossef idx++; 1086ff27e85aSGilad Ben-Yossef 1087ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K2 */ 1088ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1089ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1090ff27e85aSGilad Ben-Yossef (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1091ff27e85aSGilad Ben-Yossef AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1092ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1093ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1094ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1095ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1096ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1097ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1098ff27e85aSGilad Ben-Yossef idx++; 1099ff27e85aSGilad Ben-Yossef 1100ff27e85aSGilad Ben-Yossef /* Setup XCBC MAC K3 */ 1101ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1102ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1103ff27e85aSGilad Ben-Yossef (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1104ff27e85aSGilad Ben-Yossef 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1105ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1106ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1107ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1108ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1109ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1110ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1111ff27e85aSGilad Ben-Yossef idx++; 1112ff27e85aSGilad Ben-Yossef 1113ff27e85aSGilad Ben-Yossef *seq_size = idx; 1114ff27e85aSGilad Ben-Yossef } 1115ff27e85aSGilad Ben-Yossef 1116ff27e85aSGilad Ben-Yossef static void cc_proc_header_desc(struct aead_request *req, 1117ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 1118ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1119ff27e85aSGilad Ben-Yossef { 1120ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1121ff27e85aSGilad Ben-Yossef /* Hash associated data */ 1122ff27e85aSGilad Ben-Yossef if (req->assoclen > 0) 1123ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1124ff27e85aSGilad Ben-Yossef 1125ff27e85aSGilad Ben-Yossef /* Hash IV */ 1126ff27e85aSGilad Ben-Yossef *seq_size = idx; 1127ff27e85aSGilad Ben-Yossef } 1128ff27e85aSGilad Ben-Yossef 1129ff27e85aSGilad Ben-Yossef static void cc_proc_scheme_desc(struct aead_request *req, 1130ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 1131ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1132ff27e85aSGilad Ben-Yossef { 1133ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1134ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1135ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1136ff27e85aSGilad Ben-Yossef unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1137ff27e85aSGilad Ben-Yossef DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1138ff27e85aSGilad Ben-Yossef unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1139ff27e85aSGilad Ben-Yossef CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1140ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1141ff27e85aSGilad Ben-Yossef 1142ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1143ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1144ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1145f1e52fd0SYael Chemla ctx->hash_len); 1146ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1147ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1148ff27e85aSGilad Ben-Yossef set_cipher_do(&desc[idx], DO_PAD); 1149ff27e85aSGilad Ben-Yossef idx++; 1150ff27e85aSGilad Ben-Yossef 1151ff27e85aSGilad Ben-Yossef /* Get final ICV result */ 1152ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1153ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1154ff27e85aSGilad Ben-Yossef digest_size); 1155ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1156ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1157ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1158ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1159ff27e85aSGilad Ben-Yossef idx++; 1160ff27e85aSGilad Ben-Yossef 1161ff27e85aSGilad Ben-Yossef /* Loading hash opad xor key state */ 1162ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1163ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1164ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1165ff27e85aSGilad Ben-Yossef (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1166ff27e85aSGilad Ben-Yossef digest_size, NS_BIT); 1167ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1168ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1169ff27e85aSGilad Ben-Yossef idx++; 1170ff27e85aSGilad Ben-Yossef 1171ff27e85aSGilad Ben-Yossef /* Load init. digest len (64 bytes) */ 1172ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1173ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], hash_mode); 1174ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1175f1e52fd0SYael Chemla ctx->hash_len); 1176ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1177ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1178ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1179ff27e85aSGilad Ben-Yossef idx++; 1180ff27e85aSGilad Ben-Yossef 1181ff27e85aSGilad Ben-Yossef /* Perform HASH update */ 1182ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1183ff27e85aSGilad Ben-Yossef set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1184ff27e85aSGilad Ben-Yossef digest_size); 1185ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 1186ff27e85aSGilad Ben-Yossef idx++; 1187ff27e85aSGilad Ben-Yossef 1188ff27e85aSGilad Ben-Yossef *seq_size = idx; 1189ff27e85aSGilad Ben-Yossef } 1190ff27e85aSGilad Ben-Yossef 1191ff27e85aSGilad Ben-Yossef static void cc_mlli_to_sram(struct aead_request *req, 1192ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 1193ff27e85aSGilad Ben-Yossef { 1194ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1195ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1196ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1197ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1198ff27e85aSGilad Ben-Yossef 1199ff27e85aSGilad Ben-Yossef if (req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1200ff27e85aSGilad Ben-Yossef req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1201ff27e85aSGilad Ben-Yossef !req_ctx->is_single_pass) { 1202ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1203ff27e85aSGilad Ben-Yossef (unsigned int)ctx->drvdata->mlli_sram_addr, 1204ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len); 1205ff27e85aSGilad Ben-Yossef /* Copy MLLI table host-to-sram */ 1206ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[*seq_size]); 1207ff27e85aSGilad Ben-Yossef set_din_type(&desc[*seq_size], DMA_DLLI, 1208ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_dma_addr, 1209ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len, NS_BIT); 1210ff27e85aSGilad Ben-Yossef set_dout_sram(&desc[*seq_size], 1211ff27e85aSGilad Ben-Yossef ctx->drvdata->mlli_sram_addr, 1212ff27e85aSGilad Ben-Yossef req_ctx->mlli_params.mlli_len); 1213ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[*seq_size], BYPASS); 1214ff27e85aSGilad Ben-Yossef (*seq_size)++; 1215ff27e85aSGilad Ben-Yossef } 1216ff27e85aSGilad Ben-Yossef } 1217ff27e85aSGilad Ben-Yossef 1218ff27e85aSGilad Ben-Yossef static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1219ff27e85aSGilad Ben-Yossef enum cc_flow_mode setup_flow_mode, 1220ff27e85aSGilad Ben-Yossef bool is_single_pass) 1221ff27e85aSGilad Ben-Yossef { 1222ff27e85aSGilad Ben-Yossef enum cc_flow_mode data_flow_mode; 1223ff27e85aSGilad Ben-Yossef 1224ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1225ff27e85aSGilad Ben-Yossef if (setup_flow_mode == S_DIN_to_AES) 1226ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1227ff27e85aSGilad Ben-Yossef AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1228ff27e85aSGilad Ben-Yossef else 1229ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1230ff27e85aSGilad Ben-Yossef DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1231ff27e85aSGilad Ben-Yossef } else { /* Decrypt */ 1232ff27e85aSGilad Ben-Yossef if (setup_flow_mode == S_DIN_to_AES) 1233ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1234ff27e85aSGilad Ben-Yossef AES_and_HASH : DIN_AES_DOUT; 1235ff27e85aSGilad Ben-Yossef else 1236ff27e85aSGilad Ben-Yossef data_flow_mode = is_single_pass ? 1237ff27e85aSGilad Ben-Yossef DES_and_HASH : DIN_DES_DOUT; 1238ff27e85aSGilad Ben-Yossef } 1239ff27e85aSGilad Ben-Yossef 1240ff27e85aSGilad Ben-Yossef return data_flow_mode; 1241ff27e85aSGilad Ben-Yossef } 1242ff27e85aSGilad Ben-Yossef 1243ff27e85aSGilad Ben-Yossef static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1244ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1245ff27e85aSGilad Ben-Yossef { 1246ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1247ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1248ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1249ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 1250ff27e85aSGilad Ben-Yossef unsigned int data_flow_mode = 1251ff27e85aSGilad Ben-Yossef cc_get_data_flow(direct, ctx->flow_mode, 1252ff27e85aSGilad Ben-Yossef req_ctx->is_single_pass); 1253ff27e85aSGilad Ben-Yossef 1254ff27e85aSGilad Ben-Yossef if (req_ctx->is_single_pass) { 1255ff27e85aSGilad Ben-Yossef /** 1256ff27e85aSGilad Ben-Yossef * Single-pass flow 1257ff27e85aSGilad Ben-Yossef */ 1258ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size); 1259ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, seq_size); 1260ff27e85aSGilad Ben-Yossef cc_proc_header_desc(req, desc, seq_size); 1261ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1262ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size); 1263ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1264ff27e85aSGilad Ben-Yossef return; 1265ff27e85aSGilad Ben-Yossef } 1266ff27e85aSGilad Ben-Yossef 1267ff27e85aSGilad Ben-Yossef /** 1268ff27e85aSGilad Ben-Yossef * Double-pass flow 1269ff27e85aSGilad Ben-Yossef * Fallback for unsupported single-pass modes, 1270ff27e85aSGilad Ben-Yossef * i.e. using assoc. data of non-word-multiple 1271ff27e85aSGilad Ben-Yossef */ 1272ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1273ff27e85aSGilad Ben-Yossef /* encrypt first.. */ 1274ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1275ff27e85aSGilad Ben-Yossef /* authenc after..*/ 1276ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size); 1277ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1278ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size); 1279ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1280ff27e85aSGilad Ben-Yossef 1281ff27e85aSGilad Ben-Yossef } else { /*DECRYPT*/ 1282ff27e85aSGilad Ben-Yossef /* authenc first..*/ 1283ff27e85aSGilad Ben-Yossef cc_set_hmac_desc(req, desc, seq_size); 1284ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1285ff27e85aSGilad Ben-Yossef cc_proc_scheme_desc(req, desc, seq_size); 1286ff27e85aSGilad Ben-Yossef /* decrypt after.. */ 1287ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1288ff27e85aSGilad Ben-Yossef /* read the digest result with setting the completion bit 1289ff27e85aSGilad Ben-Yossef * must be after the cipher operation 1290ff27e85aSGilad Ben-Yossef */ 1291ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1292ff27e85aSGilad Ben-Yossef } 1293ff27e85aSGilad Ben-Yossef } 1294ff27e85aSGilad Ben-Yossef 1295ff27e85aSGilad Ben-Yossef static void 1296ff27e85aSGilad Ben-Yossef cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1297ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1298ff27e85aSGilad Ben-Yossef { 1299ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1300ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1301ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1302ff27e85aSGilad Ben-Yossef int direct = req_ctx->gen_ctx.op_type; 1303ff27e85aSGilad Ben-Yossef unsigned int data_flow_mode = 1304ff27e85aSGilad Ben-Yossef cc_get_data_flow(direct, ctx->flow_mode, 1305ff27e85aSGilad Ben-Yossef req_ctx->is_single_pass); 1306ff27e85aSGilad Ben-Yossef 1307ff27e85aSGilad Ben-Yossef if (req_ctx->is_single_pass) { 1308ff27e85aSGilad Ben-Yossef /** 1309ff27e85aSGilad Ben-Yossef * Single-pass flow 1310ff27e85aSGilad Ben-Yossef */ 1311ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size); 1312ff27e85aSGilad Ben-Yossef cc_set_cipher_desc(req, desc, seq_size); 1313ff27e85aSGilad Ben-Yossef cc_proc_header_desc(req, desc, seq_size); 1314ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1315ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1316ff27e85aSGilad Ben-Yossef return; 1317ff27e85aSGilad Ben-Yossef } 1318ff27e85aSGilad Ben-Yossef 1319ff27e85aSGilad Ben-Yossef /** 1320ff27e85aSGilad Ben-Yossef * Double-pass flow 1321ff27e85aSGilad Ben-Yossef * Fallback for unsupported single-pass modes, 1322ff27e85aSGilad Ben-Yossef * i.e. using assoc. data of non-word-multiple 1323ff27e85aSGilad Ben-Yossef */ 1324ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1325ff27e85aSGilad Ben-Yossef /* encrypt first.. */ 1326ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1327ff27e85aSGilad Ben-Yossef /* authenc after.. */ 1328ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size); 1329ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1330ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1331ff27e85aSGilad Ben-Yossef } else { /*DECRYPT*/ 1332ff27e85aSGilad Ben-Yossef /* authenc first.. */ 1333ff27e85aSGilad Ben-Yossef cc_set_xcbc_desc(req, desc, seq_size); 1334ff27e85aSGilad Ben-Yossef cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1335ff27e85aSGilad Ben-Yossef /* decrypt after..*/ 1336ff27e85aSGilad Ben-Yossef cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1337ff27e85aSGilad Ben-Yossef /* read the digest result with setting the completion bit 1338ff27e85aSGilad Ben-Yossef * must be after the cipher operation 1339ff27e85aSGilad Ben-Yossef */ 1340ff27e85aSGilad Ben-Yossef cc_proc_digest_desc(req, desc, seq_size); 1341ff27e85aSGilad Ben-Yossef } 1342ff27e85aSGilad Ben-Yossef } 1343ff27e85aSGilad Ben-Yossef 1344ff27e85aSGilad Ben-Yossef static int validate_data_size(struct cc_aead_ctx *ctx, 1345ff27e85aSGilad Ben-Yossef enum drv_crypto_direction direct, 1346ff27e85aSGilad Ben-Yossef struct aead_request *req) 1347ff27e85aSGilad Ben-Yossef { 1348ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1349ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1350ff27e85aSGilad Ben-Yossef unsigned int assoclen = req->assoclen; 1351ff27e85aSGilad Ben-Yossef unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1352ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize) : req->cryptlen; 1353ff27e85aSGilad Ben-Yossef 1354ff27e85aSGilad Ben-Yossef if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1355ff27e85aSGilad Ben-Yossef req->cryptlen < ctx->authsize) 1356ff27e85aSGilad Ben-Yossef goto data_size_err; 1357ff27e85aSGilad Ben-Yossef 1358ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1359ff27e85aSGilad Ben-Yossef 1360ff27e85aSGilad Ben-Yossef switch (ctx->flow_mode) { 1361ff27e85aSGilad Ben-Yossef case S_DIN_to_AES: 1362ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CBC && 1363ff27e85aSGilad Ben-Yossef !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1364ff27e85aSGilad Ben-Yossef goto data_size_err; 1365ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) 1366ff27e85aSGilad Ben-Yossef break; 1367ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1368ff27e85aSGilad Ben-Yossef if (areq_ctx->plaintext_authenticate_only) 1369ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1370ff27e85aSGilad Ben-Yossef break; 1371ff27e85aSGilad Ben-Yossef } 1372ff27e85aSGilad Ben-Yossef 1373ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(assoclen, sizeof(u32))) 1374ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1375ff27e85aSGilad Ben-Yossef 1376ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR && 1377ff27e85aSGilad Ben-Yossef !IS_ALIGNED(cipherlen, sizeof(u32))) 1378ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1379ff27e85aSGilad Ben-Yossef 1380ff27e85aSGilad Ben-Yossef break; 1381ff27e85aSGilad Ben-Yossef case S_DIN_to_DES: 1382ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1383ff27e85aSGilad Ben-Yossef goto data_size_err; 1384ff27e85aSGilad Ben-Yossef if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1385ff27e85aSGilad Ben-Yossef areq_ctx->is_single_pass = false; 1386ff27e85aSGilad Ben-Yossef break; 1387ff27e85aSGilad Ben-Yossef default: 1388ff27e85aSGilad Ben-Yossef dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1389ff27e85aSGilad Ben-Yossef goto data_size_err; 1390ff27e85aSGilad Ben-Yossef } 1391ff27e85aSGilad Ben-Yossef 1392ff27e85aSGilad Ben-Yossef return 0; 1393ff27e85aSGilad Ben-Yossef 1394ff27e85aSGilad Ben-Yossef data_size_err: 1395ff27e85aSGilad Ben-Yossef return -EINVAL; 1396ff27e85aSGilad Ben-Yossef } 1397ff27e85aSGilad Ben-Yossef 1398ff27e85aSGilad Ben-Yossef static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1399ff27e85aSGilad Ben-Yossef { 1400ff27e85aSGilad Ben-Yossef unsigned int len = 0; 1401ff27e85aSGilad Ben-Yossef 1402ff27e85aSGilad Ben-Yossef if (header_size == 0) 1403ff27e85aSGilad Ben-Yossef return 0; 1404ff27e85aSGilad Ben-Yossef 1405ff27e85aSGilad Ben-Yossef if (header_size < ((1UL << 16) - (1UL << 8))) { 1406ff27e85aSGilad Ben-Yossef len = 2; 1407ff27e85aSGilad Ben-Yossef 1408ff27e85aSGilad Ben-Yossef pa0_buff[0] = (header_size >> 8) & 0xFF; 1409ff27e85aSGilad Ben-Yossef pa0_buff[1] = header_size & 0xFF; 1410ff27e85aSGilad Ben-Yossef } else { 1411ff27e85aSGilad Ben-Yossef len = 6; 1412ff27e85aSGilad Ben-Yossef 1413ff27e85aSGilad Ben-Yossef pa0_buff[0] = 0xFF; 1414ff27e85aSGilad Ben-Yossef pa0_buff[1] = 0xFE; 1415ff27e85aSGilad Ben-Yossef pa0_buff[2] = (header_size >> 24) & 0xFF; 1416ff27e85aSGilad Ben-Yossef pa0_buff[3] = (header_size >> 16) & 0xFF; 1417ff27e85aSGilad Ben-Yossef pa0_buff[4] = (header_size >> 8) & 0xFF; 1418ff27e85aSGilad Ben-Yossef pa0_buff[5] = header_size & 0xFF; 1419ff27e85aSGilad Ben-Yossef } 1420ff27e85aSGilad Ben-Yossef 1421ff27e85aSGilad Ben-Yossef return len; 1422ff27e85aSGilad Ben-Yossef } 1423ff27e85aSGilad Ben-Yossef 1424ff27e85aSGilad Ben-Yossef static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1425ff27e85aSGilad Ben-Yossef { 1426ff27e85aSGilad Ben-Yossef __be32 data; 1427ff27e85aSGilad Ben-Yossef 1428ff27e85aSGilad Ben-Yossef memset(block, 0, csize); 1429ff27e85aSGilad Ben-Yossef block += csize; 1430ff27e85aSGilad Ben-Yossef 1431ff27e85aSGilad Ben-Yossef if (csize >= 4) 1432ff27e85aSGilad Ben-Yossef csize = 4; 1433ff27e85aSGilad Ben-Yossef else if (msglen > (1 << (8 * csize))) 1434ff27e85aSGilad Ben-Yossef return -EOVERFLOW; 1435ff27e85aSGilad Ben-Yossef 1436ff27e85aSGilad Ben-Yossef data = cpu_to_be32(msglen); 1437ff27e85aSGilad Ben-Yossef memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1438ff27e85aSGilad Ben-Yossef 1439ff27e85aSGilad Ben-Yossef return 0; 1440ff27e85aSGilad Ben-Yossef } 1441ff27e85aSGilad Ben-Yossef 1442ff27e85aSGilad Ben-Yossef static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1443ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1444ff27e85aSGilad Ben-Yossef { 1445ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1446ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1447ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1448ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1449ff27e85aSGilad Ben-Yossef unsigned int cipher_flow_mode; 1450ff27e85aSGilad Ben-Yossef dma_addr_t mac_result; 1451ff27e85aSGilad Ben-Yossef 1452ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1453ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_to_HASH_and_DOUT; 1454ff27e85aSGilad Ben-Yossef mac_result = req_ctx->mac_buf_dma_addr; 1455ff27e85aSGilad Ben-Yossef } else { /* Encrypt */ 1456ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_and_HASH; 1457ff27e85aSGilad Ben-Yossef mac_result = req_ctx->icv_dma_addr; 1458ff27e85aSGilad Ben-Yossef } 1459ff27e85aSGilad Ben-Yossef 1460ff27e85aSGilad Ben-Yossef /* load key */ 1461ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1462ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1463ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1464ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1465ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT); 1466ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1467ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1468ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1469ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1470ff27e85aSGilad Ben-Yossef idx++; 1471ff27e85aSGilad Ben-Yossef 1472ff27e85aSGilad Ben-Yossef /* load ctr state */ 1473ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1474ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1475ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1476ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1477ff27e85aSGilad Ben-Yossef req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1478ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1479ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1480ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1481ff27e85aSGilad Ben-Yossef idx++; 1482ff27e85aSGilad Ben-Yossef 1483ff27e85aSGilad Ben-Yossef /* load MAC key */ 1484ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1485ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1486ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1487ff27e85aSGilad Ben-Yossef ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1488ff27e85aSGilad Ben-Yossef ctx->enc_keylen), NS_BIT); 1489ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1490ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1491ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1492ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1493ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1494ff27e85aSGilad Ben-Yossef idx++; 1495ff27e85aSGilad Ben-Yossef 1496ff27e85aSGilad Ben-Yossef /* load MAC state */ 1497ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1498ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1499ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1500ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1501ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1502ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1503ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1504ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1505ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1506ff27e85aSGilad Ben-Yossef idx++; 1507ff27e85aSGilad Ben-Yossef 1508ff27e85aSGilad Ben-Yossef /* process assoc data */ 1509ff27e85aSGilad Ben-Yossef if (req->assoclen > 0) { 1510ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1511ff27e85aSGilad Ben-Yossef } else { 1512ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1513ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1514ff27e85aSGilad Ben-Yossef sg_dma_address(&req_ctx->ccm_adata_sg), 1515ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1516ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 1517ff27e85aSGilad Ben-Yossef idx++; 1518ff27e85aSGilad Ben-Yossef } 1519ff27e85aSGilad Ben-Yossef 1520ff27e85aSGilad Ben-Yossef /* process the cipher */ 1521ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen) 1522ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1523ff27e85aSGilad Ben-Yossef 1524ff27e85aSGilad Ben-Yossef /* Read temporal MAC */ 1525ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1526ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1527ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1528ff27e85aSGilad Ben-Yossef NS_BIT, 0); 1529ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1530ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1531ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1532ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1533ff27e85aSGilad Ben-Yossef idx++; 1534ff27e85aSGilad Ben-Yossef 1535ff27e85aSGilad Ben-Yossef /* load AES-CTR state (for last MAC calculation)*/ 1536ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1537ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1538ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1539ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1540ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1541ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1542ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1543ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1544ff27e85aSGilad Ben-Yossef idx++; 1545ff27e85aSGilad Ben-Yossef 1546ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1547ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1548ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1549ff27e85aSGilad Ben-Yossef idx++; 1550ff27e85aSGilad Ben-Yossef 1551ff27e85aSGilad Ben-Yossef /* encrypt the "T" value and store MAC in mac_state */ 1552ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1553ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1554ff27e85aSGilad Ben-Yossef ctx->authsize, NS_BIT); 1555ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 155627b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 1557ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT); 1558ff27e85aSGilad Ben-Yossef idx++; 1559ff27e85aSGilad Ben-Yossef 1560ff27e85aSGilad Ben-Yossef *seq_size = idx; 1561ff27e85aSGilad Ben-Yossef return 0; 1562ff27e85aSGilad Ben-Yossef } 1563ff27e85aSGilad Ben-Yossef 1564ff27e85aSGilad Ben-Yossef static int config_ccm_adata(struct aead_request *req) 1565ff27e85aSGilad Ben-Yossef { 1566ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1567ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1568ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1569ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1570ff27e85aSGilad Ben-Yossef //unsigned int size_of_a = 0, rem_a_size = 0; 1571ff27e85aSGilad Ben-Yossef unsigned int lp = req->iv[0]; 1572ff27e85aSGilad Ben-Yossef /* Note: The code assume that req->iv[0] already contains the value 1573ff27e85aSGilad Ben-Yossef * of L' of RFC3610 1574ff27e85aSGilad Ben-Yossef */ 1575ff27e85aSGilad Ben-Yossef unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1576ff27e85aSGilad Ben-Yossef unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1577ff27e85aSGilad Ben-Yossef u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1578ff27e85aSGilad Ben-Yossef u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1579ff27e85aSGilad Ben-Yossef u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1580ff27e85aSGilad Ben-Yossef unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1581ff27e85aSGilad Ben-Yossef DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1582ff27e85aSGilad Ben-Yossef req->cryptlen : 1583ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize); 1584ff27e85aSGilad Ben-Yossef int rc; 1585ff27e85aSGilad Ben-Yossef 1586ff27e85aSGilad Ben-Yossef memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1587ff27e85aSGilad Ben-Yossef memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1588ff27e85aSGilad Ben-Yossef 1589ff27e85aSGilad Ben-Yossef /* taken from crypto/ccm.c */ 1590ff27e85aSGilad Ben-Yossef /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1591ff27e85aSGilad Ben-Yossef if (l < 2 || l > 8) { 1592ff27e85aSGilad Ben-Yossef dev_err(dev, "illegal iv value %X\n", req->iv[0]); 1593ff27e85aSGilad Ben-Yossef return -EINVAL; 1594ff27e85aSGilad Ben-Yossef } 1595ff27e85aSGilad Ben-Yossef memcpy(b0, req->iv, AES_BLOCK_SIZE); 1596ff27e85aSGilad Ben-Yossef 1597ff27e85aSGilad Ben-Yossef /* format control info per RFC 3610 and 1598ff27e85aSGilad Ben-Yossef * NIST Special Publication 800-38C 1599ff27e85aSGilad Ben-Yossef */ 1600ff27e85aSGilad Ben-Yossef *b0 |= (8 * ((m - 2) / 2)); 1601ff27e85aSGilad Ben-Yossef if (req->assoclen > 0) 1602ff27e85aSGilad Ben-Yossef *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1603ff27e85aSGilad Ben-Yossef 1604ff27e85aSGilad Ben-Yossef rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1605ff27e85aSGilad Ben-Yossef if (rc) { 1606ff27e85aSGilad Ben-Yossef dev_err(dev, "message len overflow detected"); 1607ff27e85aSGilad Ben-Yossef return rc; 1608ff27e85aSGilad Ben-Yossef } 1609ff27e85aSGilad Ben-Yossef /* END of "taken from crypto/ccm.c" */ 1610ff27e85aSGilad Ben-Yossef 1611ff27e85aSGilad Ben-Yossef /* l(a) - size of associated data. */ 1612ff27e85aSGilad Ben-Yossef req_ctx->ccm_hdr_size = format_ccm_a0(a0, req->assoclen); 1613ff27e85aSGilad Ben-Yossef 1614ff27e85aSGilad Ben-Yossef memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1615ff27e85aSGilad Ben-Yossef req->iv[15] = 1; 1616ff27e85aSGilad Ben-Yossef 1617ff27e85aSGilad Ben-Yossef memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1618ff27e85aSGilad Ben-Yossef ctr_count_0[15] = 0; 1619ff27e85aSGilad Ben-Yossef 1620ff27e85aSGilad Ben-Yossef return 0; 1621ff27e85aSGilad Ben-Yossef } 1622ff27e85aSGilad Ben-Yossef 1623ff27e85aSGilad Ben-Yossef static void cc_proc_rfc4309_ccm(struct aead_request *req) 1624ff27e85aSGilad Ben-Yossef { 1625ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1626ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1627ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1628ff27e85aSGilad Ben-Yossef 1629ff27e85aSGilad Ben-Yossef /* L' */ 1630ff27e85aSGilad Ben-Yossef memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1631ff27e85aSGilad Ben-Yossef /* For RFC 4309, always use 4 bytes for message length 1632ff27e85aSGilad Ben-Yossef * (at most 2^32-1 bytes). 1633ff27e85aSGilad Ben-Yossef */ 1634ff27e85aSGilad Ben-Yossef areq_ctx->ctr_iv[0] = 3; 1635ff27e85aSGilad Ben-Yossef 1636ff27e85aSGilad Ben-Yossef /* In RFC 4309 there is an 11-bytes nonce+IV part, 1637ff27e85aSGilad Ben-Yossef * that we build here. 1638ff27e85aSGilad Ben-Yossef */ 1639ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1640ff27e85aSGilad Ben-Yossef CCM_BLOCK_NONCE_SIZE); 1641ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1642ff27e85aSGilad Ben-Yossef CCM_BLOCK_IV_SIZE); 1643ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1644ff27e85aSGilad Ben-Yossef req->assoclen -= CCM_BLOCK_IV_SIZE; 1645ff27e85aSGilad Ben-Yossef } 1646ff27e85aSGilad Ben-Yossef 1647ff27e85aSGilad Ben-Yossef static void cc_set_ghash_desc(struct aead_request *req, 1648ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], unsigned int *seq_size) 1649ff27e85aSGilad Ben-Yossef { 1650ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1651ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1652ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1653ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1654ff27e85aSGilad Ben-Yossef 1655ff27e85aSGilad Ben-Yossef /* load key to AES*/ 1656ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1657ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1658ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1659ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1660ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT); 1661ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1662ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1663ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1664ff27e85aSGilad Ben-Yossef idx++; 1665ff27e85aSGilad Ben-Yossef 1666ff27e85aSGilad Ben-Yossef /* process one zero block to generate hkey */ 1667ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1668ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1669ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1670ff27e85aSGilad Ben-Yossef NS_BIT, 0); 1671ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT); 1672ff27e85aSGilad Ben-Yossef idx++; 1673ff27e85aSGilad Ben-Yossef 1674ff27e85aSGilad Ben-Yossef /* Memory Barrier */ 1675ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1676ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1677ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1678ff27e85aSGilad Ben-Yossef idx++; 1679ff27e85aSGilad Ben-Yossef 1680ff27e85aSGilad Ben-Yossef /* Load GHASH subkey */ 1681ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1682ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1683ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1684ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1685ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1686ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1687ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1688ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1689ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1690ff27e85aSGilad Ben-Yossef idx++; 1691ff27e85aSGilad Ben-Yossef 1692ff27e85aSGilad Ben-Yossef /* Configure Hash Engine to work with GHASH. 1693ff27e85aSGilad Ben-Yossef * Since it was not possible to extend HASH submodes to add GHASH, 1694ff27e85aSGilad Ben-Yossef * The following command is necessary in order to 1695ff27e85aSGilad Ben-Yossef * select GHASH (according to HW designers) 1696ff27e85aSGilad Ben-Yossef */ 1697ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1698ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1699ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1700ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1701ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1702ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1703ff27e85aSGilad Ben-Yossef set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1704ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1705ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1706ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1707ff27e85aSGilad Ben-Yossef idx++; 1708ff27e85aSGilad Ben-Yossef 1709ff27e85aSGilad Ben-Yossef /* Load GHASH initial STATE (which is 0). (for any hash there is an 1710ff27e85aSGilad Ben-Yossef * initial state) 1711ff27e85aSGilad Ben-Yossef */ 1712ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1713ff27e85aSGilad Ben-Yossef set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1714ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1715ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_HASH); 1716ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1717ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1718ff27e85aSGilad Ben-Yossef set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1719ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1720ff27e85aSGilad Ben-Yossef idx++; 1721ff27e85aSGilad Ben-Yossef 1722ff27e85aSGilad Ben-Yossef *seq_size = idx; 1723ff27e85aSGilad Ben-Yossef } 1724ff27e85aSGilad Ben-Yossef 1725ff27e85aSGilad Ben-Yossef static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1726ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1727ff27e85aSGilad Ben-Yossef { 1728ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1729ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1730ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1731ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1732ff27e85aSGilad Ben-Yossef 1733ff27e85aSGilad Ben-Yossef /* load key to AES*/ 1734ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1735ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1736ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1737ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1738ff27e85aSGilad Ben-Yossef ctx->enc_keylen, NS_BIT); 1739ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1740ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1741ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1742ff27e85aSGilad Ben-Yossef idx++; 1743ff27e85aSGilad Ben-Yossef 1744ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1745ff27e85aSGilad Ben-Yossef /* load AES/CTR initial CTR value inc by 2*/ 1746ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1747ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1748ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1749ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, 1750ff27e85aSGilad Ben-Yossef req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1751ff27e85aSGilad Ben-Yossef NS_BIT); 1752ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1753ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1754ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1755ff27e85aSGilad Ben-Yossef idx++; 1756ff27e85aSGilad Ben-Yossef } 1757ff27e85aSGilad Ben-Yossef 1758ff27e85aSGilad Ben-Yossef *seq_size = idx; 1759ff27e85aSGilad Ben-Yossef } 1760ff27e85aSGilad Ben-Yossef 1761ff27e85aSGilad Ben-Yossef static void cc_proc_gcm_result(struct aead_request *req, 1762ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[], 1763ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1764ff27e85aSGilad Ben-Yossef { 1765ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1766ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1767ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1768ff27e85aSGilad Ben-Yossef dma_addr_t mac_result; 1769ff27e85aSGilad Ben-Yossef unsigned int idx = *seq_size; 1770ff27e85aSGilad Ben-Yossef 1771ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1772ff27e85aSGilad Ben-Yossef mac_result = req_ctx->mac_buf_dma_addr; 1773ff27e85aSGilad Ben-Yossef } else { /* Encrypt */ 1774ff27e85aSGilad Ben-Yossef mac_result = req_ctx->icv_dma_addr; 1775ff27e85aSGilad Ben-Yossef } 1776ff27e85aSGilad Ben-Yossef 1777ff27e85aSGilad Ben-Yossef /* process(ghash) gcm_block_len */ 1778ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1779ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1780ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1781ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_HASH); 1782ff27e85aSGilad Ben-Yossef idx++; 1783ff27e85aSGilad Ben-Yossef 1784ff27e85aSGilad Ben-Yossef /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1785ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1786ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1787ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1788ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1789ff27e85aSGilad Ben-Yossef NS_BIT, 0); 1790ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1791ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1792ff27e85aSGilad Ben-Yossef set_aes_not_hash_mode(&desc[idx]); 1793ff27e85aSGilad Ben-Yossef 1794ff27e85aSGilad Ben-Yossef idx++; 1795ff27e85aSGilad Ben-Yossef 1796ff27e85aSGilad Ben-Yossef /* load AES/CTR initial CTR value inc by 1*/ 1797ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1798ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1799ff27e85aSGilad Ben-Yossef set_key_size_aes(&desc[idx], ctx->enc_keylen); 1800ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1801ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1802ff27e85aSGilad Ben-Yossef set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1803ff27e85aSGilad Ben-Yossef set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1804ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], S_DIN_to_AES); 1805ff27e85aSGilad Ben-Yossef idx++; 1806ff27e85aSGilad Ben-Yossef 1807ff27e85aSGilad Ben-Yossef /* Memory Barrier */ 1808ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1809ff27e85aSGilad Ben-Yossef set_din_no_dma(&desc[idx], 0, 0xfffff0); 1810ff27e85aSGilad Ben-Yossef set_dout_no_dma(&desc[idx], 0, 0, 1); 1811ff27e85aSGilad Ben-Yossef idx++; 1812ff27e85aSGilad Ben-Yossef 1813ff27e85aSGilad Ben-Yossef /* process GCTR on stored GHASH and store MAC in mac_state*/ 1814ff27e85aSGilad Ben-Yossef hw_desc_init(&desc[idx]); 1815ff27e85aSGilad Ben-Yossef set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1816ff27e85aSGilad Ben-Yossef set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1817ff27e85aSGilad Ben-Yossef AES_BLOCK_SIZE, NS_BIT); 1818ff27e85aSGilad Ben-Yossef set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 181927b3b22dSGilad Ben-Yossef set_queue_last_ind(ctx->drvdata, &desc[idx]); 1820ff27e85aSGilad Ben-Yossef set_flow_mode(&desc[idx], DIN_AES_DOUT); 1821ff27e85aSGilad Ben-Yossef idx++; 1822ff27e85aSGilad Ben-Yossef 1823ff27e85aSGilad Ben-Yossef *seq_size = idx; 1824ff27e85aSGilad Ben-Yossef } 1825ff27e85aSGilad Ben-Yossef 1826ff27e85aSGilad Ben-Yossef static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1827ff27e85aSGilad Ben-Yossef unsigned int *seq_size) 1828ff27e85aSGilad Ben-Yossef { 1829ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1830ff27e85aSGilad Ben-Yossef unsigned int cipher_flow_mode; 1831ff27e85aSGilad Ben-Yossef 1832ff27e85aSGilad Ben-Yossef if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1833ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_and_HASH; 1834ff27e85aSGilad Ben-Yossef } else { /* Encrypt */ 1835ff27e85aSGilad Ben-Yossef cipher_flow_mode = AES_to_HASH_and_DOUT; 1836ff27e85aSGilad Ben-Yossef } 1837ff27e85aSGilad Ben-Yossef 1838ff27e85aSGilad Ben-Yossef //in RFC4543 no data to encrypt. just copy data from src to dest. 1839ff27e85aSGilad Ben-Yossef if (req_ctx->plaintext_authenticate_only) { 1840ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1841ff27e85aSGilad Ben-Yossef cc_set_ghash_desc(req, desc, seq_size); 1842ff27e85aSGilad Ben-Yossef /* process(ghash) assoc data */ 1843ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1844ff27e85aSGilad Ben-Yossef cc_set_gctr_desc(req, desc, seq_size); 1845ff27e85aSGilad Ben-Yossef cc_proc_gcm_result(req, desc, seq_size); 1846ff27e85aSGilad Ben-Yossef return 0; 1847ff27e85aSGilad Ben-Yossef } 1848ff27e85aSGilad Ben-Yossef 1849ff27e85aSGilad Ben-Yossef // for gcm and rfc4106. 1850ff27e85aSGilad Ben-Yossef cc_set_ghash_desc(req, desc, seq_size); 1851ff27e85aSGilad Ben-Yossef /* process(ghash) assoc data */ 1852ff27e85aSGilad Ben-Yossef if (req->assoclen > 0) 1853ff27e85aSGilad Ben-Yossef cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1854ff27e85aSGilad Ben-Yossef cc_set_gctr_desc(req, desc, seq_size); 1855ff27e85aSGilad Ben-Yossef /* process(gctr+ghash) */ 1856ff27e85aSGilad Ben-Yossef if (req_ctx->cryptlen) 1857ff27e85aSGilad Ben-Yossef cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1858ff27e85aSGilad Ben-Yossef cc_proc_gcm_result(req, desc, seq_size); 1859ff27e85aSGilad Ben-Yossef 1860ff27e85aSGilad Ben-Yossef return 0; 1861ff27e85aSGilad Ben-Yossef } 1862ff27e85aSGilad Ben-Yossef 1863ff27e85aSGilad Ben-Yossef static int config_gcm_context(struct aead_request *req) 1864ff27e85aSGilad Ben-Yossef { 1865ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1866ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1867ff27e85aSGilad Ben-Yossef struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1868ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1869ff27e85aSGilad Ben-Yossef 1870ff27e85aSGilad Ben-Yossef unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1871ff27e85aSGilad Ben-Yossef DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1872ff27e85aSGilad Ben-Yossef req->cryptlen : 1873ff27e85aSGilad Ben-Yossef (req->cryptlen - ctx->authsize); 1874ff27e85aSGilad Ben-Yossef __be32 counter = cpu_to_be32(2); 1875ff27e85aSGilad Ben-Yossef 1876ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n", 1877ff27e85aSGilad Ben-Yossef __func__, cryptlen, req->assoclen, ctx->authsize); 1878ff27e85aSGilad Ben-Yossef 1879ff27e85aSGilad Ben-Yossef memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1880ff27e85aSGilad Ben-Yossef 1881ff27e85aSGilad Ben-Yossef memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1882ff27e85aSGilad Ben-Yossef 1883ff27e85aSGilad Ben-Yossef memcpy(req->iv + 12, &counter, 4); 1884ff27e85aSGilad Ben-Yossef memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1885ff27e85aSGilad Ben-Yossef 1886ff27e85aSGilad Ben-Yossef counter = cpu_to_be32(1); 1887ff27e85aSGilad Ben-Yossef memcpy(req->iv + 12, &counter, 4); 1888ff27e85aSGilad Ben-Yossef memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1889ff27e85aSGilad Ben-Yossef 1890ff27e85aSGilad Ben-Yossef if (!req_ctx->plaintext_authenticate_only) { 1891ff27e85aSGilad Ben-Yossef __be64 temp64; 1892ff27e85aSGilad Ben-Yossef 1893ff27e85aSGilad Ben-Yossef temp64 = cpu_to_be64(req->assoclen * 8); 1894ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1895ff27e85aSGilad Ben-Yossef temp64 = cpu_to_be64(cryptlen * 8); 1896ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1897ff27e85aSGilad Ben-Yossef } else { 1898ff27e85aSGilad Ben-Yossef /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1899ff27e85aSGilad Ben-Yossef * data that is nothing is encrypted. 1900ff27e85aSGilad Ben-Yossef */ 1901ff27e85aSGilad Ben-Yossef __be64 temp64; 1902ff27e85aSGilad Ben-Yossef 1903ff27e85aSGilad Ben-Yossef temp64 = cpu_to_be64((req->assoclen + GCM_BLOCK_RFC4_IV_SIZE + 1904ff27e85aSGilad Ben-Yossef cryptlen) * 8); 1905ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1906ff27e85aSGilad Ben-Yossef temp64 = 0; 1907ff27e85aSGilad Ben-Yossef memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1908ff27e85aSGilad Ben-Yossef } 1909ff27e85aSGilad Ben-Yossef 1910ff27e85aSGilad Ben-Yossef return 0; 1911ff27e85aSGilad Ben-Yossef } 1912ff27e85aSGilad Ben-Yossef 1913ff27e85aSGilad Ben-Yossef static void cc_proc_rfc4_gcm(struct aead_request *req) 1914ff27e85aSGilad Ben-Yossef { 1915ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1916ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1917ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1918ff27e85aSGilad Ben-Yossef 1919ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1920ff27e85aSGilad Ben-Yossef ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1921ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1922ff27e85aSGilad Ben-Yossef GCM_BLOCK_RFC4_IV_SIZE); 1923ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1924ff27e85aSGilad Ben-Yossef req->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; 1925ff27e85aSGilad Ben-Yossef } 1926ff27e85aSGilad Ben-Yossef 1927ff27e85aSGilad Ben-Yossef static int cc_proc_aead(struct aead_request *req, 1928ff27e85aSGilad Ben-Yossef enum drv_crypto_direction direct) 1929ff27e85aSGilad Ben-Yossef { 1930ff27e85aSGilad Ben-Yossef int rc = 0; 1931ff27e85aSGilad Ben-Yossef int seq_len = 0; 1932ff27e85aSGilad Ben-Yossef struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1933ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1934ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1935ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1936ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 1937ff27e85aSGilad Ben-Yossef struct cc_crypto_req cc_req = {}; 1938ff27e85aSGilad Ben-Yossef 1939ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1940ff27e85aSGilad Ben-Yossef ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1941ff27e85aSGilad Ben-Yossef ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1942ff27e85aSGilad Ben-Yossef sg_virt(req->dst), req->dst->offset, req->cryptlen); 1943ff27e85aSGilad Ben-Yossef 1944ff27e85aSGilad Ben-Yossef /* STAT_PHASE_0: Init and sanity checks */ 1945ff27e85aSGilad Ben-Yossef 1946ff27e85aSGilad Ben-Yossef /* Check data length according to mode */ 1947ff27e85aSGilad Ben-Yossef if (validate_data_size(ctx, direct, req)) { 1948ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1949ff27e85aSGilad Ben-Yossef req->cryptlen, req->assoclen); 1950ff27e85aSGilad Ben-Yossef crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN); 1951ff27e85aSGilad Ben-Yossef return -EINVAL; 1952ff27e85aSGilad Ben-Yossef } 1953ff27e85aSGilad Ben-Yossef 1954ff27e85aSGilad Ben-Yossef /* Setup request structure */ 1955ff27e85aSGilad Ben-Yossef cc_req.user_cb = (void *)cc_aead_complete; 1956ff27e85aSGilad Ben-Yossef cc_req.user_arg = (void *)req; 1957ff27e85aSGilad Ben-Yossef 1958ff27e85aSGilad Ben-Yossef /* Setup request context */ 1959ff27e85aSGilad Ben-Yossef areq_ctx->gen_ctx.op_type = direct; 1960ff27e85aSGilad Ben-Yossef areq_ctx->req_authsize = ctx->authsize; 1961ff27e85aSGilad Ben-Yossef areq_ctx->cipher_mode = ctx->cipher_mode; 1962ff27e85aSGilad Ben-Yossef 1963ff27e85aSGilad Ben-Yossef /* STAT_PHASE_1: Map buffers */ 1964ff27e85aSGilad Ben-Yossef 1965ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1966ff27e85aSGilad Ben-Yossef /* Build CTR IV - Copy nonce from last 4 bytes in 1967ff27e85aSGilad Ben-Yossef * CTR key to first 4 bytes in CTR IV 1968ff27e85aSGilad Ben-Yossef */ 1969ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1970ff27e85aSGilad Ben-Yossef CTR_RFC3686_NONCE_SIZE); 1971ff27e85aSGilad Ben-Yossef if (!areq_ctx->backup_giv) /*User none-generated IV*/ 1972ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, 1973ff27e85aSGilad Ben-Yossef req->iv, CTR_RFC3686_IV_SIZE); 1974ff27e85aSGilad Ben-Yossef /* Initialize counter portion of counter block */ 1975ff27e85aSGilad Ben-Yossef *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1976ff27e85aSGilad Ben-Yossef CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1977ff27e85aSGilad Ben-Yossef 1978ff27e85aSGilad Ben-Yossef /* Replace with counter iv */ 1979ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1980ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1981ff27e85aSGilad Ben-Yossef } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1982ff27e85aSGilad Ben-Yossef (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1983ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1984ff27e85aSGilad Ben-Yossef if (areq_ctx->ctr_iv != req->iv) { 1985ff27e85aSGilad Ben-Yossef memcpy(areq_ctx->ctr_iv, req->iv, 1986ff27e85aSGilad Ben-Yossef crypto_aead_ivsize(tfm)); 1987ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->ctr_iv; 1988ff27e85aSGilad Ben-Yossef } 1989ff27e85aSGilad Ben-Yossef } else { 1990ff27e85aSGilad Ben-Yossef areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1991ff27e85aSGilad Ben-Yossef } 1992ff27e85aSGilad Ben-Yossef 1993ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1994ff27e85aSGilad Ben-Yossef rc = config_ccm_adata(req); 1995ff27e85aSGilad Ben-Yossef if (rc) { 1996ff27e85aSGilad Ben-Yossef dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1997ff27e85aSGilad Ben-Yossef rc); 1998ff27e85aSGilad Ben-Yossef goto exit; 1999ff27e85aSGilad Ben-Yossef } 2000ff27e85aSGilad Ben-Yossef } else { 2001ff27e85aSGilad Ben-Yossef areq_ctx->ccm_hdr_size = ccm_header_size_null; 2002ff27e85aSGilad Ben-Yossef } 2003ff27e85aSGilad Ben-Yossef 2004ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 2005ff27e85aSGilad Ben-Yossef rc = config_gcm_context(req); 2006ff27e85aSGilad Ben-Yossef if (rc) { 2007ff27e85aSGilad Ben-Yossef dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 2008ff27e85aSGilad Ben-Yossef rc); 2009ff27e85aSGilad Ben-Yossef goto exit; 2010ff27e85aSGilad Ben-Yossef } 2011ff27e85aSGilad Ben-Yossef } 2012ff27e85aSGilad Ben-Yossef 2013ff27e85aSGilad Ben-Yossef rc = cc_map_aead_request(ctx->drvdata, req); 2014ff27e85aSGilad Ben-Yossef if (rc) { 2015ff27e85aSGilad Ben-Yossef dev_err(dev, "map_request() failed\n"); 2016ff27e85aSGilad Ben-Yossef goto exit; 2017ff27e85aSGilad Ben-Yossef } 2018ff27e85aSGilad Ben-Yossef 2019ff27e85aSGilad Ben-Yossef /* do we need to generate IV? */ 2020ff27e85aSGilad Ben-Yossef if (areq_ctx->backup_giv) { 2021ff27e85aSGilad Ben-Yossef /* set the DMA mapped IV address*/ 2022ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CTR) { 2023ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr[0] = 2024ff27e85aSGilad Ben-Yossef areq_ctx->gen_ctx.iv_dma_addr + 2025ff27e85aSGilad Ben-Yossef CTR_RFC3686_NONCE_SIZE; 2026ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr_len = 1; 2027ff27e85aSGilad Ben-Yossef } else if (ctx->cipher_mode == DRV_CIPHER_CCM) { 2028ff27e85aSGilad Ben-Yossef /* In ccm, the IV needs to exist both inside B0 and 2029ff27e85aSGilad Ben-Yossef * inside the counter.It is also copied to iv_dma_addr 2030ff27e85aSGilad Ben-Yossef * for other reasons (like returning it to the user). 2031ff27e85aSGilad Ben-Yossef * So, using 3 (identical) IV outputs. 2032ff27e85aSGilad Ben-Yossef */ 2033ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr[0] = 2034ff27e85aSGilad Ben-Yossef areq_ctx->gen_ctx.iv_dma_addr + 2035ff27e85aSGilad Ben-Yossef CCM_BLOCK_IV_OFFSET; 2036ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr[1] = 2037ff27e85aSGilad Ben-Yossef sg_dma_address(&areq_ctx->ccm_adata_sg) + 2038ff27e85aSGilad Ben-Yossef CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET; 2039ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr[2] = 2040ff27e85aSGilad Ben-Yossef sg_dma_address(&areq_ctx->ccm_adata_sg) + 2041ff27e85aSGilad Ben-Yossef CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET; 2042ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr_len = 3; 2043ff27e85aSGilad Ben-Yossef } else { 2044ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr[0] = 2045ff27e85aSGilad Ben-Yossef areq_ctx->gen_ctx.iv_dma_addr; 2046ff27e85aSGilad Ben-Yossef cc_req.ivgen_dma_addr_len = 1; 2047ff27e85aSGilad Ben-Yossef } 2048ff27e85aSGilad Ben-Yossef 2049ff27e85aSGilad Ben-Yossef /* set the IV size (8/16 B long)*/ 2050ff27e85aSGilad Ben-Yossef cc_req.ivgen_size = crypto_aead_ivsize(tfm); 2051ff27e85aSGilad Ben-Yossef } 2052ff27e85aSGilad Ben-Yossef 2053ff27e85aSGilad Ben-Yossef /* STAT_PHASE_2: Create sequence */ 2054ff27e85aSGilad Ben-Yossef 2055ff27e85aSGilad Ben-Yossef /* Load MLLI tables to SRAM if necessary */ 2056ff27e85aSGilad Ben-Yossef cc_mlli_to_sram(req, desc, &seq_len); 2057ff27e85aSGilad Ben-Yossef 2058ff27e85aSGilad Ben-Yossef /*TODO: move seq len by reference */ 2059ff27e85aSGilad Ben-Yossef switch (ctx->auth_mode) { 2060ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA1: 2061ff27e85aSGilad Ben-Yossef case DRV_HASH_SHA256: 2062ff27e85aSGilad Ben-Yossef cc_hmac_authenc(req, desc, &seq_len); 2063ff27e85aSGilad Ben-Yossef break; 2064ff27e85aSGilad Ben-Yossef case DRV_HASH_XCBC_MAC: 2065ff27e85aSGilad Ben-Yossef cc_xcbc_authenc(req, desc, &seq_len); 2066ff27e85aSGilad Ben-Yossef break; 2067ff27e85aSGilad Ben-Yossef case DRV_HASH_NULL: 2068ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_CCM) 2069ff27e85aSGilad Ben-Yossef cc_ccm(req, desc, &seq_len); 2070ff27e85aSGilad Ben-Yossef if (ctx->cipher_mode == DRV_CIPHER_GCTR) 2071ff27e85aSGilad Ben-Yossef cc_gcm(req, desc, &seq_len); 2072ff27e85aSGilad Ben-Yossef break; 2073ff27e85aSGilad Ben-Yossef default: 2074ff27e85aSGilad Ben-Yossef dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2075ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, req); 2076ff27e85aSGilad Ben-Yossef rc = -ENOTSUPP; 2077ff27e85aSGilad Ben-Yossef goto exit; 2078ff27e85aSGilad Ben-Yossef } 2079ff27e85aSGilad Ben-Yossef 2080ff27e85aSGilad Ben-Yossef /* STAT_PHASE_3: Lock HW and push sequence */ 2081ff27e85aSGilad Ben-Yossef 2082ff27e85aSGilad Ben-Yossef rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2083ff27e85aSGilad Ben-Yossef 2084ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) { 2085ff27e85aSGilad Ben-Yossef dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2086ff27e85aSGilad Ben-Yossef cc_unmap_aead_request(dev, req); 2087ff27e85aSGilad Ben-Yossef } 2088ff27e85aSGilad Ben-Yossef 2089ff27e85aSGilad Ben-Yossef exit: 2090ff27e85aSGilad Ben-Yossef return rc; 2091ff27e85aSGilad Ben-Yossef } 2092ff27e85aSGilad Ben-Yossef 2093ff27e85aSGilad Ben-Yossef static int cc_aead_encrypt(struct aead_request *req) 2094ff27e85aSGilad Ben-Yossef { 2095ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2096ff27e85aSGilad Ben-Yossef int rc; 2097ff27e85aSGilad Ben-Yossef 20989f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 20999f31eb6eSGilad Ben-Yossef 2100ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2101ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2102ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2103ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = false; 2104ff27e85aSGilad Ben-Yossef 2105ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2106ff27e85aSGilad Ben-Yossef 2107ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2108ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2109ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2110ff27e85aSGilad Ben-Yossef 2111ff27e85aSGilad Ben-Yossef return rc; 2112ff27e85aSGilad Ben-Yossef } 2113ff27e85aSGilad Ben-Yossef 2114ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2115ff27e85aSGilad Ben-Yossef { 2116ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */ 2117ff27e85aSGilad Ben-Yossef 2118ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2119ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2120ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2121ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2122ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2123ff27e85aSGilad Ben-Yossef 2124ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2125ff27e85aSGilad Ben-Yossef dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2126ff27e85aSGilad Ben-Yossef goto out; 2127ff27e85aSGilad Ben-Yossef } 2128ff27e85aSGilad Ben-Yossef 21299f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 21309f31eb6eSGilad Ben-Yossef 2131ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2132ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2133ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2134ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2135ff27e85aSGilad Ben-Yossef 2136ff27e85aSGilad Ben-Yossef cc_proc_rfc4309_ccm(req); 2137ff27e85aSGilad Ben-Yossef 2138ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2139ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2140ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2141ff27e85aSGilad Ben-Yossef out: 2142ff27e85aSGilad Ben-Yossef return rc; 2143ff27e85aSGilad Ben-Yossef } 2144ff27e85aSGilad Ben-Yossef 2145ff27e85aSGilad Ben-Yossef static int cc_aead_decrypt(struct aead_request *req) 2146ff27e85aSGilad Ben-Yossef { 2147ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2148ff27e85aSGilad Ben-Yossef int rc; 2149ff27e85aSGilad Ben-Yossef 21509f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 21519f31eb6eSGilad Ben-Yossef 2152ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2153ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2154ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2155ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = false; 2156ff27e85aSGilad Ben-Yossef 2157ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2158ff27e85aSGilad Ben-Yossef 2159ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2160ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2161ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2162ff27e85aSGilad Ben-Yossef 2163ff27e85aSGilad Ben-Yossef return rc; 2164ff27e85aSGilad Ben-Yossef } 2165ff27e85aSGilad Ben-Yossef 2166ff27e85aSGilad Ben-Yossef static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2167ff27e85aSGilad Ben-Yossef { 2168ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2169ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2170ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2171ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2172ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2173ff27e85aSGilad Ben-Yossef 2174ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2175ff27e85aSGilad Ben-Yossef dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2176ff27e85aSGilad Ben-Yossef goto out; 2177ff27e85aSGilad Ben-Yossef } 2178ff27e85aSGilad Ben-Yossef 21799f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 21809f31eb6eSGilad Ben-Yossef 2181ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2182ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2183ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2184ff27e85aSGilad Ben-Yossef 2185ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2186ff27e85aSGilad Ben-Yossef cc_proc_rfc4309_ccm(req); 2187ff27e85aSGilad Ben-Yossef 2188ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2189ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2190ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2191ff27e85aSGilad Ben-Yossef 2192ff27e85aSGilad Ben-Yossef out: 2193ff27e85aSGilad Ben-Yossef return rc; 2194ff27e85aSGilad Ben-Yossef } 2195ff27e85aSGilad Ben-Yossef 2196ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2197ff27e85aSGilad Ben-Yossef unsigned int keylen) 2198ff27e85aSGilad Ben-Yossef { 2199ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2200ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2201ff27e85aSGilad Ben-Yossef 2202ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2203ff27e85aSGilad Ben-Yossef 2204ff27e85aSGilad Ben-Yossef if (keylen < 4) 2205ff27e85aSGilad Ben-Yossef return -EINVAL; 2206ff27e85aSGilad Ben-Yossef 2207ff27e85aSGilad Ben-Yossef keylen -= 4; 2208ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 4); 2209ff27e85aSGilad Ben-Yossef 2210ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen); 2211ff27e85aSGilad Ben-Yossef } 2212ff27e85aSGilad Ben-Yossef 2213ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2214ff27e85aSGilad Ben-Yossef unsigned int keylen) 2215ff27e85aSGilad Ben-Yossef { 2216ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2217ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2218ff27e85aSGilad Ben-Yossef 2219ff27e85aSGilad Ben-Yossef dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2220ff27e85aSGilad Ben-Yossef 2221ff27e85aSGilad Ben-Yossef if (keylen < 4) 2222ff27e85aSGilad Ben-Yossef return -EINVAL; 2223ff27e85aSGilad Ben-Yossef 2224ff27e85aSGilad Ben-Yossef keylen -= 4; 2225ff27e85aSGilad Ben-Yossef memcpy(ctx->ctr_nonce, key + keylen, 4); 2226ff27e85aSGilad Ben-Yossef 2227ff27e85aSGilad Ben-Yossef return cc_aead_setkey(tfm, key, keylen); 2228ff27e85aSGilad Ben-Yossef } 2229ff27e85aSGilad Ben-Yossef 2230ff27e85aSGilad Ben-Yossef static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2231ff27e85aSGilad Ben-Yossef unsigned int authsize) 2232ff27e85aSGilad Ben-Yossef { 2233ff27e85aSGilad Ben-Yossef switch (authsize) { 2234ff27e85aSGilad Ben-Yossef case 4: 2235ff27e85aSGilad Ben-Yossef case 8: 2236ff27e85aSGilad Ben-Yossef case 12: 2237ff27e85aSGilad Ben-Yossef case 13: 2238ff27e85aSGilad Ben-Yossef case 14: 2239ff27e85aSGilad Ben-Yossef case 15: 2240ff27e85aSGilad Ben-Yossef case 16: 2241ff27e85aSGilad Ben-Yossef break; 2242ff27e85aSGilad Ben-Yossef default: 2243ff27e85aSGilad Ben-Yossef return -EINVAL; 2244ff27e85aSGilad Ben-Yossef } 2245ff27e85aSGilad Ben-Yossef 2246ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 2247ff27e85aSGilad Ben-Yossef } 2248ff27e85aSGilad Ben-Yossef 2249ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2250ff27e85aSGilad Ben-Yossef unsigned int authsize) 2251ff27e85aSGilad Ben-Yossef { 2252ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2253ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2254ff27e85aSGilad Ben-Yossef 2255ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authsize %d\n", authsize); 2256ff27e85aSGilad Ben-Yossef 2257ff27e85aSGilad Ben-Yossef switch (authsize) { 2258ff27e85aSGilad Ben-Yossef case 8: 2259ff27e85aSGilad Ben-Yossef case 12: 2260ff27e85aSGilad Ben-Yossef case 16: 2261ff27e85aSGilad Ben-Yossef break; 2262ff27e85aSGilad Ben-Yossef default: 2263ff27e85aSGilad Ben-Yossef return -EINVAL; 2264ff27e85aSGilad Ben-Yossef } 2265ff27e85aSGilad Ben-Yossef 2266ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 2267ff27e85aSGilad Ben-Yossef } 2268ff27e85aSGilad Ben-Yossef 2269ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2270ff27e85aSGilad Ben-Yossef unsigned int authsize) 2271ff27e85aSGilad Ben-Yossef { 2272ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2273ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2274ff27e85aSGilad Ben-Yossef 2275ff27e85aSGilad Ben-Yossef dev_dbg(dev, "authsize %d\n", authsize); 2276ff27e85aSGilad Ben-Yossef 2277ff27e85aSGilad Ben-Yossef if (authsize != 16) 2278ff27e85aSGilad Ben-Yossef return -EINVAL; 2279ff27e85aSGilad Ben-Yossef 2280ff27e85aSGilad Ben-Yossef return cc_aead_setauthsize(authenc, authsize); 2281ff27e85aSGilad Ben-Yossef } 2282ff27e85aSGilad Ben-Yossef 2283ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2284ff27e85aSGilad Ben-Yossef { 2285ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */ 2286ff27e85aSGilad Ben-Yossef 2287ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2288ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2289ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2290ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2291ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2292ff27e85aSGilad Ben-Yossef 2293ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2294ff27e85aSGilad Ben-Yossef dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2295ff27e85aSGilad Ben-Yossef goto out; 2296ff27e85aSGilad Ben-Yossef } 2297ff27e85aSGilad Ben-Yossef 22989f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 22999f31eb6eSGilad Ben-Yossef 2300ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2301ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2302ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2303ff27e85aSGilad Ben-Yossef 2304ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2305ff27e85aSGilad Ben-Yossef 2306ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2307ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2308ff27e85aSGilad Ben-Yossef 2309ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2310ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2311ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2312ff27e85aSGilad Ben-Yossef out: 2313ff27e85aSGilad Ben-Yossef return rc; 2314ff27e85aSGilad Ben-Yossef } 2315ff27e85aSGilad Ben-Yossef 2316ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2317ff27e85aSGilad Ben-Yossef { 2318ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_encrypt() above. */ 2319ff27e85aSGilad Ben-Yossef 2320ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2321ff27e85aSGilad Ben-Yossef int rc; 2322ff27e85aSGilad Ben-Yossef 23239f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 23249f31eb6eSGilad Ben-Yossef 2325ff27e85aSGilad Ben-Yossef //plaintext is not encryped with rfc4543 2326ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = true; 2327ff27e85aSGilad Ben-Yossef 2328ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2329ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2330ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2331ff27e85aSGilad Ben-Yossef 2332ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2333ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2334ff27e85aSGilad Ben-Yossef 2335ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2336ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2337ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2338ff27e85aSGilad Ben-Yossef 2339ff27e85aSGilad Ben-Yossef return rc; 2340ff27e85aSGilad Ben-Yossef } 2341ff27e85aSGilad Ben-Yossef 2342ff27e85aSGilad Ben-Yossef static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2343ff27e85aSGilad Ben-Yossef { 2344ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_decrypt() above. */ 2345ff27e85aSGilad Ben-Yossef 2346ff27e85aSGilad Ben-Yossef struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2347ff27e85aSGilad Ben-Yossef struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2348ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(ctx->drvdata); 2349ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2350ff27e85aSGilad Ben-Yossef int rc = -EINVAL; 2351ff27e85aSGilad Ben-Yossef 2352ff27e85aSGilad Ben-Yossef if (!valid_assoclen(req)) { 2353ff27e85aSGilad Ben-Yossef dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2354ff27e85aSGilad Ben-Yossef goto out; 2355ff27e85aSGilad Ben-Yossef } 2356ff27e85aSGilad Ben-Yossef 23579f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 23589f31eb6eSGilad Ben-Yossef 2359ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2360ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2361ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2362ff27e85aSGilad Ben-Yossef 2363ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = false; 2364ff27e85aSGilad Ben-Yossef 2365ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2366ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2367ff27e85aSGilad Ben-Yossef 2368ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2369ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2370ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2371ff27e85aSGilad Ben-Yossef out: 2372ff27e85aSGilad Ben-Yossef return rc; 2373ff27e85aSGilad Ben-Yossef } 2374ff27e85aSGilad Ben-Yossef 2375ff27e85aSGilad Ben-Yossef static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2376ff27e85aSGilad Ben-Yossef { 2377ff27e85aSGilad Ben-Yossef /* Very similar to cc_aead_decrypt() above. */ 2378ff27e85aSGilad Ben-Yossef 2379ff27e85aSGilad Ben-Yossef struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2380ff27e85aSGilad Ben-Yossef int rc; 2381ff27e85aSGilad Ben-Yossef 23829f31eb6eSGilad Ben-Yossef memset(areq_ctx, 0, sizeof(*areq_ctx)); 23839f31eb6eSGilad Ben-Yossef 2384ff27e85aSGilad Ben-Yossef //plaintext is not decryped with rfc4543 2385ff27e85aSGilad Ben-Yossef areq_ctx->plaintext_authenticate_only = true; 2386ff27e85aSGilad Ben-Yossef 2387ff27e85aSGilad Ben-Yossef /* No generated IV required */ 2388ff27e85aSGilad Ben-Yossef areq_ctx->backup_iv = req->iv; 2389ff27e85aSGilad Ben-Yossef areq_ctx->backup_giv = NULL; 2390ff27e85aSGilad Ben-Yossef 2391ff27e85aSGilad Ben-Yossef cc_proc_rfc4_gcm(req); 2392ff27e85aSGilad Ben-Yossef areq_ctx->is_gcm4543 = true; 2393ff27e85aSGilad Ben-Yossef 2394ff27e85aSGilad Ben-Yossef rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2395ff27e85aSGilad Ben-Yossef if (rc != -EINPROGRESS && rc != -EBUSY) 2396ff27e85aSGilad Ben-Yossef req->iv = areq_ctx->backup_iv; 2397ff27e85aSGilad Ben-Yossef 2398ff27e85aSGilad Ben-Yossef return rc; 2399ff27e85aSGilad Ben-Yossef } 2400ff27e85aSGilad Ben-Yossef 2401ff27e85aSGilad Ben-Yossef /* aead alg */ 2402ff27e85aSGilad Ben-Yossef static struct cc_alg_template aead_algs[] = { 2403ff27e85aSGilad Ben-Yossef { 2404ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),cbc(aes))", 2405ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2406ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 2407ff27e85aSGilad Ben-Yossef .template_aead = { 2408ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2409ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2410ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2411ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2412ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2413ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2414ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2415ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE, 2416ff27e85aSGilad Ben-Yossef }, 2417ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2418ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2419ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1, 242027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24211c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2422ff27e85aSGilad Ben-Yossef }, 2423ff27e85aSGilad Ben-Yossef { 2424ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),cbc(des3_ede))", 2425ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2426ff27e85aSGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE, 2427ff27e85aSGilad Ben-Yossef .template_aead = { 24289fbfcefcSHerbert Xu .setkey = cc_des3_aead_setkey, 2429ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2430ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2431ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2432ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2433ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2434ff27e85aSGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE, 2435ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE, 2436ff27e85aSGilad Ben-Yossef }, 2437ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2438ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 2439ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1, 244027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24411c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2442ff27e85aSGilad Ben-Yossef }, 2443ff27e85aSGilad Ben-Yossef { 2444ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),cbc(aes))", 2445ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2446ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 2447ff27e85aSGilad Ben-Yossef .template_aead = { 2448ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2449ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2450ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2451ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2452ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2453ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2454ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2455ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE, 2456ff27e85aSGilad Ben-Yossef }, 2457ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2458ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2459ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256, 246027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24611c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2462ff27e85aSGilad Ben-Yossef }, 2463ff27e85aSGilad Ben-Yossef { 2464ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),cbc(des3_ede))", 2465ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2466ff27e85aSGilad Ben-Yossef .blocksize = DES3_EDE_BLOCK_SIZE, 2467ff27e85aSGilad Ben-Yossef .template_aead = { 24689fbfcefcSHerbert Xu .setkey = cc_des3_aead_setkey, 2469ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2470ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2471ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2472ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2473ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2474ff27e85aSGilad Ben-Yossef .ivsize = DES3_EDE_BLOCK_SIZE, 2475ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE, 2476ff27e85aSGilad Ben-Yossef }, 2477ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2478ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_DES, 2479ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256, 248027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 24811c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2482ff27e85aSGilad Ben-Yossef }, 2483ff27e85aSGilad Ben-Yossef { 2484ff27e85aSGilad Ben-Yossef .name = "authenc(xcbc(aes),cbc(aes))", 2485ff27e85aSGilad Ben-Yossef .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2486ff27e85aSGilad Ben-Yossef .blocksize = AES_BLOCK_SIZE, 2487ff27e85aSGilad Ben-Yossef .template_aead = { 2488ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2489ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2490ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2491ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2492ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2493ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2494ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2495ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2496ff27e85aSGilad Ben-Yossef }, 2497ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CBC, 2498ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2499ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_XCBC_MAC, 250027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25011c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2502ff27e85aSGilad Ben-Yossef }, 2503ff27e85aSGilad Ben-Yossef { 2504ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2505ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2506ff27e85aSGilad Ben-Yossef .blocksize = 1, 2507ff27e85aSGilad Ben-Yossef .template_aead = { 2508ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2509ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2510ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2511ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2512ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2513ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2514ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE, 2515ff27e85aSGilad Ben-Yossef .maxauthsize = SHA1_DIGEST_SIZE, 2516ff27e85aSGilad Ben-Yossef }, 2517ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 2518ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2519ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA1, 252027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25211c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2522ff27e85aSGilad Ben-Yossef }, 2523ff27e85aSGilad Ben-Yossef { 2524ff27e85aSGilad Ben-Yossef .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2525ff27e85aSGilad Ben-Yossef .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2526ff27e85aSGilad Ben-Yossef .blocksize = 1, 2527ff27e85aSGilad Ben-Yossef .template_aead = { 2528ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2529ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2530ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2531ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2532ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2533ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2534ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE, 2535ff27e85aSGilad Ben-Yossef .maxauthsize = SHA256_DIGEST_SIZE, 2536ff27e85aSGilad Ben-Yossef }, 2537ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 2538ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2539ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_SHA256, 254027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25411c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2542ff27e85aSGilad Ben-Yossef }, 2543ff27e85aSGilad Ben-Yossef { 2544ff27e85aSGilad Ben-Yossef .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2545ff27e85aSGilad Ben-Yossef .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2546ff27e85aSGilad Ben-Yossef .blocksize = 1, 2547ff27e85aSGilad Ben-Yossef .template_aead = { 2548ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2549ff27e85aSGilad Ben-Yossef .setauthsize = cc_aead_setauthsize, 2550ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2551ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2552ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2553ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2554ff27e85aSGilad Ben-Yossef .ivsize = CTR_RFC3686_IV_SIZE, 2555ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2556ff27e85aSGilad Ben-Yossef }, 2557ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CTR, 2558ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2559ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_XCBC_MAC, 256027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25611c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2562ff27e85aSGilad Ben-Yossef }, 2563ff27e85aSGilad Ben-Yossef { 2564ff27e85aSGilad Ben-Yossef .name = "ccm(aes)", 2565ff27e85aSGilad Ben-Yossef .driver_name = "ccm-aes-ccree", 2566ff27e85aSGilad Ben-Yossef .blocksize = 1, 2567ff27e85aSGilad Ben-Yossef .template_aead = { 2568ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2569ff27e85aSGilad Ben-Yossef .setauthsize = cc_ccm_setauthsize, 2570ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2571ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2572ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2573ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2574ff27e85aSGilad Ben-Yossef .ivsize = AES_BLOCK_SIZE, 2575ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2576ff27e85aSGilad Ben-Yossef }, 2577ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CCM, 2578ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2579ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 258027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 25811c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2582ff27e85aSGilad Ben-Yossef }, 2583ff27e85aSGilad Ben-Yossef { 2584ff27e85aSGilad Ben-Yossef .name = "rfc4309(ccm(aes))", 2585ff27e85aSGilad Ben-Yossef .driver_name = "rfc4309-ccm-aes-ccree", 2586ff27e85aSGilad Ben-Yossef .blocksize = 1, 2587ff27e85aSGilad Ben-Yossef .template_aead = { 2588ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4309_ccm_setkey, 2589ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4309_ccm_setauthsize, 2590ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4309_ccm_encrypt, 2591ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4309_ccm_decrypt, 2592ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2593ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2594ff27e85aSGilad Ben-Yossef .ivsize = CCM_BLOCK_IV_SIZE, 2595ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2596ff27e85aSGilad Ben-Yossef }, 2597ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_CCM, 2598ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2599ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 260027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 26011c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2602ff27e85aSGilad Ben-Yossef }, 2603ff27e85aSGilad Ben-Yossef { 2604ff27e85aSGilad Ben-Yossef .name = "gcm(aes)", 2605ff27e85aSGilad Ben-Yossef .driver_name = "gcm-aes-ccree", 2606ff27e85aSGilad Ben-Yossef .blocksize = 1, 2607ff27e85aSGilad Ben-Yossef .template_aead = { 2608ff27e85aSGilad Ben-Yossef .setkey = cc_aead_setkey, 2609ff27e85aSGilad Ben-Yossef .setauthsize = cc_gcm_setauthsize, 2610ff27e85aSGilad Ben-Yossef .encrypt = cc_aead_encrypt, 2611ff27e85aSGilad Ben-Yossef .decrypt = cc_aead_decrypt, 2612ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2613ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2614ff27e85aSGilad Ben-Yossef .ivsize = 12, 2615ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2616ff27e85aSGilad Ben-Yossef }, 2617ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR, 2618ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2619ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 262027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 26211c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2622ff27e85aSGilad Ben-Yossef }, 2623ff27e85aSGilad Ben-Yossef { 2624ff27e85aSGilad Ben-Yossef .name = "rfc4106(gcm(aes))", 2625ff27e85aSGilad Ben-Yossef .driver_name = "rfc4106-gcm-aes-ccree", 2626ff27e85aSGilad Ben-Yossef .blocksize = 1, 2627ff27e85aSGilad Ben-Yossef .template_aead = { 2628ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4106_gcm_setkey, 2629ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4106_gcm_setauthsize, 2630ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4106_gcm_encrypt, 2631ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4106_gcm_decrypt, 2632ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2633ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2634ff27e85aSGilad Ben-Yossef .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2635ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2636ff27e85aSGilad Ben-Yossef }, 2637ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR, 2638ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2639ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 264027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 26411c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2642ff27e85aSGilad Ben-Yossef }, 2643ff27e85aSGilad Ben-Yossef { 2644ff27e85aSGilad Ben-Yossef .name = "rfc4543(gcm(aes))", 2645ff27e85aSGilad Ben-Yossef .driver_name = "rfc4543-gcm-aes-ccree", 2646ff27e85aSGilad Ben-Yossef .blocksize = 1, 2647ff27e85aSGilad Ben-Yossef .template_aead = { 2648ff27e85aSGilad Ben-Yossef .setkey = cc_rfc4543_gcm_setkey, 2649ff27e85aSGilad Ben-Yossef .setauthsize = cc_rfc4543_gcm_setauthsize, 2650ff27e85aSGilad Ben-Yossef .encrypt = cc_rfc4543_gcm_encrypt, 2651ff27e85aSGilad Ben-Yossef .decrypt = cc_rfc4543_gcm_decrypt, 2652ff27e85aSGilad Ben-Yossef .init = cc_aead_init, 2653ff27e85aSGilad Ben-Yossef .exit = cc_aead_exit, 2654ff27e85aSGilad Ben-Yossef .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2655ff27e85aSGilad Ben-Yossef .maxauthsize = AES_BLOCK_SIZE, 2656ff27e85aSGilad Ben-Yossef }, 2657ff27e85aSGilad Ben-Yossef .cipher_mode = DRV_CIPHER_GCTR, 2658ff27e85aSGilad Ben-Yossef .flow_mode = S_DIN_to_AES, 2659ff27e85aSGilad Ben-Yossef .auth_mode = DRV_HASH_NULL, 266027b3b22dSGilad Ben-Yossef .min_hw_rev = CC_HW_REV_630, 26611c876a90SGilad Ben-Yossef .std_body = CC_STD_NIST, 2662ff27e85aSGilad Ben-Yossef }, 2663ff27e85aSGilad Ben-Yossef }; 2664ff27e85aSGilad Ben-Yossef 2665ff27e85aSGilad Ben-Yossef static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2666ff27e85aSGilad Ben-Yossef struct device *dev) 2667ff27e85aSGilad Ben-Yossef { 2668ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg; 2669ff27e85aSGilad Ben-Yossef struct aead_alg *alg; 2670ff27e85aSGilad Ben-Yossef 2671ff27e85aSGilad Ben-Yossef t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2672ff27e85aSGilad Ben-Yossef if (!t_alg) 2673ff27e85aSGilad Ben-Yossef return ERR_PTR(-ENOMEM); 2674ff27e85aSGilad Ben-Yossef 2675ff27e85aSGilad Ben-Yossef alg = &tmpl->template_aead; 2676ff27e85aSGilad Ben-Yossef 2677ff27e85aSGilad Ben-Yossef snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 2678ff27e85aSGilad Ben-Yossef snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2679ff27e85aSGilad Ben-Yossef tmpl->driver_name); 2680ff27e85aSGilad Ben-Yossef alg->base.cra_module = THIS_MODULE; 2681ff27e85aSGilad Ben-Yossef alg->base.cra_priority = CC_CRA_PRIO; 2682ff27e85aSGilad Ben-Yossef 2683ff27e85aSGilad Ben-Yossef alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 268476c9e53eSGilad Ben-Yossef alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2685ff27e85aSGilad Ben-Yossef alg->init = cc_aead_init; 2686ff27e85aSGilad Ben-Yossef alg->exit = cc_aead_exit; 2687ff27e85aSGilad Ben-Yossef 2688ff27e85aSGilad Ben-Yossef t_alg->aead_alg = *alg; 2689ff27e85aSGilad Ben-Yossef 2690ff27e85aSGilad Ben-Yossef t_alg->cipher_mode = tmpl->cipher_mode; 2691ff27e85aSGilad Ben-Yossef t_alg->flow_mode = tmpl->flow_mode; 2692ff27e85aSGilad Ben-Yossef t_alg->auth_mode = tmpl->auth_mode; 2693ff27e85aSGilad Ben-Yossef 2694ff27e85aSGilad Ben-Yossef return t_alg; 2695ff27e85aSGilad Ben-Yossef } 2696ff27e85aSGilad Ben-Yossef 2697ff27e85aSGilad Ben-Yossef int cc_aead_free(struct cc_drvdata *drvdata) 2698ff27e85aSGilad Ben-Yossef { 2699ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg, *n; 2700ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle = 2701ff27e85aSGilad Ben-Yossef (struct cc_aead_handle *)drvdata->aead_handle; 2702ff27e85aSGilad Ben-Yossef 2703ff27e85aSGilad Ben-Yossef if (aead_handle) { 2704ff27e85aSGilad Ben-Yossef /* Remove registered algs */ 2705ff27e85aSGilad Ben-Yossef list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, 2706ff27e85aSGilad Ben-Yossef entry) { 2707ff27e85aSGilad Ben-Yossef crypto_unregister_aead(&t_alg->aead_alg); 2708ff27e85aSGilad Ben-Yossef list_del(&t_alg->entry); 2709ff27e85aSGilad Ben-Yossef kfree(t_alg); 2710ff27e85aSGilad Ben-Yossef } 2711ff27e85aSGilad Ben-Yossef kfree(aead_handle); 2712ff27e85aSGilad Ben-Yossef drvdata->aead_handle = NULL; 2713ff27e85aSGilad Ben-Yossef } 2714ff27e85aSGilad Ben-Yossef 2715ff27e85aSGilad Ben-Yossef return 0; 2716ff27e85aSGilad Ben-Yossef } 2717ff27e85aSGilad Ben-Yossef 2718ff27e85aSGilad Ben-Yossef int cc_aead_alloc(struct cc_drvdata *drvdata) 2719ff27e85aSGilad Ben-Yossef { 2720ff27e85aSGilad Ben-Yossef struct cc_aead_handle *aead_handle; 2721ff27e85aSGilad Ben-Yossef struct cc_crypto_alg *t_alg; 2722ff27e85aSGilad Ben-Yossef int rc = -ENOMEM; 2723ff27e85aSGilad Ben-Yossef int alg; 2724ff27e85aSGilad Ben-Yossef struct device *dev = drvdata_to_dev(drvdata); 2725ff27e85aSGilad Ben-Yossef 2726ff27e85aSGilad Ben-Yossef aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL); 2727ff27e85aSGilad Ben-Yossef if (!aead_handle) { 2728ff27e85aSGilad Ben-Yossef rc = -ENOMEM; 2729ff27e85aSGilad Ben-Yossef goto fail0; 2730ff27e85aSGilad Ben-Yossef } 2731ff27e85aSGilad Ben-Yossef 2732ff27e85aSGilad Ben-Yossef INIT_LIST_HEAD(&aead_handle->aead_list); 2733ff27e85aSGilad Ben-Yossef drvdata->aead_handle = aead_handle; 2734ff27e85aSGilad Ben-Yossef 2735ff27e85aSGilad Ben-Yossef aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2736ff27e85aSGilad Ben-Yossef MAX_HMAC_DIGEST_SIZE); 2737ff27e85aSGilad Ben-Yossef 2738ff27e85aSGilad Ben-Yossef if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2739ff27e85aSGilad Ben-Yossef dev_err(dev, "SRAM pool exhausted\n"); 2740ff27e85aSGilad Ben-Yossef rc = -ENOMEM; 2741ff27e85aSGilad Ben-Yossef goto fail1; 2742ff27e85aSGilad Ben-Yossef } 2743ff27e85aSGilad Ben-Yossef 2744ff27e85aSGilad Ben-Yossef /* Linux crypto */ 2745ff27e85aSGilad Ben-Yossef for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 27461c876a90SGilad Ben-Yossef if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 27471c876a90SGilad Ben-Yossef !(drvdata->std_bodies & aead_algs[alg].std_body)) 274827b3b22dSGilad Ben-Yossef continue; 274927b3b22dSGilad Ben-Yossef 2750ff27e85aSGilad Ben-Yossef t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2751ff27e85aSGilad Ben-Yossef if (IS_ERR(t_alg)) { 2752ff27e85aSGilad Ben-Yossef rc = PTR_ERR(t_alg); 2753ff27e85aSGilad Ben-Yossef dev_err(dev, "%s alg allocation failed\n", 2754ff27e85aSGilad Ben-Yossef aead_algs[alg].driver_name); 2755ff27e85aSGilad Ben-Yossef goto fail1; 2756ff27e85aSGilad Ben-Yossef } 2757ff27e85aSGilad Ben-Yossef t_alg->drvdata = drvdata; 2758ff27e85aSGilad Ben-Yossef rc = crypto_register_aead(&t_alg->aead_alg); 2759ff27e85aSGilad Ben-Yossef if (rc) { 2760ff27e85aSGilad Ben-Yossef dev_err(dev, "%s alg registration failed\n", 2761ff27e85aSGilad Ben-Yossef t_alg->aead_alg.base.cra_driver_name); 2762ff27e85aSGilad Ben-Yossef goto fail2; 2763ff27e85aSGilad Ben-Yossef } else { 2764ff27e85aSGilad Ben-Yossef list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2765ff27e85aSGilad Ben-Yossef dev_dbg(dev, "Registered %s\n", 2766ff27e85aSGilad Ben-Yossef t_alg->aead_alg.base.cra_driver_name); 2767ff27e85aSGilad Ben-Yossef } 2768ff27e85aSGilad Ben-Yossef } 2769ff27e85aSGilad Ben-Yossef 2770ff27e85aSGilad Ben-Yossef return 0; 2771ff27e85aSGilad Ben-Yossef 2772ff27e85aSGilad Ben-Yossef fail2: 2773ff27e85aSGilad Ben-Yossef kfree(t_alg); 2774ff27e85aSGilad Ben-Yossef fail1: 2775ff27e85aSGilad Ben-Yossef cc_aead_free(drvdata); 2776ff27e85aSGilad Ben-Yossef fail0: 2777ff27e85aSGilad Ben-Yossef return rc; 2778ff27e85aSGilad Ben-Yossef } 2779