18e8ec596SKim Phillips /* 28e8ec596SKim Phillips * caam - Freescale FSL CAAM support for crypto API 38e8ec596SKim Phillips * 48e8ec596SKim Phillips * Copyright 2008-2011 Freescale Semiconductor, Inc. 58e8ec596SKim Phillips * 68e8ec596SKim Phillips * Based on talitos crypto API driver. 78e8ec596SKim Phillips * 88e8ec596SKim Phillips * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 98e8ec596SKim Phillips * 108e8ec596SKim Phillips * --------------- --------------- 118e8ec596SKim Phillips * | JobDesc #1 |-------------------->| ShareDesc | 128e8ec596SKim Phillips * | *(packet 1) | | (PDB) | 138e8ec596SKim Phillips * --------------- |------------->| (hashKey) | 148e8ec596SKim Phillips * . | | (cipherKey) | 158e8ec596SKim Phillips * . | |-------->| (operation) | 168e8ec596SKim Phillips * --------------- | | --------------- 178e8ec596SKim Phillips * | JobDesc #2 |------| | 188e8ec596SKim Phillips * | *(packet 2) | | 198e8ec596SKim Phillips * --------------- | 208e8ec596SKim Phillips * . | 218e8ec596SKim Phillips * . | 228e8ec596SKim Phillips * --------------- | 238e8ec596SKim Phillips * | JobDesc #3 |------------ 248e8ec596SKim Phillips * | *(packet 3) | 258e8ec596SKim Phillips * --------------- 268e8ec596SKim Phillips * 278e8ec596SKim Phillips * The SharedDesc never changes for a connection unless rekeyed, but 288e8ec596SKim Phillips * each packet will likely be in a different place. So all we need 298e8ec596SKim Phillips * to know to process the packet is where the input is, where the 308e8ec596SKim Phillips * output goes, and what context we want to process with. Context is 318e8ec596SKim Phillips * in the SharedDesc, packet references in the JobDesc. 328e8ec596SKim Phillips * 338e8ec596SKim Phillips * So, a job desc looks like: 348e8ec596SKim Phillips * 358e8ec596SKim Phillips * --------------------- 368e8ec596SKim Phillips * | Header | 378e8ec596SKim Phillips * | ShareDesc Pointer | 388e8ec596SKim Phillips * | SEQ_OUT_PTR | 398e8ec596SKim Phillips * | (output buffer) | 408e8ec596SKim Phillips * | SEQ_IN_PTR | 418e8ec596SKim Phillips * | (input buffer) | 428e8ec596SKim Phillips * | LOAD (to DECO) | 438e8ec596SKim Phillips * --------------------- 448e8ec596SKim Phillips */ 458e8ec596SKim Phillips 468e8ec596SKim Phillips #include "compat.h" 478e8ec596SKim Phillips 488e8ec596SKim Phillips #include "regs.h" 498e8ec596SKim Phillips #include "intern.h" 508e8ec596SKim Phillips #include "desc_constr.h" 518e8ec596SKim Phillips #include "jr.h" 528e8ec596SKim Phillips #include "error.h" 538e8ec596SKim Phillips 548e8ec596SKim Phillips /* 558e8ec596SKim Phillips * crypto alg 568e8ec596SKim Phillips */ 578e8ec596SKim Phillips #define CAAM_CRA_PRIORITY 3000 588e8ec596SKim Phillips /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 598e8ec596SKim Phillips #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 608e8ec596SKim Phillips SHA512_DIGEST_SIZE * 2) 618e8ec596SKim Phillips /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 628e8ec596SKim Phillips #define CAAM_MAX_IV_LENGTH 16 638e8ec596SKim Phillips 648e8ec596SKim Phillips #ifdef DEBUG 658e8ec596SKim Phillips /* for print_hex_dumps with line references */ 668e8ec596SKim Phillips #define xstr(s) str(s) 678e8ec596SKim Phillips #define str(s) #s 688e8ec596SKim Phillips #define debug(format, arg...) printk(format, arg) 698e8ec596SKim Phillips #else 708e8ec596SKim Phillips #define debug(format, arg...) 718e8ec596SKim Phillips #endif 728e8ec596SKim Phillips 738e8ec596SKim Phillips /* 748e8ec596SKim Phillips * per-session context 758e8ec596SKim Phillips */ 768e8ec596SKim Phillips struct caam_ctx { 778e8ec596SKim Phillips struct device *jrdev; 788e8ec596SKim Phillips u32 *sh_desc; 798e8ec596SKim Phillips dma_addr_t shared_desc_phys; 808e8ec596SKim Phillips u32 class1_alg_type; 818e8ec596SKim Phillips u32 class2_alg_type; 828e8ec596SKim Phillips u32 alg_op; 838e8ec596SKim Phillips u8 *key; 848e8ec596SKim Phillips dma_addr_t key_phys; 858e8ec596SKim Phillips unsigned int keylen; 868e8ec596SKim Phillips unsigned int enckeylen; 878e8ec596SKim Phillips unsigned int authkeylen; 888e8ec596SKim Phillips unsigned int split_key_len; 898e8ec596SKim Phillips unsigned int split_key_pad_len; 908e8ec596SKim Phillips unsigned int authsize; 918e8ec596SKim Phillips }; 928e8ec596SKim Phillips 938e8ec596SKim Phillips static int aead_authenc_setauthsize(struct crypto_aead *authenc, 948e8ec596SKim Phillips unsigned int authsize) 958e8ec596SKim Phillips { 968e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(authenc); 978e8ec596SKim Phillips 988e8ec596SKim Phillips ctx->authsize = authsize; 998e8ec596SKim Phillips 1008e8ec596SKim Phillips return 0; 1018e8ec596SKim Phillips } 1028e8ec596SKim Phillips 1038e8ec596SKim Phillips struct split_key_result { 1048e8ec596SKim Phillips struct completion completion; 1058e8ec596SKim Phillips int err; 1068e8ec596SKim Phillips }; 1078e8ec596SKim Phillips 1088e8ec596SKim Phillips static void split_key_done(struct device *dev, u32 *desc, u32 err, 1098e8ec596SKim Phillips void *context) 1108e8ec596SKim Phillips { 1118e8ec596SKim Phillips struct split_key_result *res = context; 1128e8ec596SKim Phillips 1138e8ec596SKim Phillips #ifdef DEBUG 1148e8ec596SKim Phillips dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1158e8ec596SKim Phillips #endif 1168e8ec596SKim Phillips if (err) { 1178e8ec596SKim Phillips char tmp[256]; 1188e8ec596SKim Phillips 1198e8ec596SKim Phillips dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 1208e8ec596SKim Phillips } 1218e8ec596SKim Phillips 1228e8ec596SKim Phillips res->err = err; 1238e8ec596SKim Phillips 1248e8ec596SKim Phillips complete(&res->completion); 1258e8ec596SKim Phillips } 1268e8ec596SKim Phillips 1278e8ec596SKim Phillips /* 1288e8ec596SKim Phillips get a split ipad/opad key 1298e8ec596SKim Phillips 1308e8ec596SKim Phillips Split key generation----------------------------------------------- 1318e8ec596SKim Phillips 1328e8ec596SKim Phillips [00] 0xb0810008 jobdesc: stidx=1 share=never len=8 1338e8ec596SKim Phillips [01] 0x04000014 key: class2->keyreg len=20 1348e8ec596SKim Phillips @0xffe01000 1358e8ec596SKim Phillips [03] 0x84410014 operation: cls2-op sha1 hmac init dec 1368e8ec596SKim Phillips [04] 0x24940000 fifold: class2 msgdata-last2 len=0 imm 1378e8ec596SKim Phillips [05] 0xa4000001 jump: class2 local all ->1 [06] 1388e8ec596SKim Phillips [06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 1398e8ec596SKim Phillips @0xffe04000 1408e8ec596SKim Phillips */ 1418e8ec596SKim Phillips static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen) 1428e8ec596SKim Phillips { 1438e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 1448e8ec596SKim Phillips u32 *desc; 1458e8ec596SKim Phillips struct split_key_result result; 1468e8ec596SKim Phillips dma_addr_t dma_addr_in, dma_addr_out; 1478e8ec596SKim Phillips int ret = 0; 1488e8ec596SKim Phillips 1498e8ec596SKim Phillips desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 1508e8ec596SKim Phillips 1518e8ec596SKim Phillips init_job_desc(desc, 0); 1528e8ec596SKim Phillips 1538e8ec596SKim Phillips dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen, 1548e8ec596SKim Phillips DMA_TO_DEVICE); 1558e8ec596SKim Phillips if (dma_mapping_error(jrdev, dma_addr_in)) { 1568e8ec596SKim Phillips dev_err(jrdev, "unable to map key input memory\n"); 1578e8ec596SKim Phillips kfree(desc); 1588e8ec596SKim Phillips return -ENOMEM; 1598e8ec596SKim Phillips } 1608e8ec596SKim Phillips append_key(desc, dma_addr_in, authkeylen, CLASS_2 | 1618e8ec596SKim Phillips KEY_DEST_CLASS_REG); 1628e8ec596SKim Phillips 1638e8ec596SKim Phillips /* Sets MDHA up into an HMAC-INIT */ 1648e8ec596SKim Phillips append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT | 1658e8ec596SKim Phillips OP_ALG_AS_INIT); 1668e8ec596SKim Phillips 1678e8ec596SKim Phillips /* 1688e8ec596SKim Phillips * do a FIFO_LOAD of zero, this will trigger the internal key expansion 1698e8ec596SKim Phillips into both pads inside MDHA 1708e8ec596SKim Phillips */ 1718e8ec596SKim Phillips append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB | 1728e8ec596SKim Phillips FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2); 1738e8ec596SKim Phillips 1748e8ec596SKim Phillips /* 1758e8ec596SKim Phillips * FIFO_STORE with the explicit split-key content store 1768e8ec596SKim Phillips * (0x26 output type) 1778e8ec596SKim Phillips */ 1788e8ec596SKim Phillips dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len, 1798e8ec596SKim Phillips DMA_FROM_DEVICE); 1808e8ec596SKim Phillips if (dma_mapping_error(jrdev, dma_addr_out)) { 1818e8ec596SKim Phillips dev_err(jrdev, "unable to map key output memory\n"); 1828e8ec596SKim Phillips kfree(desc); 1838e8ec596SKim Phillips return -ENOMEM; 1848e8ec596SKim Phillips } 1858e8ec596SKim Phillips append_fifo_store(desc, dma_addr_out, ctx->split_key_len, 1868e8ec596SKim Phillips LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); 1878e8ec596SKim Phillips 1888e8ec596SKim Phillips #ifdef DEBUG 1898e8ec596SKim Phillips print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", 1908e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1); 1918e8ec596SKim Phillips print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", 1928e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1938e8ec596SKim Phillips #endif 1948e8ec596SKim Phillips 1958e8ec596SKim Phillips result.err = 0; 1968e8ec596SKim Phillips init_completion(&result.completion); 1978e8ec596SKim Phillips 1988e8ec596SKim Phillips ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 1998e8ec596SKim Phillips if (!ret) { 2008e8ec596SKim Phillips /* in progress */ 2018e8ec596SKim Phillips wait_for_completion_interruptible(&result.completion); 2028e8ec596SKim Phillips ret = result.err; 2038e8ec596SKim Phillips #ifdef DEBUG 2048e8ec596SKim Phillips print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", 2058e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 2068e8ec596SKim Phillips ctx->split_key_pad_len, 1); 2078e8ec596SKim Phillips #endif 2088e8ec596SKim Phillips } 2098e8ec596SKim Phillips 2108e8ec596SKim Phillips dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len, 2118e8ec596SKim Phillips DMA_FROM_DEVICE); 2128e8ec596SKim Phillips dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE); 2138e8ec596SKim Phillips 2148e8ec596SKim Phillips kfree(desc); 2158e8ec596SKim Phillips 2168e8ec596SKim Phillips return ret; 2178e8ec596SKim Phillips } 2188e8ec596SKim Phillips 2198e8ec596SKim Phillips static int build_sh_desc_ipsec(struct caam_ctx *ctx) 2208e8ec596SKim Phillips { 2218e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 2228e8ec596SKim Phillips u32 *sh_desc; 2238e8ec596SKim Phillips u32 *jump_cmd; 2248e8ec596SKim Phillips 2258e8ec596SKim Phillips /* build shared descriptor for this session */ 2268e8ec596SKim Phillips sh_desc = kmalloc(CAAM_CMD_SZ * 4 + ctx->split_key_pad_len + 2278e8ec596SKim Phillips ctx->enckeylen, GFP_DMA | GFP_KERNEL); 2288e8ec596SKim Phillips if (!sh_desc) { 2298e8ec596SKim Phillips dev_err(jrdev, "could not allocate shared descriptor\n"); 2308e8ec596SKim Phillips return -ENOMEM; 2318e8ec596SKim Phillips } 2328e8ec596SKim Phillips 2338e8ec596SKim Phillips init_sh_desc(sh_desc, HDR_SAVECTX | HDR_SHARE_SERIAL); 2348e8ec596SKim Phillips 2358e8ec596SKim Phillips jump_cmd = append_jump(sh_desc, CLASS_BOTH | JUMP_TEST_ALL | 2368e8ec596SKim Phillips JUMP_COND_SHRD | JUMP_COND_SELF); 2378e8ec596SKim Phillips 2388e8ec596SKim Phillips /* process keys, starting with class 2/authentication */ 2398e8ec596SKim Phillips append_key_as_imm(sh_desc, ctx->key, ctx->split_key_pad_len, 2408e8ec596SKim Phillips ctx->split_key_len, 2418e8ec596SKim Phillips CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 2428e8ec596SKim Phillips 2438e8ec596SKim Phillips append_key_as_imm(sh_desc, (void *)ctx->key + ctx->split_key_pad_len, 2448e8ec596SKim Phillips ctx->enckeylen, ctx->enckeylen, 2458e8ec596SKim Phillips CLASS_1 | KEY_DEST_CLASS_REG); 2468e8ec596SKim Phillips 2478e8ec596SKim Phillips /* update jump cmd now that we are at the jump target */ 2488e8ec596SKim Phillips set_jump_tgt_here(sh_desc, jump_cmd); 2498e8ec596SKim Phillips 2508e8ec596SKim Phillips ctx->shared_desc_phys = dma_map_single(jrdev, sh_desc, 2518e8ec596SKim Phillips desc_bytes(sh_desc), 2528e8ec596SKim Phillips DMA_TO_DEVICE); 2538e8ec596SKim Phillips if (dma_mapping_error(jrdev, ctx->shared_desc_phys)) { 2548e8ec596SKim Phillips dev_err(jrdev, "unable to map shared descriptor\n"); 2558e8ec596SKim Phillips kfree(sh_desc); 2568e8ec596SKim Phillips return -ENOMEM; 2578e8ec596SKim Phillips } 2588e8ec596SKim Phillips 2598e8ec596SKim Phillips ctx->sh_desc = sh_desc; 2608e8ec596SKim Phillips 2618e8ec596SKim Phillips return 0; 2628e8ec596SKim Phillips } 2638e8ec596SKim Phillips 2648e8ec596SKim Phillips static int aead_authenc_setkey(struct crypto_aead *aead, 2658e8ec596SKim Phillips const u8 *key, unsigned int keylen) 2668e8ec596SKim Phillips { 2678e8ec596SKim Phillips /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 2688e8ec596SKim Phillips static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 2698e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 2708e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 2718e8ec596SKim Phillips struct rtattr *rta = (void *)key; 2728e8ec596SKim Phillips struct crypto_authenc_key_param *param; 2738e8ec596SKim Phillips unsigned int authkeylen; 2748e8ec596SKim Phillips unsigned int enckeylen; 2758e8ec596SKim Phillips int ret = 0; 2768e8ec596SKim Phillips 2778e8ec596SKim Phillips param = RTA_DATA(rta); 2788e8ec596SKim Phillips enckeylen = be32_to_cpu(param->enckeylen); 2798e8ec596SKim Phillips 2808e8ec596SKim Phillips key += RTA_ALIGN(rta->rta_len); 2818e8ec596SKim Phillips keylen -= RTA_ALIGN(rta->rta_len); 2828e8ec596SKim Phillips 2838e8ec596SKim Phillips if (keylen < enckeylen) 2848e8ec596SKim Phillips goto badkey; 2858e8ec596SKim Phillips 2868e8ec596SKim Phillips authkeylen = keylen - enckeylen; 2878e8ec596SKim Phillips 2888e8ec596SKim Phillips if (keylen > CAAM_MAX_KEY_SIZE) 2898e8ec596SKim Phillips goto badkey; 2908e8ec596SKim Phillips 2918e8ec596SKim Phillips /* Pick class 2 key length from algorithm submask */ 2928e8ec596SKim Phillips ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 2938e8ec596SKim Phillips OP_ALG_ALGSEL_SHIFT] * 2; 2948e8ec596SKim Phillips ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 2958e8ec596SKim Phillips 2968e8ec596SKim Phillips #ifdef DEBUG 2978e8ec596SKim Phillips printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 2988e8ec596SKim Phillips keylen, enckeylen, authkeylen); 2998e8ec596SKim Phillips printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 3008e8ec596SKim Phillips ctx->split_key_len, ctx->split_key_pad_len); 3018e8ec596SKim Phillips print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", 3028e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 3038e8ec596SKim Phillips #endif 3048e8ec596SKim Phillips ctx->key = kmalloc(ctx->split_key_pad_len + enckeylen, 3058e8ec596SKim Phillips GFP_KERNEL | GFP_DMA); 3068e8ec596SKim Phillips if (!ctx->key) { 3078e8ec596SKim Phillips dev_err(jrdev, "could not allocate key output memory\n"); 3088e8ec596SKim Phillips return -ENOMEM; 3098e8ec596SKim Phillips } 3108e8ec596SKim Phillips 3118e8ec596SKim Phillips ret = gen_split_key(ctx, key, authkeylen); 3128e8ec596SKim Phillips if (ret) { 3138e8ec596SKim Phillips kfree(ctx->key); 3148e8ec596SKim Phillips goto badkey; 3158e8ec596SKim Phillips } 3168e8ec596SKim Phillips 3178e8ec596SKim Phillips /* postpend encryption key to auth split key */ 3188e8ec596SKim Phillips memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); 3198e8ec596SKim Phillips 3208e8ec596SKim Phillips ctx->key_phys = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 3218e8ec596SKim Phillips enckeylen, DMA_TO_DEVICE); 3228e8ec596SKim Phillips if (dma_mapping_error(jrdev, ctx->key_phys)) { 3238e8ec596SKim Phillips dev_err(jrdev, "unable to map key i/o memory\n"); 3248e8ec596SKim Phillips kfree(ctx->key); 3258e8ec596SKim Phillips return -ENOMEM; 3268e8ec596SKim Phillips } 3278e8ec596SKim Phillips #ifdef DEBUG 3288e8ec596SKim Phillips print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", 3298e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 3308e8ec596SKim Phillips ctx->split_key_pad_len + enckeylen, 1); 3318e8ec596SKim Phillips #endif 3328e8ec596SKim Phillips 3338e8ec596SKim Phillips ctx->keylen = keylen; 3348e8ec596SKim Phillips ctx->enckeylen = enckeylen; 3358e8ec596SKim Phillips ctx->authkeylen = authkeylen; 3368e8ec596SKim Phillips 3378e8ec596SKim Phillips ret = build_sh_desc_ipsec(ctx); 3388e8ec596SKim Phillips if (ret) { 3398e8ec596SKim Phillips dma_unmap_single(jrdev, ctx->key_phys, ctx->split_key_pad_len + 3408e8ec596SKim Phillips enckeylen, DMA_TO_DEVICE); 3418e8ec596SKim Phillips kfree(ctx->key); 3428e8ec596SKim Phillips } 3438e8ec596SKim Phillips 3448e8ec596SKim Phillips return ret; 3458e8ec596SKim Phillips badkey: 3468e8ec596SKim Phillips crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 3478e8ec596SKim Phillips return -EINVAL; 3488e8ec596SKim Phillips } 3498e8ec596SKim Phillips 3508e8ec596SKim Phillips struct link_tbl_entry { 3518e8ec596SKim Phillips u64 ptr; 3528e8ec596SKim Phillips u32 len; 3538e8ec596SKim Phillips u8 reserved; 3548e8ec596SKim Phillips u8 buf_pool_id; 3558e8ec596SKim Phillips u16 offset; 3568e8ec596SKim Phillips }; 3578e8ec596SKim Phillips 3588e8ec596SKim Phillips /* 3598e8ec596SKim Phillips * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor 3608e8ec596SKim Phillips * @src_nents: number of segments in input scatterlist 3618e8ec596SKim Phillips * @dst_nents: number of segments in output scatterlist 3628e8ec596SKim Phillips * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 3638e8ec596SKim Phillips * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 3648e8ec596SKim Phillips * @link_tbl_bytes: length of dma mapped link_tbl space 3658e8ec596SKim Phillips * @link_tbl_dma: bus physical mapped address of h/w link table 3668e8ec596SKim Phillips * @hw_desc: the h/w job descriptor followed by any referenced link tables 3678e8ec596SKim Phillips */ 3688e8ec596SKim Phillips struct ipsec_esp_edesc { 3698e8ec596SKim Phillips int assoc_nents; 3708e8ec596SKim Phillips int src_nents; 3718e8ec596SKim Phillips int dst_nents; 3728e8ec596SKim Phillips int link_tbl_bytes; 3738e8ec596SKim Phillips dma_addr_t link_tbl_dma; 3748e8ec596SKim Phillips struct link_tbl_entry *link_tbl; 3758e8ec596SKim Phillips u32 hw_desc[0]; 3768e8ec596SKim Phillips }; 3778e8ec596SKim Phillips 3788e8ec596SKim Phillips static void ipsec_esp_unmap(struct device *dev, 3798e8ec596SKim Phillips struct ipsec_esp_edesc *edesc, 3808e8ec596SKim Phillips struct aead_request *areq) 3818e8ec596SKim Phillips { 3828e8ec596SKim Phillips dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents, DMA_TO_DEVICE); 3838e8ec596SKim Phillips 3848e8ec596SKim Phillips if (unlikely(areq->dst != areq->src)) { 3858e8ec596SKim Phillips dma_unmap_sg(dev, areq->src, edesc->src_nents, 3868e8ec596SKim Phillips DMA_TO_DEVICE); 3878e8ec596SKim Phillips dma_unmap_sg(dev, areq->dst, edesc->dst_nents, 3888e8ec596SKim Phillips DMA_FROM_DEVICE); 3898e8ec596SKim Phillips } else { 3908e8ec596SKim Phillips dma_unmap_sg(dev, areq->src, edesc->src_nents, 3918e8ec596SKim Phillips DMA_BIDIRECTIONAL); 3928e8ec596SKim Phillips } 3938e8ec596SKim Phillips 3948e8ec596SKim Phillips if (edesc->link_tbl_bytes) 3958e8ec596SKim Phillips dma_unmap_single(dev, edesc->link_tbl_dma, 3968e8ec596SKim Phillips edesc->link_tbl_bytes, 3978e8ec596SKim Phillips DMA_TO_DEVICE); 3988e8ec596SKim Phillips } 3998e8ec596SKim Phillips 4008e8ec596SKim Phillips /* 4018e8ec596SKim Phillips * ipsec_esp descriptor callbacks 4028e8ec596SKim Phillips */ 4038e8ec596SKim Phillips static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 4048e8ec596SKim Phillips void *context) 4058e8ec596SKim Phillips { 4068e8ec596SKim Phillips struct aead_request *areq = context; 4078e8ec596SKim Phillips struct ipsec_esp_edesc *edesc; 4088e8ec596SKim Phillips #ifdef DEBUG 4098e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 4108e8ec596SKim Phillips int ivsize = crypto_aead_ivsize(aead); 4118e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 4128e8ec596SKim Phillips 4138e8ec596SKim Phillips dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 4148e8ec596SKim Phillips #endif 4158e8ec596SKim Phillips edesc = (struct ipsec_esp_edesc *)((char *)desc - 4168e8ec596SKim Phillips offsetof(struct ipsec_esp_edesc, hw_desc)); 4178e8ec596SKim Phillips 4188e8ec596SKim Phillips if (err) { 4198e8ec596SKim Phillips char tmp[256]; 4208e8ec596SKim Phillips 4218e8ec596SKim Phillips dev_err(jrdev, "%s\n", caam_jr_strstatus(tmp, err)); 4228e8ec596SKim Phillips dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 4238e8ec596SKim Phillips } 4248e8ec596SKim Phillips 4258e8ec596SKim Phillips ipsec_esp_unmap(jrdev, edesc, areq); 4268e8ec596SKim Phillips 4278e8ec596SKim Phillips #ifdef DEBUG 4288e8ec596SKim Phillips print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 4298e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc), 4308e8ec596SKim Phillips areq->assoclen , 1); 4318e8ec596SKim Phillips print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", 4328e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize, 4338e8ec596SKim Phillips edesc->src_nents ? 100 : ivsize, 1); 4348e8ec596SKim Phillips print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", 4358e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src), 4368e8ec596SKim Phillips edesc->src_nents ? 100 : areq->cryptlen + 4378e8ec596SKim Phillips ctx->authsize + 4, 1); 4388e8ec596SKim Phillips #endif 4398e8ec596SKim Phillips 4408e8ec596SKim Phillips kfree(edesc); 4418e8ec596SKim Phillips 4428e8ec596SKim Phillips aead_request_complete(areq, err); 4438e8ec596SKim Phillips } 4448e8ec596SKim Phillips 4458e8ec596SKim Phillips static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 4468e8ec596SKim Phillips void *context) 4478e8ec596SKim Phillips { 4488e8ec596SKim Phillips struct aead_request *areq = context; 4498e8ec596SKim Phillips struct ipsec_esp_edesc *edesc; 4508e8ec596SKim Phillips #ifdef DEBUG 4518e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 4528e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 4538e8ec596SKim Phillips 4548e8ec596SKim Phillips dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 4558e8ec596SKim Phillips #endif 4568e8ec596SKim Phillips edesc = (struct ipsec_esp_edesc *)((char *)desc - 4578e8ec596SKim Phillips offsetof(struct ipsec_esp_edesc, hw_desc)); 4588e8ec596SKim Phillips 4598e8ec596SKim Phillips if (err) { 4608e8ec596SKim Phillips char tmp[256]; 4618e8ec596SKim Phillips 4628e8ec596SKim Phillips dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 4638e8ec596SKim Phillips } 4648e8ec596SKim Phillips 4658e8ec596SKim Phillips ipsec_esp_unmap(jrdev, edesc, areq); 4668e8ec596SKim Phillips 4678e8ec596SKim Phillips /* 4688e8ec596SKim Phillips * verify hw auth check passed else return -EBADMSG 4698e8ec596SKim Phillips */ 4708e8ec596SKim Phillips if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 4718e8ec596SKim Phillips err = -EBADMSG; 4728e8ec596SKim Phillips 4738e8ec596SKim Phillips #ifdef DEBUG 4748e8ec596SKim Phillips print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ", 4758e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, 4768e8ec596SKim Phillips ((char *)sg_virt(areq->assoc) - sizeof(struct iphdr)), 4778e8ec596SKim Phillips sizeof(struct iphdr) + areq->assoclen + 4788e8ec596SKim Phillips ((areq->cryptlen > 1500) ? 1500 : areq->cryptlen) + 4798e8ec596SKim Phillips ctx->authsize + 36, 1); 4808e8ec596SKim Phillips if (!err && edesc->link_tbl_bytes) { 4818e8ec596SKim Phillips struct scatterlist *sg = sg_last(areq->src, edesc->src_nents); 4828e8ec596SKim Phillips print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ", 4838e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), 4848e8ec596SKim Phillips sg->length + ctx->authsize + 16, 1); 4858e8ec596SKim Phillips } 4868e8ec596SKim Phillips #endif 4878e8ec596SKim Phillips kfree(edesc); 4888e8ec596SKim Phillips 4898e8ec596SKim Phillips aead_request_complete(areq, err); 4908e8ec596SKim Phillips } 4918e8ec596SKim Phillips 4928e8ec596SKim Phillips /* 4938e8ec596SKim Phillips * convert scatterlist to h/w link table format 4948e8ec596SKim Phillips * scatterlist must have been previously dma mapped 4958e8ec596SKim Phillips */ 4968e8ec596SKim Phillips static void sg_to_link_tbl(struct scatterlist *sg, int sg_count, 4978e8ec596SKim Phillips struct link_tbl_entry *link_tbl_ptr, u32 offset) 4988e8ec596SKim Phillips { 4998e8ec596SKim Phillips while (sg_count) { 5008e8ec596SKim Phillips link_tbl_ptr->ptr = sg_dma_address(sg); 5018e8ec596SKim Phillips link_tbl_ptr->len = sg_dma_len(sg); 5028e8ec596SKim Phillips link_tbl_ptr->reserved = 0; 5038e8ec596SKim Phillips link_tbl_ptr->buf_pool_id = 0; 5048e8ec596SKim Phillips link_tbl_ptr->offset = offset; 5058e8ec596SKim Phillips link_tbl_ptr++; 5068e8ec596SKim Phillips sg = sg_next(sg); 5078e8ec596SKim Phillips sg_count--; 5088e8ec596SKim Phillips } 5098e8ec596SKim Phillips 5108e8ec596SKim Phillips /* set Final bit (marks end of link table) */ 5118e8ec596SKim Phillips link_tbl_ptr--; 5128e8ec596SKim Phillips link_tbl_ptr->len |= 0x40000000; 5138e8ec596SKim Phillips } 5148e8ec596SKim Phillips 5158e8ec596SKim Phillips /* 5168e8ec596SKim Phillips * fill in and submit ipsec_esp job descriptor 5178e8ec596SKim Phillips */ 5188e8ec596SKim Phillips static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, 5198e8ec596SKim Phillips u32 encrypt, 5208e8ec596SKim Phillips void (*callback) (struct device *dev, u32 *desc, 5218e8ec596SKim Phillips u32 err, void *context)) 5228e8ec596SKim Phillips { 5238e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 5248e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 5258e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 5268e8ec596SKim Phillips u32 *desc = edesc->hw_desc, options; 5278e8ec596SKim Phillips int ret, sg_count, assoc_sg_count; 5288e8ec596SKim Phillips int ivsize = crypto_aead_ivsize(aead); 5298e8ec596SKim Phillips int authsize = ctx->authsize; 5308e8ec596SKim Phillips dma_addr_t ptr, dst_dma, src_dma; 5318e8ec596SKim Phillips #ifdef DEBUG 5328e8ec596SKim Phillips u32 *sh_desc = ctx->sh_desc; 5338e8ec596SKim Phillips 5348e8ec596SKim Phillips debug("assoclen %d cryptlen %d authsize %d\n", 5358e8ec596SKim Phillips areq->assoclen, areq->cryptlen, authsize); 5368e8ec596SKim Phillips print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 5378e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc), 5388e8ec596SKim Phillips areq->assoclen , 1); 5398e8ec596SKim Phillips print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", 5408e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize, 5418e8ec596SKim Phillips edesc->src_nents ? 100 : ivsize, 1); 5428e8ec596SKim Phillips print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", 5438e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src), 5448e8ec596SKim Phillips edesc->src_nents ? 100 : areq->cryptlen + authsize, 1); 5458e8ec596SKim Phillips print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", 5468e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 5478e8ec596SKim Phillips desc_bytes(sh_desc), 1); 5488e8ec596SKim Phillips #endif 5498e8ec596SKim Phillips assoc_sg_count = dma_map_sg(jrdev, areq->assoc, edesc->assoc_nents ?: 1, 5508e8ec596SKim Phillips DMA_TO_DEVICE); 5518e8ec596SKim Phillips if (areq->src == areq->dst) 5528e8ec596SKim Phillips sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1, 5538e8ec596SKim Phillips DMA_BIDIRECTIONAL); 5548e8ec596SKim Phillips else 5558e8ec596SKim Phillips sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1, 5568e8ec596SKim Phillips DMA_TO_DEVICE); 5578e8ec596SKim Phillips 5588e8ec596SKim Phillips /* start auth operation */ 5598e8ec596SKim Phillips append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | 5608e8ec596SKim Phillips (encrypt ? : OP_ALG_ICV_ON)); 5618e8ec596SKim Phillips 5628e8ec596SKim Phillips /* Load FIFO with data for Class 2 CHA */ 5638e8ec596SKim Phillips options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG; 5648e8ec596SKim Phillips if (!edesc->assoc_nents) { 5658e8ec596SKim Phillips ptr = sg_dma_address(areq->assoc); 5668e8ec596SKim Phillips } else { 5678e8ec596SKim Phillips sg_to_link_tbl(areq->assoc, edesc->assoc_nents, 5688e8ec596SKim Phillips edesc->link_tbl, 0); 5698e8ec596SKim Phillips ptr = edesc->link_tbl_dma; 5708e8ec596SKim Phillips options |= LDST_SGF; 5718e8ec596SKim Phillips } 5728e8ec596SKim Phillips append_fifo_load(desc, ptr, areq->assoclen, options); 5738e8ec596SKim Phillips 5748e8ec596SKim Phillips /* copy iv from cipher/class1 input context to class2 infifo */ 5758e8ec596SKim Phillips append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); 5768e8ec596SKim Phillips 5778e8ec596SKim Phillips /* start class 1 (cipher) operation */ 5788e8ec596SKim Phillips append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | 5798e8ec596SKim Phillips encrypt); 5808e8ec596SKim Phillips 5818e8ec596SKim Phillips /* load payload & instruct to class2 to snoop class 1 if encrypting */ 5828e8ec596SKim Phillips options = 0; 5838e8ec596SKim Phillips if (!edesc->src_nents) { 5848e8ec596SKim Phillips src_dma = sg_dma_address(areq->src); 5858e8ec596SKim Phillips } else { 5868e8ec596SKim Phillips sg_to_link_tbl(areq->src, edesc->src_nents, edesc->link_tbl + 5878e8ec596SKim Phillips edesc->assoc_nents, 0); 5888e8ec596SKim Phillips src_dma = edesc->link_tbl_dma + edesc->assoc_nents * 5898e8ec596SKim Phillips sizeof(struct link_tbl_entry); 5908e8ec596SKim Phillips options |= LDST_SGF; 5918e8ec596SKim Phillips } 5928e8ec596SKim Phillips append_seq_in_ptr(desc, src_dma, areq->cryptlen + authsize, options); 5938e8ec596SKim Phillips append_seq_fifo_load(desc, areq->cryptlen, FIFOLD_CLASS_BOTH | 5948e8ec596SKim Phillips FIFOLD_TYPE_LASTBOTH | 5958e8ec596SKim Phillips (encrypt ? FIFOLD_TYPE_MSG1OUT2 5968e8ec596SKim Phillips : FIFOLD_TYPE_MSG)); 5978e8ec596SKim Phillips 5988e8ec596SKim Phillips /* specify destination */ 5998e8ec596SKim Phillips if (areq->src == areq->dst) { 6008e8ec596SKim Phillips dst_dma = src_dma; 6018e8ec596SKim Phillips } else { 6028e8ec596SKim Phillips sg_count = dma_map_sg(jrdev, areq->dst, edesc->dst_nents ? : 1, 6038e8ec596SKim Phillips DMA_FROM_DEVICE); 6048e8ec596SKim Phillips if (!edesc->dst_nents) { 6058e8ec596SKim Phillips dst_dma = sg_dma_address(areq->dst); 6068e8ec596SKim Phillips options = 0; 6078e8ec596SKim Phillips } else { 6088e8ec596SKim Phillips sg_to_link_tbl(areq->dst, edesc->dst_nents, 6098e8ec596SKim Phillips edesc->link_tbl + edesc->assoc_nents + 6108e8ec596SKim Phillips edesc->src_nents, 0); 6118e8ec596SKim Phillips dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents + 6128e8ec596SKim Phillips edesc->src_nents) * 6138e8ec596SKim Phillips sizeof(struct link_tbl_entry); 6148e8ec596SKim Phillips options = LDST_SGF; 6158e8ec596SKim Phillips } 6168e8ec596SKim Phillips } 6178e8ec596SKim Phillips append_seq_out_ptr(desc, dst_dma, areq->cryptlen + authsize, options); 6188e8ec596SKim Phillips append_seq_fifo_store(desc, areq->cryptlen, FIFOST_TYPE_MESSAGE_DATA); 6198e8ec596SKim Phillips 6208e8ec596SKim Phillips /* ICV */ 6218e8ec596SKim Phillips if (encrypt) 6228e8ec596SKim Phillips append_seq_store(desc, authsize, LDST_CLASS_2_CCB | 6238e8ec596SKim Phillips LDST_SRCDST_BYTE_CONTEXT); 6248e8ec596SKim Phillips else 6258e8ec596SKim Phillips append_seq_fifo_load(desc, authsize, FIFOLD_CLASS_CLASS2 | 6268e8ec596SKim Phillips FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 6278e8ec596SKim Phillips 6288e8ec596SKim Phillips #ifdef DEBUG 6298e8ec596SKim Phillips debug("job_desc_len %d\n", desc_len(desc)); 6308e8ec596SKim Phillips print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", 6318e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc) , 1); 6328e8ec596SKim Phillips print_hex_dump(KERN_ERR, "jdlinkt@"xstr(__LINE__)": ", 6338e8ec596SKim Phillips DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl, 6348e8ec596SKim Phillips edesc->link_tbl_bytes, 1); 6358e8ec596SKim Phillips #endif 6368e8ec596SKim Phillips 6378e8ec596SKim Phillips ret = caam_jr_enqueue(jrdev, desc, callback, areq); 6388e8ec596SKim Phillips if (!ret) 6398e8ec596SKim Phillips ret = -EINPROGRESS; 6408e8ec596SKim Phillips else { 6418e8ec596SKim Phillips ipsec_esp_unmap(jrdev, edesc, areq); 6428e8ec596SKim Phillips kfree(edesc); 6438e8ec596SKim Phillips } 6448e8ec596SKim Phillips 6458e8ec596SKim Phillips return ret; 6468e8ec596SKim Phillips } 6478e8ec596SKim Phillips 6488e8ec596SKim Phillips /* 6498e8ec596SKim Phillips * derive number of elements in scatterlist 6508e8ec596SKim Phillips */ 6518e8ec596SKim Phillips static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained) 6528e8ec596SKim Phillips { 6538e8ec596SKim Phillips struct scatterlist *sg = sg_list; 6548e8ec596SKim Phillips int sg_nents = 0; 6558e8ec596SKim Phillips 6568e8ec596SKim Phillips *chained = 0; 6578e8ec596SKim Phillips while (nbytes > 0) { 6588e8ec596SKim Phillips sg_nents++; 6598e8ec596SKim Phillips nbytes -= sg->length; 6608e8ec596SKim Phillips if (!sg_is_last(sg) && (sg + 1)->length == 0) 6618e8ec596SKim Phillips *chained = 1; 6628e8ec596SKim Phillips sg = scatterwalk_sg_next(sg); 6638e8ec596SKim Phillips } 6648e8ec596SKim Phillips 6658e8ec596SKim Phillips return sg_nents; 6668e8ec596SKim Phillips } 6678e8ec596SKim Phillips 6688e8ec596SKim Phillips /* 6698e8ec596SKim Phillips * allocate and map the ipsec_esp extended descriptor 6708e8ec596SKim Phillips */ 6718e8ec596SKim Phillips static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, 6728e8ec596SKim Phillips int desc_bytes) 6738e8ec596SKim Phillips { 6748e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 6758e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 6768e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 6778e8ec596SKim Phillips gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 6788e8ec596SKim Phillips GFP_ATOMIC; 6798e8ec596SKim Phillips int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes; 6808e8ec596SKim Phillips struct ipsec_esp_edesc *edesc; 6818e8ec596SKim Phillips 6828e8ec596SKim Phillips assoc_nents = sg_count(areq->assoc, areq->assoclen, &chained); 6838e8ec596SKim Phillips BUG_ON(chained); 6848e8ec596SKim Phillips if (likely(assoc_nents == 1)) 6858e8ec596SKim Phillips assoc_nents = 0; 6868e8ec596SKim Phillips 6878e8ec596SKim Phillips src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize, 6888e8ec596SKim Phillips &chained); 6898e8ec596SKim Phillips BUG_ON(chained); 6908e8ec596SKim Phillips if (src_nents == 1) 6918e8ec596SKim Phillips src_nents = 0; 6928e8ec596SKim Phillips 6938e8ec596SKim Phillips if (unlikely(areq->dst != areq->src)) { 6948e8ec596SKim Phillips dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize, 6958e8ec596SKim Phillips &chained); 6968e8ec596SKim Phillips BUG_ON(chained); 6978e8ec596SKim Phillips if (dst_nents == 1) 6988e8ec596SKim Phillips dst_nents = 0; 6998e8ec596SKim Phillips } 7008e8ec596SKim Phillips 7018e8ec596SKim Phillips link_tbl_bytes = (assoc_nents + src_nents + dst_nents) * 7028e8ec596SKim Phillips sizeof(struct link_tbl_entry); 7038e8ec596SKim Phillips debug("link_tbl_bytes %d\n", link_tbl_bytes); 7048e8ec596SKim Phillips 7058e8ec596SKim Phillips /* allocate space for base edesc and hw desc commands, link tables */ 7068e8ec596SKim Phillips edesc = kmalloc(sizeof(struct ipsec_esp_edesc) + desc_bytes + 7078e8ec596SKim Phillips link_tbl_bytes, GFP_DMA | flags); 7088e8ec596SKim Phillips if (!edesc) { 7098e8ec596SKim Phillips dev_err(jrdev, "could not allocate extended descriptor\n"); 7108e8ec596SKim Phillips return ERR_PTR(-ENOMEM); 7118e8ec596SKim Phillips } 7128e8ec596SKim Phillips 7138e8ec596SKim Phillips edesc->assoc_nents = assoc_nents; 7148e8ec596SKim Phillips edesc->src_nents = src_nents; 7158e8ec596SKim Phillips edesc->dst_nents = dst_nents; 7168e8ec596SKim Phillips edesc->link_tbl = (void *)edesc + sizeof(struct ipsec_esp_edesc) + 7178e8ec596SKim Phillips desc_bytes; 7188e8ec596SKim Phillips edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, 7198e8ec596SKim Phillips link_tbl_bytes, DMA_TO_DEVICE); 7208e8ec596SKim Phillips edesc->link_tbl_bytes = link_tbl_bytes; 7218e8ec596SKim Phillips 7228e8ec596SKim Phillips return edesc; 7238e8ec596SKim Phillips } 7248e8ec596SKim Phillips 7258e8ec596SKim Phillips static int aead_authenc_encrypt(struct aead_request *areq) 7268e8ec596SKim Phillips { 7278e8ec596SKim Phillips struct ipsec_esp_edesc *edesc; 7288e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 7298e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 7308e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 7318e8ec596SKim Phillips int ivsize = crypto_aead_ivsize(aead); 7328e8ec596SKim Phillips u32 *desc; 7338e8ec596SKim Phillips dma_addr_t iv_dma; 7348e8ec596SKim Phillips 7358e8ec596SKim Phillips /* allocate extended descriptor */ 7368e8ec596SKim Phillips edesc = ipsec_esp_edesc_alloc(areq, 21 * sizeof(u32)); 7378e8ec596SKim Phillips if (IS_ERR(edesc)) 7388e8ec596SKim Phillips return PTR_ERR(edesc); 7398e8ec596SKim Phillips 7408e8ec596SKim Phillips desc = edesc->hw_desc; 7418e8ec596SKim Phillips 7428e8ec596SKim Phillips /* insert shared descriptor pointer */ 7438e8ec596SKim Phillips init_job_desc_shared(desc, ctx->shared_desc_phys, 7448e8ec596SKim Phillips desc_len(ctx->sh_desc), HDR_SHARE_DEFER); 7458e8ec596SKim Phillips 7468e8ec596SKim Phillips iv_dma = dma_map_single(jrdev, areq->iv, ivsize, DMA_TO_DEVICE); 7478e8ec596SKim Phillips /* check dma error */ 7488e8ec596SKim Phillips 7498e8ec596SKim Phillips append_load(desc, iv_dma, ivsize, 7508e8ec596SKim Phillips LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); 7518e8ec596SKim Phillips 7528e8ec596SKim Phillips return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done); 7538e8ec596SKim Phillips } 7548e8ec596SKim Phillips 7558e8ec596SKim Phillips static int aead_authenc_decrypt(struct aead_request *req) 7568e8ec596SKim Phillips { 7578e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(req); 7588e8ec596SKim Phillips int ivsize = crypto_aead_ivsize(aead); 7598e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 7608e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 7618e8ec596SKim Phillips struct ipsec_esp_edesc *edesc; 7628e8ec596SKim Phillips u32 *desc; 7638e8ec596SKim Phillips dma_addr_t iv_dma; 7648e8ec596SKim Phillips 7658e8ec596SKim Phillips req->cryptlen -= ctx->authsize; 7668e8ec596SKim Phillips 7678e8ec596SKim Phillips /* allocate extended descriptor */ 7688e8ec596SKim Phillips edesc = ipsec_esp_edesc_alloc(req, 21 * sizeof(u32)); 7698e8ec596SKim Phillips if (IS_ERR(edesc)) 7708e8ec596SKim Phillips return PTR_ERR(edesc); 7718e8ec596SKim Phillips 7728e8ec596SKim Phillips desc = edesc->hw_desc; 7738e8ec596SKim Phillips 7748e8ec596SKim Phillips /* insert shared descriptor pointer */ 7758e8ec596SKim Phillips init_job_desc_shared(desc, ctx->shared_desc_phys, 7768e8ec596SKim Phillips desc_len(ctx->sh_desc), HDR_SHARE_DEFER); 7778e8ec596SKim Phillips 7788e8ec596SKim Phillips iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); 7798e8ec596SKim Phillips /* check dma error */ 7808e8ec596SKim Phillips 7818e8ec596SKim Phillips append_load(desc, iv_dma, ivsize, 7828e8ec596SKim Phillips LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); 7838e8ec596SKim Phillips 7848e8ec596SKim Phillips return ipsec_esp(edesc, req, !OP_ALG_ENCRYPT, ipsec_esp_decrypt_done); 7858e8ec596SKim Phillips } 7868e8ec596SKim Phillips 7878e8ec596SKim Phillips static int aead_authenc_givencrypt(struct aead_givcrypt_request *req) 7888e8ec596SKim Phillips { 7898e8ec596SKim Phillips struct aead_request *areq = &req->areq; 7908e8ec596SKim Phillips struct ipsec_esp_edesc *edesc; 7918e8ec596SKim Phillips struct crypto_aead *aead = crypto_aead_reqtfm(areq); 7928e8ec596SKim Phillips struct caam_ctx *ctx = crypto_aead_ctx(aead); 7938e8ec596SKim Phillips struct device *jrdev = ctx->jrdev; 7948e8ec596SKim Phillips int ivsize = crypto_aead_ivsize(aead); 7958e8ec596SKim Phillips dma_addr_t iv_dma; 7968e8ec596SKim Phillips u32 *desc; 7978e8ec596SKim Phillips 7988e8ec596SKim Phillips iv_dma = dma_map_single(jrdev, req->giv, ivsize, DMA_FROM_DEVICE); 7998e8ec596SKim Phillips 8008e8ec596SKim Phillips debug("%s: giv %p\n", __func__, req->giv); 8018e8ec596SKim Phillips 8028e8ec596SKim Phillips /* allocate extended descriptor */ 8038e8ec596SKim Phillips edesc = ipsec_esp_edesc_alloc(areq, 27 * sizeof(u32)); 8048e8ec596SKim Phillips if (IS_ERR(edesc)) 8058e8ec596SKim Phillips return PTR_ERR(edesc); 8068e8ec596SKim Phillips 8078e8ec596SKim Phillips desc = edesc->hw_desc; 8088e8ec596SKim Phillips 8098e8ec596SKim Phillips /* insert shared descriptor pointer */ 8108e8ec596SKim Phillips init_job_desc_shared(desc, ctx->shared_desc_phys, 8118e8ec596SKim Phillips desc_len(ctx->sh_desc), HDR_SHARE_DEFER); 8128e8ec596SKim Phillips 8138e8ec596SKim Phillips /* 8148e8ec596SKim Phillips * LOAD IMM Info FIFO 8158e8ec596SKim Phillips * to DECO, Last, Padding, Random, Message, 16 bytes 8168e8ec596SKim Phillips */ 8178e8ec596SKim Phillips append_load_imm_u32(desc, NFIFOENTRY_DEST_DECO | NFIFOENTRY_LC1 | 8188e8ec596SKim Phillips NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG | 8198e8ec596SKim Phillips NFIFOENTRY_PTYPE_RND | ivsize, 8208e8ec596SKim Phillips LDST_SRCDST_WORD_INFO_FIFO); 8218e8ec596SKim Phillips 8228e8ec596SKim Phillips /* 8238e8ec596SKim Phillips * disable info fifo entries since the above serves as the entry 8248e8ec596SKim Phillips * this way, the MOVE command won't generate an entry. 8258e8ec596SKim Phillips * Note that this isn't required in more recent versions of 8268e8ec596SKim Phillips * SEC as a MOVE that doesn't do info FIFO entries is available. 8278e8ec596SKim Phillips */ 8288e8ec596SKim Phillips append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 8298e8ec596SKim Phillips 8308e8ec596SKim Phillips /* MOVE DECO Alignment -> C1 Context 16 bytes */ 8318e8ec596SKim Phillips append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | 8328e8ec596SKim Phillips MOVE_DEST_CLASS1CTX | ivsize); 8338e8ec596SKim Phillips 8348e8ec596SKim Phillips /* re-enable info fifo entries */ 8358e8ec596SKim Phillips append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 8368e8ec596SKim Phillips 8378e8ec596SKim Phillips /* MOVE C1 Context -> OFIFO 16 bytes */ 8388e8ec596SKim Phillips append_move(desc, MOVE_WAITCOMP | MOVE_SRC_CLASS1CTX | 8398e8ec596SKim Phillips MOVE_DEST_OUTFIFO | ivsize); 8408e8ec596SKim Phillips 8418e8ec596SKim Phillips append_fifo_store(desc, iv_dma, ivsize, FIFOST_TYPE_MESSAGE_DATA); 8428e8ec596SKim Phillips 8438e8ec596SKim Phillips return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done); 8448e8ec596SKim Phillips } 8458e8ec596SKim Phillips 8468e8ec596SKim Phillips struct caam_alg_template { 8478e8ec596SKim Phillips char name[CRYPTO_MAX_ALG_NAME]; 8488e8ec596SKim Phillips char driver_name[CRYPTO_MAX_ALG_NAME]; 8498e8ec596SKim Phillips unsigned int blocksize; 8508e8ec596SKim Phillips struct aead_alg aead; 8518e8ec596SKim Phillips u32 class1_alg_type; 8528e8ec596SKim Phillips u32 class2_alg_type; 8538e8ec596SKim Phillips u32 alg_op; 8548e8ec596SKim Phillips }; 8558e8ec596SKim Phillips 8568e8ec596SKim Phillips static struct caam_alg_template driver_algs[] = { 8578e8ec596SKim Phillips /* single-pass ipsec_esp descriptor */ 8588e8ec596SKim Phillips { 8598e8ec596SKim Phillips .name = "authenc(hmac(sha1),cbc(aes))", 8608e8ec596SKim Phillips .driver_name = "authenc-hmac-sha1-cbc-aes-caam", 8618e8ec596SKim Phillips .blocksize = AES_BLOCK_SIZE, 8628e8ec596SKim Phillips .aead = { 8638e8ec596SKim Phillips .setkey = aead_authenc_setkey, 8648e8ec596SKim Phillips .setauthsize = aead_authenc_setauthsize, 8658e8ec596SKim Phillips .encrypt = aead_authenc_encrypt, 8668e8ec596SKim Phillips .decrypt = aead_authenc_decrypt, 8678e8ec596SKim Phillips .givencrypt = aead_authenc_givencrypt, 8688e8ec596SKim Phillips .geniv = "<built-in>", 8698e8ec596SKim Phillips .ivsize = AES_BLOCK_SIZE, 8708e8ec596SKim Phillips .maxauthsize = SHA1_DIGEST_SIZE, 8718e8ec596SKim Phillips }, 8728e8ec596SKim Phillips .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 8738e8ec596SKim Phillips .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 8748e8ec596SKim Phillips .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 8758e8ec596SKim Phillips }, 8768e8ec596SKim Phillips { 8778e8ec596SKim Phillips .name = "authenc(hmac(sha256),cbc(aes))", 8788e8ec596SKim Phillips .driver_name = "authenc-hmac-sha256-cbc-aes-caam", 8798e8ec596SKim Phillips .blocksize = AES_BLOCK_SIZE, 8808e8ec596SKim Phillips .aead = { 8818e8ec596SKim Phillips .setkey = aead_authenc_setkey, 8828e8ec596SKim Phillips .setauthsize = aead_authenc_setauthsize, 8838e8ec596SKim Phillips .encrypt = aead_authenc_encrypt, 8848e8ec596SKim Phillips .decrypt = aead_authenc_decrypt, 8858e8ec596SKim Phillips .givencrypt = aead_authenc_givencrypt, 8868e8ec596SKim Phillips .geniv = "<built-in>", 8878e8ec596SKim Phillips .ivsize = AES_BLOCK_SIZE, 8888e8ec596SKim Phillips .maxauthsize = SHA256_DIGEST_SIZE, 8898e8ec596SKim Phillips }, 8908e8ec596SKim Phillips .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 8918e8ec596SKim Phillips .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 8928e8ec596SKim Phillips OP_ALG_AAI_HMAC_PRECOMP, 8938e8ec596SKim Phillips .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 8948e8ec596SKim Phillips }, 8958e8ec596SKim Phillips { 8968e8ec596SKim Phillips .name = "authenc(hmac(sha1),cbc(des3_ede))", 8978e8ec596SKim Phillips .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", 8988e8ec596SKim Phillips .blocksize = DES3_EDE_BLOCK_SIZE, 8998e8ec596SKim Phillips .aead = { 9008e8ec596SKim Phillips .setkey = aead_authenc_setkey, 9018e8ec596SKim Phillips .setauthsize = aead_authenc_setauthsize, 9028e8ec596SKim Phillips .encrypt = aead_authenc_encrypt, 9038e8ec596SKim Phillips .decrypt = aead_authenc_decrypt, 9048e8ec596SKim Phillips .givencrypt = aead_authenc_givencrypt, 9058e8ec596SKim Phillips .geniv = "<built-in>", 9068e8ec596SKim Phillips .ivsize = DES3_EDE_BLOCK_SIZE, 9078e8ec596SKim Phillips .maxauthsize = SHA1_DIGEST_SIZE, 9088e8ec596SKim Phillips }, 9098e8ec596SKim Phillips .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 9108e8ec596SKim Phillips .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 9118e8ec596SKim Phillips .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 9128e8ec596SKim Phillips }, 9138e8ec596SKim Phillips { 9148e8ec596SKim Phillips .name = "authenc(hmac(sha256),cbc(des3_ede))", 9158e8ec596SKim Phillips .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", 9168e8ec596SKim Phillips .blocksize = DES3_EDE_BLOCK_SIZE, 9178e8ec596SKim Phillips .aead = { 9188e8ec596SKim Phillips .setkey = aead_authenc_setkey, 9198e8ec596SKim Phillips .setauthsize = aead_authenc_setauthsize, 9208e8ec596SKim Phillips .encrypt = aead_authenc_encrypt, 9218e8ec596SKim Phillips .decrypt = aead_authenc_decrypt, 9228e8ec596SKim Phillips .givencrypt = aead_authenc_givencrypt, 9238e8ec596SKim Phillips .geniv = "<built-in>", 9248e8ec596SKim Phillips .ivsize = DES3_EDE_BLOCK_SIZE, 9258e8ec596SKim Phillips .maxauthsize = SHA256_DIGEST_SIZE, 9268e8ec596SKim Phillips }, 9278e8ec596SKim Phillips .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 9288e8ec596SKim Phillips .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 9298e8ec596SKim Phillips OP_ALG_AAI_HMAC_PRECOMP, 9308e8ec596SKim Phillips .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 9318e8ec596SKim Phillips }, 9328e8ec596SKim Phillips { 9338e8ec596SKim Phillips .name = "authenc(hmac(sha1),cbc(des))", 9348e8ec596SKim Phillips .driver_name = "authenc-hmac-sha1-cbc-des-caam", 9358e8ec596SKim Phillips .blocksize = DES_BLOCK_SIZE, 9368e8ec596SKim Phillips .aead = { 9378e8ec596SKim Phillips .setkey = aead_authenc_setkey, 9388e8ec596SKim Phillips .setauthsize = aead_authenc_setauthsize, 9398e8ec596SKim Phillips .encrypt = aead_authenc_encrypt, 9408e8ec596SKim Phillips .decrypt = aead_authenc_decrypt, 9418e8ec596SKim Phillips .givencrypt = aead_authenc_givencrypt, 9428e8ec596SKim Phillips .geniv = "<built-in>", 9438e8ec596SKim Phillips .ivsize = DES_BLOCK_SIZE, 9448e8ec596SKim Phillips .maxauthsize = SHA1_DIGEST_SIZE, 9458e8ec596SKim Phillips }, 9468e8ec596SKim Phillips .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 9478e8ec596SKim Phillips .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 9488e8ec596SKim Phillips .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 9498e8ec596SKim Phillips }, 9508e8ec596SKim Phillips { 9518e8ec596SKim Phillips .name = "authenc(hmac(sha256),cbc(des))", 9528e8ec596SKim Phillips .driver_name = "authenc-hmac-sha256-cbc-des-caam", 9538e8ec596SKim Phillips .blocksize = DES_BLOCK_SIZE, 9548e8ec596SKim Phillips .aead = { 9558e8ec596SKim Phillips .setkey = aead_authenc_setkey, 9568e8ec596SKim Phillips .setauthsize = aead_authenc_setauthsize, 9578e8ec596SKim Phillips .encrypt = aead_authenc_encrypt, 9588e8ec596SKim Phillips .decrypt = aead_authenc_decrypt, 9598e8ec596SKim Phillips .givencrypt = aead_authenc_givencrypt, 9608e8ec596SKim Phillips .geniv = "<built-in>", 9618e8ec596SKim Phillips .ivsize = DES_BLOCK_SIZE, 9628e8ec596SKim Phillips .maxauthsize = SHA256_DIGEST_SIZE, 9638e8ec596SKim Phillips }, 9648e8ec596SKim Phillips .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 9658e8ec596SKim Phillips .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 9668e8ec596SKim Phillips OP_ALG_AAI_HMAC_PRECOMP, 9678e8ec596SKim Phillips .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 9688e8ec596SKim Phillips }, 9698e8ec596SKim Phillips }; 9708e8ec596SKim Phillips 9718e8ec596SKim Phillips struct caam_crypto_alg { 9728e8ec596SKim Phillips struct list_head entry; 9738e8ec596SKim Phillips struct device *ctrldev; 9748e8ec596SKim Phillips int class1_alg_type; 9758e8ec596SKim Phillips int class2_alg_type; 9768e8ec596SKim Phillips int alg_op; 9778e8ec596SKim Phillips struct crypto_alg crypto_alg; 9788e8ec596SKim Phillips }; 9798e8ec596SKim Phillips 9808e8ec596SKim Phillips static int caam_cra_init(struct crypto_tfm *tfm) 9818e8ec596SKim Phillips { 9828e8ec596SKim Phillips struct crypto_alg *alg = tfm->__crt_alg; 9838e8ec596SKim Phillips struct caam_crypto_alg *caam_alg = 9848e8ec596SKim Phillips container_of(alg, struct caam_crypto_alg, crypto_alg); 9858e8ec596SKim Phillips struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 9868e8ec596SKim Phillips struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev); 9878e8ec596SKim Phillips int tgt_jr = atomic_inc_return(&priv->tfm_count); 9888e8ec596SKim Phillips 9898e8ec596SKim Phillips /* 9908e8ec596SKim Phillips * distribute tfms across job rings to ensure in-order 9918e8ec596SKim Phillips * crypto request processing per tfm 9928e8ec596SKim Phillips */ 9938e8ec596SKim Phillips ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi]; 9948e8ec596SKim Phillips 9958e8ec596SKim Phillips /* copy descriptor header template value */ 9968e8ec596SKim Phillips ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; 9978e8ec596SKim Phillips ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type; 9988e8ec596SKim Phillips ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op; 9998e8ec596SKim Phillips 10008e8ec596SKim Phillips return 0; 10018e8ec596SKim Phillips } 10028e8ec596SKim Phillips 10038e8ec596SKim Phillips static void caam_cra_exit(struct crypto_tfm *tfm) 10048e8ec596SKim Phillips { 10058e8ec596SKim Phillips struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 10068e8ec596SKim Phillips 10078e8ec596SKim Phillips if (!dma_mapping_error(ctx->jrdev, ctx->shared_desc_phys)) 10088e8ec596SKim Phillips dma_unmap_single(ctx->jrdev, ctx->shared_desc_phys, 10098e8ec596SKim Phillips desc_bytes(ctx->sh_desc), DMA_TO_DEVICE); 10108e8ec596SKim Phillips kfree(ctx->sh_desc); 10118e8ec596SKim Phillips } 10128e8ec596SKim Phillips 10138e8ec596SKim Phillips static void __exit caam_algapi_exit(void) 10148e8ec596SKim Phillips { 10158e8ec596SKim Phillips 10168e8ec596SKim Phillips struct device_node *dev_node; 10178e8ec596SKim Phillips struct platform_device *pdev; 10188e8ec596SKim Phillips struct device *ctrldev; 10198e8ec596SKim Phillips struct caam_drv_private *priv; 10208e8ec596SKim Phillips struct caam_crypto_alg *t_alg, *n; 10218e8ec596SKim Phillips int i, err; 10228e8ec596SKim Phillips 102354e198d4SKim Phillips dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 10248e8ec596SKim Phillips if (!dev_node) 10258e8ec596SKim Phillips return; 10268e8ec596SKim Phillips 10278e8ec596SKim Phillips pdev = of_find_device_by_node(dev_node); 10288e8ec596SKim Phillips if (!pdev) 10298e8ec596SKim Phillips return; 10308e8ec596SKim Phillips 10318e8ec596SKim Phillips ctrldev = &pdev->dev; 10328e8ec596SKim Phillips of_node_put(dev_node); 10338e8ec596SKim Phillips priv = dev_get_drvdata(ctrldev); 10348e8ec596SKim Phillips 10358e8ec596SKim Phillips if (!priv->alg_list.next) 10368e8ec596SKim Phillips return; 10378e8ec596SKim Phillips 10388e8ec596SKim Phillips list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 10398e8ec596SKim Phillips crypto_unregister_alg(&t_alg->crypto_alg); 10408e8ec596SKim Phillips list_del(&t_alg->entry); 10418e8ec596SKim Phillips kfree(t_alg); 10428e8ec596SKim Phillips } 10438e8ec596SKim Phillips 10448e8ec596SKim Phillips for (i = 0; i < priv->total_jobrs; i++) { 10458e8ec596SKim Phillips err = caam_jr_deregister(priv->algapi_jr[i]); 10468e8ec596SKim Phillips if (err < 0) 10478e8ec596SKim Phillips break; 10488e8ec596SKim Phillips } 10498e8ec596SKim Phillips kfree(priv->algapi_jr); 10508e8ec596SKim Phillips } 10518e8ec596SKim Phillips 10528e8ec596SKim Phillips static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, 10538e8ec596SKim Phillips struct caam_alg_template 10548e8ec596SKim Phillips *template) 10558e8ec596SKim Phillips { 10568e8ec596SKim Phillips struct caam_crypto_alg *t_alg; 10578e8ec596SKim Phillips struct crypto_alg *alg; 10588e8ec596SKim Phillips 10598e8ec596SKim Phillips t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); 10608e8ec596SKim Phillips if (!t_alg) { 10618e8ec596SKim Phillips dev_err(ctrldev, "failed to allocate t_alg\n"); 10628e8ec596SKim Phillips return ERR_PTR(-ENOMEM); 10638e8ec596SKim Phillips } 10648e8ec596SKim Phillips 10658e8ec596SKim Phillips alg = &t_alg->crypto_alg; 10668e8ec596SKim Phillips 10678e8ec596SKim Phillips snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 10688e8ec596SKim Phillips snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 10698e8ec596SKim Phillips template->driver_name); 10708e8ec596SKim Phillips alg->cra_module = THIS_MODULE; 10718e8ec596SKim Phillips alg->cra_init = caam_cra_init; 10728e8ec596SKim Phillips alg->cra_exit = caam_cra_exit; 10738e8ec596SKim Phillips alg->cra_priority = CAAM_CRA_PRIORITY; 10748e8ec596SKim Phillips alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; 10758e8ec596SKim Phillips alg->cra_blocksize = template->blocksize; 10768e8ec596SKim Phillips alg->cra_alignmask = 0; 10778e8ec596SKim Phillips alg->cra_type = &crypto_aead_type; 10788e8ec596SKim Phillips alg->cra_ctxsize = sizeof(struct caam_ctx); 10798e8ec596SKim Phillips alg->cra_u.aead = template->aead; 10808e8ec596SKim Phillips 10818e8ec596SKim Phillips t_alg->class1_alg_type = template->class1_alg_type; 10828e8ec596SKim Phillips t_alg->class2_alg_type = template->class2_alg_type; 10838e8ec596SKim Phillips t_alg->alg_op = template->alg_op; 10848e8ec596SKim Phillips t_alg->ctrldev = ctrldev; 10858e8ec596SKim Phillips 10868e8ec596SKim Phillips return t_alg; 10878e8ec596SKim Phillips } 10888e8ec596SKim Phillips 10898e8ec596SKim Phillips static int __init caam_algapi_init(void) 10908e8ec596SKim Phillips { 10918e8ec596SKim Phillips struct device_node *dev_node; 10928e8ec596SKim Phillips struct platform_device *pdev; 10938e8ec596SKim Phillips struct device *ctrldev, **jrdev; 10948e8ec596SKim Phillips struct caam_drv_private *priv; 10958e8ec596SKim Phillips int i = 0, err = 0; 10968e8ec596SKim Phillips 109754e198d4SKim Phillips dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 10988e8ec596SKim Phillips if (!dev_node) 10998e8ec596SKim Phillips return -ENODEV; 11008e8ec596SKim Phillips 11018e8ec596SKim Phillips pdev = of_find_device_by_node(dev_node); 11028e8ec596SKim Phillips if (!pdev) 11038e8ec596SKim Phillips return -ENODEV; 11048e8ec596SKim Phillips 11058e8ec596SKim Phillips ctrldev = &pdev->dev; 11068e8ec596SKim Phillips priv = dev_get_drvdata(ctrldev); 11078e8ec596SKim Phillips of_node_put(dev_node); 11088e8ec596SKim Phillips 11098e8ec596SKim Phillips INIT_LIST_HEAD(&priv->alg_list); 11108e8ec596SKim Phillips 11118e8ec596SKim Phillips jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_KERNEL); 11128e8ec596SKim Phillips if (!jrdev) 11138e8ec596SKim Phillips return -ENOMEM; 11148e8ec596SKim Phillips 11158e8ec596SKim Phillips for (i = 0; i < priv->total_jobrs; i++) { 11168e8ec596SKim Phillips err = caam_jr_register(ctrldev, &jrdev[i]); 11178e8ec596SKim Phillips if (err < 0) 11188e8ec596SKim Phillips break; 11198e8ec596SKim Phillips } 11208e8ec596SKim Phillips if (err < 0 && i == 0) { 11218e8ec596SKim Phillips dev_err(ctrldev, "algapi error in job ring registration: %d\n", 11228e8ec596SKim Phillips err); 11238e8ec596SKim Phillips return err; 11248e8ec596SKim Phillips } 11258e8ec596SKim Phillips 11268e8ec596SKim Phillips priv->num_jrs_for_algapi = i; 11278e8ec596SKim Phillips priv->algapi_jr = jrdev; 11288e8ec596SKim Phillips atomic_set(&priv->tfm_count, -1); 11298e8ec596SKim Phillips 11308e8ec596SKim Phillips /* register crypto algorithms the device supports */ 11318e8ec596SKim Phillips for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 11328e8ec596SKim Phillips /* TODO: check if h/w supports alg */ 11338e8ec596SKim Phillips struct caam_crypto_alg *t_alg; 11348e8ec596SKim Phillips 11358e8ec596SKim Phillips t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]); 11368e8ec596SKim Phillips if (IS_ERR(t_alg)) { 11378e8ec596SKim Phillips err = PTR_ERR(t_alg); 11388e8ec596SKim Phillips dev_warn(ctrldev, "%s alg allocation failed\n", 1139cdc712d8SDan Carpenter driver_algs[i].driver_name); 11408e8ec596SKim Phillips continue; 11418e8ec596SKim Phillips } 11428e8ec596SKim Phillips 11438e8ec596SKim Phillips err = crypto_register_alg(&t_alg->crypto_alg); 11448e8ec596SKim Phillips if (err) { 11458e8ec596SKim Phillips dev_warn(ctrldev, "%s alg registration failed\n", 11468e8ec596SKim Phillips t_alg->crypto_alg.cra_driver_name); 11478e8ec596SKim Phillips kfree(t_alg); 11488e8ec596SKim Phillips } else { 11498e8ec596SKim Phillips list_add_tail(&t_alg->entry, &priv->alg_list); 11508e8ec596SKim Phillips dev_info(ctrldev, "%s\n", 11518e8ec596SKim Phillips t_alg->crypto_alg.cra_driver_name); 11528e8ec596SKim Phillips } 11538e8ec596SKim Phillips } 11548e8ec596SKim Phillips 11558e8ec596SKim Phillips return err; 11568e8ec596SKim Phillips } 11578e8ec596SKim Phillips 11588e8ec596SKim Phillips module_init(caam_algapi_init); 11598e8ec596SKim Phillips module_exit(caam_algapi_exit); 11608e8ec596SKim Phillips 11618e8ec596SKim Phillips MODULE_LICENSE("GPL"); 11628e8ec596SKim Phillips MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 11638e8ec596SKim Phillips MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 1164