1ce0183cbSArd Biesheuvel // SPDX-License-Identifier: GPL-2.0-only 2ce0183cbSArd Biesheuvel /* 3ce0183cbSArd Biesheuvel * Crypto acceleration support for Rockchip RK3288 4ce0183cbSArd Biesheuvel * 5ce0183cbSArd Biesheuvel * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd 6ce0183cbSArd Biesheuvel * 7ce0183cbSArd Biesheuvel * Author: Zain Wang <zain.wang@rock-chips.com> 8ce0183cbSArd Biesheuvel * 9ce0183cbSArd Biesheuvel * Some ideas are from marvell-cesa.c and s5p-sss.c driver. 10ce0183cbSArd Biesheuvel */ 110c3dc787SHerbert Xu #include <linux/device.h> 1257d67c6eSCorentin Labbe #include <crypto/scatterwalk.h> 13ce0183cbSArd Biesheuvel #include "rk3288_crypto.h" 14ce0183cbSArd Biesheuvel 15ce0183cbSArd Biesheuvel #define RK_CRYPTO_DEC BIT(0) 16ce0183cbSArd Biesheuvel 1768ef8af0SCorentin Labbe static int rk_cipher_need_fallback(struct skcipher_request *req) 1868ef8af0SCorentin Labbe { 1968ef8af0SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 2068ef8af0SCorentin Labbe unsigned int bs = crypto_skcipher_blocksize(tfm); 2148d904d4SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2248d904d4SCorentin Labbe struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher); 2368ef8af0SCorentin Labbe struct scatterlist *sgs, *sgd; 2468ef8af0SCorentin Labbe unsigned int stodo, dtodo, len; 2568ef8af0SCorentin Labbe 2668ef8af0SCorentin Labbe if (!req->cryptlen) 2768ef8af0SCorentin Labbe return true; 2868ef8af0SCorentin Labbe 2968ef8af0SCorentin Labbe len = req->cryptlen; 3068ef8af0SCorentin Labbe sgs = req->src; 3168ef8af0SCorentin Labbe sgd = req->dst; 3268ef8af0SCorentin Labbe while (sgs && sgd) { 3368ef8af0SCorentin Labbe if (!IS_ALIGNED(sgs->offset, sizeof(u32))) { 3448d904d4SCorentin Labbe algt->stat_fb_align++; 3568ef8af0SCorentin Labbe return true; 3668ef8af0SCorentin Labbe } 3768ef8af0SCorentin Labbe if (!IS_ALIGNED(sgd->offset, sizeof(u32))) { 3848d904d4SCorentin Labbe algt->stat_fb_align++; 3968ef8af0SCorentin Labbe return true; 4068ef8af0SCorentin Labbe } 4168ef8af0SCorentin Labbe stodo = min(len, sgs->length); 4268ef8af0SCorentin Labbe if (stodo % bs) { 4348d904d4SCorentin Labbe algt->stat_fb_len++; 4468ef8af0SCorentin Labbe return true; 4568ef8af0SCorentin Labbe } 4668ef8af0SCorentin Labbe dtodo = min(len, sgd->length); 4768ef8af0SCorentin Labbe if (dtodo % bs) { 4848d904d4SCorentin Labbe algt->stat_fb_len++; 4968ef8af0SCorentin Labbe return true; 5068ef8af0SCorentin Labbe } 5168ef8af0SCorentin Labbe if (stodo != dtodo) { 5248d904d4SCorentin Labbe algt->stat_fb_sgdiff++; 5368ef8af0SCorentin Labbe return true; 5468ef8af0SCorentin Labbe } 5568ef8af0SCorentin Labbe len -= stodo; 5668ef8af0SCorentin Labbe sgs = sg_next(sgs); 5768ef8af0SCorentin Labbe sgd = sg_next(sgd); 5868ef8af0SCorentin Labbe } 5968ef8af0SCorentin Labbe return false; 6068ef8af0SCorentin Labbe } 6168ef8af0SCorentin Labbe 6268ef8af0SCorentin Labbe static int rk_cipher_fallback(struct skcipher_request *areq) 6368ef8af0SCorentin Labbe { 6468ef8af0SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 6568ef8af0SCorentin Labbe struct rk_cipher_ctx *op = crypto_skcipher_ctx(tfm); 6668ef8af0SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq); 6748d904d4SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 6848d904d4SCorentin Labbe struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher); 6968ef8af0SCorentin Labbe int err; 7068ef8af0SCorentin Labbe 7148d904d4SCorentin Labbe algt->stat_fb++; 7248d904d4SCorentin Labbe 7368ef8af0SCorentin Labbe skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); 7468ef8af0SCorentin Labbe skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, 7568ef8af0SCorentin Labbe areq->base.complete, areq->base.data); 7668ef8af0SCorentin Labbe skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, 7768ef8af0SCorentin Labbe areq->cryptlen, areq->iv); 7868ef8af0SCorentin Labbe if (rctx->mode & RK_CRYPTO_DEC) 7968ef8af0SCorentin Labbe err = crypto_skcipher_decrypt(&rctx->fallback_req); 8068ef8af0SCorentin Labbe else 8168ef8af0SCorentin Labbe err = crypto_skcipher_encrypt(&rctx->fallback_req); 8268ef8af0SCorentin Labbe return err; 8368ef8af0SCorentin Labbe } 8468ef8af0SCorentin Labbe 852e3b1495SCorentin Labbe static int rk_cipher_handle_req(struct skcipher_request *req) 86ce0183cbSArd Biesheuvel { 872e3b1495SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 882e3b1495SCorentin Labbe struct rk_cipher_ctx *tctx = crypto_skcipher_ctx(tfm); 892e3b1495SCorentin Labbe struct rk_crypto_info *rkc = tctx->dev; 902e3b1495SCorentin Labbe struct crypto_engine *engine = rkc->engine; 9157d67c6eSCorentin Labbe 9268ef8af0SCorentin Labbe if (rk_cipher_need_fallback(req)) 9368ef8af0SCorentin Labbe return rk_cipher_fallback(req); 9468ef8af0SCorentin Labbe 9557d67c6eSCorentin Labbe return crypto_transfer_skcipher_request_to_engine(engine, req); 96ce0183cbSArd Biesheuvel } 97ce0183cbSArd Biesheuvel 98ce0183cbSArd Biesheuvel static int rk_aes_setkey(struct crypto_skcipher *cipher, 99ce0183cbSArd Biesheuvel const u8 *key, unsigned int keylen) 100ce0183cbSArd Biesheuvel { 101ce0183cbSArd Biesheuvel struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 102ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 103ce0183cbSArd Biesheuvel 104ce0183cbSArd Biesheuvel if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 105674f368aSEric Biggers keylen != AES_KEYSIZE_256) 106ce0183cbSArd Biesheuvel return -EINVAL; 107ce0183cbSArd Biesheuvel ctx->keylen = keylen; 108d6b23cceSCorentin Labbe memcpy(ctx->key, key, keylen); 10968ef8af0SCorentin Labbe 11068ef8af0SCorentin Labbe return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 111ce0183cbSArd Biesheuvel } 112ce0183cbSArd Biesheuvel 113ce0183cbSArd Biesheuvel static int rk_des_setkey(struct crypto_skcipher *cipher, 114ce0183cbSArd Biesheuvel const u8 *key, unsigned int keylen) 115ce0183cbSArd Biesheuvel { 116ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); 117ce0183cbSArd Biesheuvel int err; 118ce0183cbSArd Biesheuvel 119ce0183cbSArd Biesheuvel err = verify_skcipher_des_key(cipher, key); 120ce0183cbSArd Biesheuvel if (err) 121ce0183cbSArd Biesheuvel return err; 122ce0183cbSArd Biesheuvel 123ce0183cbSArd Biesheuvel ctx->keylen = keylen; 124d6b23cceSCorentin Labbe memcpy(ctx->key, key, keylen); 12568ef8af0SCorentin Labbe 12668ef8af0SCorentin Labbe return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 127ce0183cbSArd Biesheuvel } 128ce0183cbSArd Biesheuvel 129ce0183cbSArd Biesheuvel static int rk_tdes_setkey(struct crypto_skcipher *cipher, 130ce0183cbSArd Biesheuvel const u8 *key, unsigned int keylen) 131ce0183cbSArd Biesheuvel { 132ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); 133ce0183cbSArd Biesheuvel int err; 134ce0183cbSArd Biesheuvel 135ce0183cbSArd Biesheuvel err = verify_skcipher_des3_key(cipher, key); 136ce0183cbSArd Biesheuvel if (err) 137ce0183cbSArd Biesheuvel return err; 138ce0183cbSArd Biesheuvel 139ce0183cbSArd Biesheuvel ctx->keylen = keylen; 140d6b23cceSCorentin Labbe memcpy(ctx->key, key, keylen); 141d6b23cceSCorentin Labbe 14268ef8af0SCorentin Labbe return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); 143ce0183cbSArd Biesheuvel } 144ce0183cbSArd Biesheuvel 145ce0183cbSArd Biesheuvel static int rk_aes_ecb_encrypt(struct skcipher_request *req) 146ce0183cbSArd Biesheuvel { 14787e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 148ce0183cbSArd Biesheuvel 14987e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_AES_ECB_MODE; 1502e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 151ce0183cbSArd Biesheuvel } 152ce0183cbSArd Biesheuvel 153ce0183cbSArd Biesheuvel static int rk_aes_ecb_decrypt(struct skcipher_request *req) 154ce0183cbSArd Biesheuvel { 15587e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 156ce0183cbSArd Biesheuvel 15787e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; 1582e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 159ce0183cbSArd Biesheuvel } 160ce0183cbSArd Biesheuvel 161ce0183cbSArd Biesheuvel static int rk_aes_cbc_encrypt(struct skcipher_request *req) 162ce0183cbSArd Biesheuvel { 16387e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 164ce0183cbSArd Biesheuvel 16587e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_AES_CBC_MODE; 1662e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 167ce0183cbSArd Biesheuvel } 168ce0183cbSArd Biesheuvel 169ce0183cbSArd Biesheuvel static int rk_aes_cbc_decrypt(struct skcipher_request *req) 170ce0183cbSArd Biesheuvel { 17187e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 172ce0183cbSArd Biesheuvel 17387e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; 1742e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 175ce0183cbSArd Biesheuvel } 176ce0183cbSArd Biesheuvel 177ce0183cbSArd Biesheuvel static int rk_des_ecb_encrypt(struct skcipher_request *req) 178ce0183cbSArd Biesheuvel { 17987e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 180ce0183cbSArd Biesheuvel 18187e356c4SCorentin Labbe rctx->mode = 0; 1822e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 183ce0183cbSArd Biesheuvel } 184ce0183cbSArd Biesheuvel 185ce0183cbSArd Biesheuvel static int rk_des_ecb_decrypt(struct skcipher_request *req) 186ce0183cbSArd Biesheuvel { 18787e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 188ce0183cbSArd Biesheuvel 18987e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_DEC; 1902e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 191ce0183cbSArd Biesheuvel } 192ce0183cbSArd Biesheuvel 193ce0183cbSArd Biesheuvel static int rk_des_cbc_encrypt(struct skcipher_request *req) 194ce0183cbSArd Biesheuvel { 19587e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 196ce0183cbSArd Biesheuvel 19787e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; 1982e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 199ce0183cbSArd Biesheuvel } 200ce0183cbSArd Biesheuvel 201ce0183cbSArd Biesheuvel static int rk_des_cbc_decrypt(struct skcipher_request *req) 202ce0183cbSArd Biesheuvel { 20387e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 204ce0183cbSArd Biesheuvel 20587e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; 2062e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 207ce0183cbSArd Biesheuvel } 208ce0183cbSArd Biesheuvel 209ce0183cbSArd Biesheuvel static int rk_des3_ede_ecb_encrypt(struct skcipher_request *req) 210ce0183cbSArd Biesheuvel { 21187e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 212ce0183cbSArd Biesheuvel 21387e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_TDES_SELECT; 2142e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 215ce0183cbSArd Biesheuvel } 216ce0183cbSArd Biesheuvel 217ce0183cbSArd Biesheuvel static int rk_des3_ede_ecb_decrypt(struct skcipher_request *req) 218ce0183cbSArd Biesheuvel { 21987e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 220ce0183cbSArd Biesheuvel 22187e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; 2222e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 223ce0183cbSArd Biesheuvel } 224ce0183cbSArd Biesheuvel 225ce0183cbSArd Biesheuvel static int rk_des3_ede_cbc_encrypt(struct skcipher_request *req) 226ce0183cbSArd Biesheuvel { 22787e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 228ce0183cbSArd Biesheuvel 22987e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; 2302e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 231ce0183cbSArd Biesheuvel } 232ce0183cbSArd Biesheuvel 233ce0183cbSArd Biesheuvel static int rk_des3_ede_cbc_decrypt(struct skcipher_request *req) 234ce0183cbSArd Biesheuvel { 23587e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 236ce0183cbSArd Biesheuvel 23787e356c4SCorentin Labbe rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | 238ce0183cbSArd Biesheuvel RK_CRYPTO_DEC; 2392e3b1495SCorentin Labbe return rk_cipher_handle_req(req); 240ce0183cbSArd Biesheuvel } 241ce0183cbSArd Biesheuvel 242a7fa0644SCorentin Labbe static void rk_cipher_hw_init(struct rk_crypto_info *dev, struct skcipher_request *req) 243ce0183cbSArd Biesheuvel { 244ce0183cbSArd Biesheuvel struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); 245ce0183cbSArd Biesheuvel struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 24687e356c4SCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(req); 247ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); 24857d67c6eSCorentin Labbe u32 block, conf_reg = 0; 249ce0183cbSArd Biesheuvel 250ce0183cbSArd Biesheuvel block = crypto_tfm_alg_blocksize(tfm); 251ce0183cbSArd Biesheuvel 252ce0183cbSArd Biesheuvel if (block == DES_BLOCK_SIZE) { 25387e356c4SCorentin Labbe rctx->mode |= RK_CRYPTO_TDES_FIFO_MODE | 254ce0183cbSArd Biesheuvel RK_CRYPTO_TDES_BYTESWAP_KEY | 255ce0183cbSArd Biesheuvel RK_CRYPTO_TDES_BYTESWAP_IV; 25687e356c4SCorentin Labbe CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, rctx->mode); 257d6b23cceSCorentin Labbe memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, ctx->key, ctx->keylen); 258ce0183cbSArd Biesheuvel conf_reg = RK_CRYPTO_DESSEL; 259ce0183cbSArd Biesheuvel } else { 26087e356c4SCorentin Labbe rctx->mode |= RK_CRYPTO_AES_FIFO_MODE | 261ce0183cbSArd Biesheuvel RK_CRYPTO_AES_KEY_CHANGE | 262ce0183cbSArd Biesheuvel RK_CRYPTO_AES_BYTESWAP_KEY | 263ce0183cbSArd Biesheuvel RK_CRYPTO_AES_BYTESWAP_IV; 264ce0183cbSArd Biesheuvel if (ctx->keylen == AES_KEYSIZE_192) 26587e356c4SCorentin Labbe rctx->mode |= RK_CRYPTO_AES_192BIT_key; 266ce0183cbSArd Biesheuvel else if (ctx->keylen == AES_KEYSIZE_256) 26787e356c4SCorentin Labbe rctx->mode |= RK_CRYPTO_AES_256BIT_key; 26887e356c4SCorentin Labbe CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, rctx->mode); 269d6b23cceSCorentin Labbe memcpy_toio(ctx->dev->reg + RK_CRYPTO_AES_KEY_0, ctx->key, ctx->keylen); 270ce0183cbSArd Biesheuvel } 271ce0183cbSArd Biesheuvel conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO | 272ce0183cbSArd Biesheuvel RK_CRYPTO_BYTESWAP_BRFIFO; 273ce0183cbSArd Biesheuvel CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg); 274ce0183cbSArd Biesheuvel CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, 275ce0183cbSArd Biesheuvel RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA); 276ce0183cbSArd Biesheuvel } 277ce0183cbSArd Biesheuvel 27857d67c6eSCorentin Labbe static void crypto_dma_start(struct rk_crypto_info *dev, 27957d67c6eSCorentin Labbe struct scatterlist *sgs, 28057d67c6eSCorentin Labbe struct scatterlist *sgd, unsigned int todo) 281ce0183cbSArd Biesheuvel { 28257d67c6eSCorentin Labbe CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, sg_dma_address(sgs)); 28357d67c6eSCorentin Labbe CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, todo); 28457d67c6eSCorentin Labbe CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, sg_dma_address(sgd)); 285ce0183cbSArd Biesheuvel CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START | 286ce0183cbSArd Biesheuvel _SBF(RK_CRYPTO_BLOCK_START, 16)); 287ce0183cbSArd Biesheuvel } 288ce0183cbSArd Biesheuvel 28957d67c6eSCorentin Labbe static int rk_cipher_run(struct crypto_engine *engine, void *async_req) 290ce0183cbSArd Biesheuvel { 29157d67c6eSCorentin Labbe struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); 29257d67c6eSCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 293ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); 29457d67c6eSCorentin Labbe struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq); 29557d67c6eSCorentin Labbe struct scatterlist *sgs, *sgd; 296ce0183cbSArd Biesheuvel int err = 0; 29757d67c6eSCorentin Labbe int ivsize = crypto_skcipher_ivsize(tfm); 29857d67c6eSCorentin Labbe int offset; 29957d67c6eSCorentin Labbe u8 iv[AES_BLOCK_SIZE]; 30057d67c6eSCorentin Labbe u8 biv[AES_BLOCK_SIZE]; 30157d67c6eSCorentin Labbe u8 *ivtouse = areq->iv; 30257d67c6eSCorentin Labbe unsigned int len = areq->cryptlen; 30357d67c6eSCorentin Labbe unsigned int todo; 30448d904d4SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 30548d904d4SCorentin Labbe struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher); 306*c018c7a9SCorentin Labbe struct rk_crypto_info *rkc = ctx->dev; 30748d904d4SCorentin Labbe 30848d904d4SCorentin Labbe algt->stat_req++; 309ce0183cbSArd Biesheuvel 31057d67c6eSCorentin Labbe ivsize = crypto_skcipher_ivsize(tfm); 31157d67c6eSCorentin Labbe if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { 31287e356c4SCorentin Labbe if (rctx->mode & RK_CRYPTO_DEC) { 31357d67c6eSCorentin Labbe offset = areq->cryptlen - ivsize; 31457d67c6eSCorentin Labbe scatterwalk_map_and_copy(rctx->backup_iv, areq->src, 31557d67c6eSCorentin Labbe offset, ivsize, 0); 31657d67c6eSCorentin Labbe } 317ce0183cbSArd Biesheuvel } 318ce0183cbSArd Biesheuvel 31957d67c6eSCorentin Labbe sgs = areq->src; 32057d67c6eSCorentin Labbe sgd = areq->dst; 32157d67c6eSCorentin Labbe 32257d67c6eSCorentin Labbe while (sgs && sgd && len) { 32357d67c6eSCorentin Labbe if (!sgs->length) { 32457d67c6eSCorentin Labbe sgs = sg_next(sgs); 32557d67c6eSCorentin Labbe sgd = sg_next(sgd); 32657d67c6eSCorentin Labbe continue; 32757d67c6eSCorentin Labbe } 32857d67c6eSCorentin Labbe if (rctx->mode & RK_CRYPTO_DEC) { 32957d67c6eSCorentin Labbe /* we backup last block of source to be used as IV at next step */ 33057d67c6eSCorentin Labbe offset = sgs->length - ivsize; 33157d67c6eSCorentin Labbe scatterwalk_map_and_copy(biv, sgs, offset, ivsize, 0); 33257d67c6eSCorentin Labbe } 33357d67c6eSCorentin Labbe if (sgs == sgd) { 334*c018c7a9SCorentin Labbe err = dma_map_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); 33557d67c6eSCorentin Labbe if (err <= 0) { 33657d67c6eSCorentin Labbe err = -EINVAL; 33757d67c6eSCorentin Labbe goto theend_iv; 33857d67c6eSCorentin Labbe } 33957d67c6eSCorentin Labbe } else { 340*c018c7a9SCorentin Labbe err = dma_map_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); 34157d67c6eSCorentin Labbe if (err <= 0) { 34257d67c6eSCorentin Labbe err = -EINVAL; 34357d67c6eSCorentin Labbe goto theend_iv; 34457d67c6eSCorentin Labbe } 345*c018c7a9SCorentin Labbe err = dma_map_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); 34657d67c6eSCorentin Labbe if (err <= 0) { 34757d67c6eSCorentin Labbe err = -EINVAL; 34857d67c6eSCorentin Labbe goto theend_sgs; 34957d67c6eSCorentin Labbe } 35057d67c6eSCorentin Labbe } 35157d67c6eSCorentin Labbe err = 0; 352*c018c7a9SCorentin Labbe rk_cipher_hw_init(rkc, areq); 35357d67c6eSCorentin Labbe if (ivsize) { 354ce0183cbSArd Biesheuvel if (ivsize == DES_BLOCK_SIZE) 355*c018c7a9SCorentin Labbe memcpy_toio(rkc->reg + RK_CRYPTO_TDES_IV_0, ivtouse, ivsize); 35657d67c6eSCorentin Labbe else 357*c018c7a9SCorentin Labbe memcpy_toio(rkc->reg + RK_CRYPTO_AES_IV_0, ivtouse, ivsize); 358ce0183cbSArd Biesheuvel } 359*c018c7a9SCorentin Labbe reinit_completion(&rkc->complete); 360*c018c7a9SCorentin Labbe rkc->status = 0; 361ce0183cbSArd Biesheuvel 36257d67c6eSCorentin Labbe todo = min(sg_dma_len(sgs), len); 36357d67c6eSCorentin Labbe len -= todo; 364*c018c7a9SCorentin Labbe crypto_dma_start(rkc, sgs, sgd, todo / 4); 365*c018c7a9SCorentin Labbe wait_for_completion_interruptible_timeout(&rkc->complete, 36657d67c6eSCorentin Labbe msecs_to_jiffies(2000)); 367*c018c7a9SCorentin Labbe if (!rkc->status) { 368*c018c7a9SCorentin Labbe dev_err(rkc->dev, "DMA timeout\n"); 36957d67c6eSCorentin Labbe err = -EFAULT; 37057d67c6eSCorentin Labbe goto theend; 371ce0183cbSArd Biesheuvel } 37257d67c6eSCorentin Labbe if (sgs == sgd) { 373*c018c7a9SCorentin Labbe dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); 374ce0183cbSArd Biesheuvel } else { 375*c018c7a9SCorentin Labbe dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); 376*c018c7a9SCorentin Labbe dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); 377ce0183cbSArd Biesheuvel } 37857d67c6eSCorentin Labbe if (rctx->mode & RK_CRYPTO_DEC) { 37957d67c6eSCorentin Labbe memcpy(iv, biv, ivsize); 38057d67c6eSCorentin Labbe ivtouse = iv; 38157d67c6eSCorentin Labbe } else { 38257d67c6eSCorentin Labbe offset = sgd->length - ivsize; 38357d67c6eSCorentin Labbe scatterwalk_map_and_copy(iv, sgd, offset, ivsize, 0); 38457d67c6eSCorentin Labbe ivtouse = iv; 38557d67c6eSCorentin Labbe } 38657d67c6eSCorentin Labbe sgs = sg_next(sgs); 38757d67c6eSCorentin Labbe sgd = sg_next(sgd); 38857d67c6eSCorentin Labbe } 38957d67c6eSCorentin Labbe 39057d67c6eSCorentin Labbe if (areq->iv && ivsize > 0) { 39157d67c6eSCorentin Labbe offset = areq->cryptlen - ivsize; 39257d67c6eSCorentin Labbe if (rctx->mode & RK_CRYPTO_DEC) { 39357d67c6eSCorentin Labbe memcpy(areq->iv, rctx->backup_iv, ivsize); 39457d67c6eSCorentin Labbe memzero_explicit(rctx->backup_iv, ivsize); 39557d67c6eSCorentin Labbe } else { 39657d67c6eSCorentin Labbe scatterwalk_map_and_copy(areq->iv, areq->dst, offset, 39757d67c6eSCorentin Labbe ivsize, 0); 39857d67c6eSCorentin Labbe } 39957d67c6eSCorentin Labbe } 40057d67c6eSCorentin Labbe 40157d67c6eSCorentin Labbe theend: 40257d67c6eSCorentin Labbe local_bh_disable(); 40357d67c6eSCorentin Labbe crypto_finalize_skcipher_request(engine, areq, err); 40457d67c6eSCorentin Labbe local_bh_enable(); 40557d67c6eSCorentin Labbe return 0; 40657d67c6eSCorentin Labbe 40757d67c6eSCorentin Labbe theend_sgs: 40857d67c6eSCorentin Labbe if (sgs == sgd) { 409*c018c7a9SCorentin Labbe dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL); 41057d67c6eSCorentin Labbe } else { 411*c018c7a9SCorentin Labbe dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE); 412*c018c7a9SCorentin Labbe dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE); 41357d67c6eSCorentin Labbe } 41457d67c6eSCorentin Labbe theend_iv: 415ce0183cbSArd Biesheuvel return err; 416ce0183cbSArd Biesheuvel } 417ce0183cbSArd Biesheuvel 418a7fa0644SCorentin Labbe static int rk_cipher_tfm_init(struct crypto_skcipher *tfm) 419ce0183cbSArd Biesheuvel { 420ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); 421ce0183cbSArd Biesheuvel struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 42268ef8af0SCorentin Labbe const char *name = crypto_tfm_alg_name(&tfm->base); 423ce0183cbSArd Biesheuvel struct rk_crypto_tmp *algt; 424a216be39SCorentin Labbe int err; 425ce0183cbSArd Biesheuvel 426ce0183cbSArd Biesheuvel algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher); 427ce0183cbSArd Biesheuvel 428ce0183cbSArd Biesheuvel ctx->dev = algt->dev; 429ce0183cbSArd Biesheuvel 43068ef8af0SCorentin Labbe ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 43168ef8af0SCorentin Labbe if (IS_ERR(ctx->fallback_tfm)) { 43268ef8af0SCorentin Labbe dev_err(ctx->dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", 43368ef8af0SCorentin Labbe name, PTR_ERR(ctx->fallback_tfm)); 43468ef8af0SCorentin Labbe return PTR_ERR(ctx->fallback_tfm); 43568ef8af0SCorentin Labbe } 43668ef8af0SCorentin Labbe 43768ef8af0SCorentin Labbe tfm->reqsize = sizeof(struct rk_cipher_rctx) + 43868ef8af0SCorentin Labbe crypto_skcipher_reqsize(ctx->fallback_tfm); 43968ef8af0SCorentin Labbe 44057d67c6eSCorentin Labbe ctx->enginectx.op.do_one_request = rk_cipher_run; 44157d67c6eSCorentin Labbe 442a216be39SCorentin Labbe err = pm_runtime_resume_and_get(ctx->dev->dev); 443a216be39SCorentin Labbe if (err < 0) 444a216be39SCorentin Labbe goto error_pm; 445a216be39SCorentin Labbe 446c50ef141SCorentin Labbe return 0; 447a216be39SCorentin Labbe error_pm: 448a216be39SCorentin Labbe crypto_free_skcipher(ctx->fallback_tfm); 449a216be39SCorentin Labbe return err; 450ce0183cbSArd Biesheuvel } 451ce0183cbSArd Biesheuvel 452a7fa0644SCorentin Labbe static void rk_cipher_tfm_exit(struct crypto_skcipher *tfm) 453ce0183cbSArd Biesheuvel { 454ce0183cbSArd Biesheuvel struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); 455ce0183cbSArd Biesheuvel 456d6b23cceSCorentin Labbe memzero_explicit(ctx->key, ctx->keylen); 45768ef8af0SCorentin Labbe crypto_free_skcipher(ctx->fallback_tfm); 458a216be39SCorentin Labbe pm_runtime_put_autosuspend(ctx->dev->dev); 459ce0183cbSArd Biesheuvel } 460ce0183cbSArd Biesheuvel 461ce0183cbSArd Biesheuvel struct rk_crypto_tmp rk_ecb_aes_alg = { 4626d55c4a2SCorentin Labbe .type = CRYPTO_ALG_TYPE_SKCIPHER, 463ce0183cbSArd Biesheuvel .alg.skcipher = { 464ce0183cbSArd Biesheuvel .base.cra_name = "ecb(aes)", 465ce0183cbSArd Biesheuvel .base.cra_driver_name = "ecb-aes-rk", 466ce0183cbSArd Biesheuvel .base.cra_priority = 300, 46768ef8af0SCorentin Labbe .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 468ce0183cbSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 469ce0183cbSArd Biesheuvel .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 470ce0183cbSArd Biesheuvel .base.cra_alignmask = 0x0f, 471ce0183cbSArd Biesheuvel .base.cra_module = THIS_MODULE, 472ce0183cbSArd Biesheuvel 473a7fa0644SCorentin Labbe .init = rk_cipher_tfm_init, 474a7fa0644SCorentin Labbe .exit = rk_cipher_tfm_exit, 475ce0183cbSArd Biesheuvel .min_keysize = AES_MIN_KEY_SIZE, 476ce0183cbSArd Biesheuvel .max_keysize = AES_MAX_KEY_SIZE, 477ce0183cbSArd Biesheuvel .setkey = rk_aes_setkey, 478ce0183cbSArd Biesheuvel .encrypt = rk_aes_ecb_encrypt, 479ce0183cbSArd Biesheuvel .decrypt = rk_aes_ecb_decrypt, 480ce0183cbSArd Biesheuvel } 481ce0183cbSArd Biesheuvel }; 482ce0183cbSArd Biesheuvel 483ce0183cbSArd Biesheuvel struct rk_crypto_tmp rk_cbc_aes_alg = { 4846d55c4a2SCorentin Labbe .type = CRYPTO_ALG_TYPE_SKCIPHER, 485ce0183cbSArd Biesheuvel .alg.skcipher = { 486ce0183cbSArd Biesheuvel .base.cra_name = "cbc(aes)", 487ce0183cbSArd Biesheuvel .base.cra_driver_name = "cbc-aes-rk", 488ce0183cbSArd Biesheuvel .base.cra_priority = 300, 48968ef8af0SCorentin Labbe .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 490ce0183cbSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 491ce0183cbSArd Biesheuvel .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 492ce0183cbSArd Biesheuvel .base.cra_alignmask = 0x0f, 493ce0183cbSArd Biesheuvel .base.cra_module = THIS_MODULE, 494ce0183cbSArd Biesheuvel 495a7fa0644SCorentin Labbe .init = rk_cipher_tfm_init, 496a7fa0644SCorentin Labbe .exit = rk_cipher_tfm_exit, 497ce0183cbSArd Biesheuvel .min_keysize = AES_MIN_KEY_SIZE, 498ce0183cbSArd Biesheuvel .max_keysize = AES_MAX_KEY_SIZE, 499ce0183cbSArd Biesheuvel .ivsize = AES_BLOCK_SIZE, 500ce0183cbSArd Biesheuvel .setkey = rk_aes_setkey, 501ce0183cbSArd Biesheuvel .encrypt = rk_aes_cbc_encrypt, 502ce0183cbSArd Biesheuvel .decrypt = rk_aes_cbc_decrypt, 503ce0183cbSArd Biesheuvel } 504ce0183cbSArd Biesheuvel }; 505ce0183cbSArd Biesheuvel 506ce0183cbSArd Biesheuvel struct rk_crypto_tmp rk_ecb_des_alg = { 5076d55c4a2SCorentin Labbe .type = CRYPTO_ALG_TYPE_SKCIPHER, 508ce0183cbSArd Biesheuvel .alg.skcipher = { 509ce0183cbSArd Biesheuvel .base.cra_name = "ecb(des)", 510ce0183cbSArd Biesheuvel .base.cra_driver_name = "ecb-des-rk", 511ce0183cbSArd Biesheuvel .base.cra_priority = 300, 51268ef8af0SCorentin Labbe .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 513ce0183cbSArd Biesheuvel .base.cra_blocksize = DES_BLOCK_SIZE, 514ce0183cbSArd Biesheuvel .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 515ce0183cbSArd Biesheuvel .base.cra_alignmask = 0x07, 516ce0183cbSArd Biesheuvel .base.cra_module = THIS_MODULE, 517ce0183cbSArd Biesheuvel 518a7fa0644SCorentin Labbe .init = rk_cipher_tfm_init, 519a7fa0644SCorentin Labbe .exit = rk_cipher_tfm_exit, 520ce0183cbSArd Biesheuvel .min_keysize = DES_KEY_SIZE, 521ce0183cbSArd Biesheuvel .max_keysize = DES_KEY_SIZE, 522ce0183cbSArd Biesheuvel .setkey = rk_des_setkey, 523ce0183cbSArd Biesheuvel .encrypt = rk_des_ecb_encrypt, 524ce0183cbSArd Biesheuvel .decrypt = rk_des_ecb_decrypt, 525ce0183cbSArd Biesheuvel } 526ce0183cbSArd Biesheuvel }; 527ce0183cbSArd Biesheuvel 528ce0183cbSArd Biesheuvel struct rk_crypto_tmp rk_cbc_des_alg = { 5296d55c4a2SCorentin Labbe .type = CRYPTO_ALG_TYPE_SKCIPHER, 530ce0183cbSArd Biesheuvel .alg.skcipher = { 531ce0183cbSArd Biesheuvel .base.cra_name = "cbc(des)", 532ce0183cbSArd Biesheuvel .base.cra_driver_name = "cbc-des-rk", 533ce0183cbSArd Biesheuvel .base.cra_priority = 300, 53468ef8af0SCorentin Labbe .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 535ce0183cbSArd Biesheuvel .base.cra_blocksize = DES_BLOCK_SIZE, 536ce0183cbSArd Biesheuvel .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 537ce0183cbSArd Biesheuvel .base.cra_alignmask = 0x07, 538ce0183cbSArd Biesheuvel .base.cra_module = THIS_MODULE, 539ce0183cbSArd Biesheuvel 540a7fa0644SCorentin Labbe .init = rk_cipher_tfm_init, 541a7fa0644SCorentin Labbe .exit = rk_cipher_tfm_exit, 542ce0183cbSArd Biesheuvel .min_keysize = DES_KEY_SIZE, 543ce0183cbSArd Biesheuvel .max_keysize = DES_KEY_SIZE, 544ce0183cbSArd Biesheuvel .ivsize = DES_BLOCK_SIZE, 545ce0183cbSArd Biesheuvel .setkey = rk_des_setkey, 546ce0183cbSArd Biesheuvel .encrypt = rk_des_cbc_encrypt, 547ce0183cbSArd Biesheuvel .decrypt = rk_des_cbc_decrypt, 548ce0183cbSArd Biesheuvel } 549ce0183cbSArd Biesheuvel }; 550ce0183cbSArd Biesheuvel 551ce0183cbSArd Biesheuvel struct rk_crypto_tmp rk_ecb_des3_ede_alg = { 5526d55c4a2SCorentin Labbe .type = CRYPTO_ALG_TYPE_SKCIPHER, 553ce0183cbSArd Biesheuvel .alg.skcipher = { 554ce0183cbSArd Biesheuvel .base.cra_name = "ecb(des3_ede)", 555ce0183cbSArd Biesheuvel .base.cra_driver_name = "ecb-des3-ede-rk", 556ce0183cbSArd Biesheuvel .base.cra_priority = 300, 55768ef8af0SCorentin Labbe .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 558ce0183cbSArd Biesheuvel .base.cra_blocksize = DES_BLOCK_SIZE, 559ce0183cbSArd Biesheuvel .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 560ce0183cbSArd Biesheuvel .base.cra_alignmask = 0x07, 561ce0183cbSArd Biesheuvel .base.cra_module = THIS_MODULE, 562ce0183cbSArd Biesheuvel 563a7fa0644SCorentin Labbe .init = rk_cipher_tfm_init, 564a7fa0644SCorentin Labbe .exit = rk_cipher_tfm_exit, 565ce0183cbSArd Biesheuvel .min_keysize = DES3_EDE_KEY_SIZE, 566ce0183cbSArd Biesheuvel .max_keysize = DES3_EDE_KEY_SIZE, 567ce0183cbSArd Biesheuvel .setkey = rk_tdes_setkey, 568ce0183cbSArd Biesheuvel .encrypt = rk_des3_ede_ecb_encrypt, 569ce0183cbSArd Biesheuvel .decrypt = rk_des3_ede_ecb_decrypt, 570ce0183cbSArd Biesheuvel } 571ce0183cbSArd Biesheuvel }; 572ce0183cbSArd Biesheuvel 573ce0183cbSArd Biesheuvel struct rk_crypto_tmp rk_cbc_des3_ede_alg = { 5746d55c4a2SCorentin Labbe .type = CRYPTO_ALG_TYPE_SKCIPHER, 575ce0183cbSArd Biesheuvel .alg.skcipher = { 576ce0183cbSArd Biesheuvel .base.cra_name = "cbc(des3_ede)", 577ce0183cbSArd Biesheuvel .base.cra_driver_name = "cbc-des3-ede-rk", 578ce0183cbSArd Biesheuvel .base.cra_priority = 300, 57968ef8af0SCorentin Labbe .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 580ce0183cbSArd Biesheuvel .base.cra_blocksize = DES_BLOCK_SIZE, 581ce0183cbSArd Biesheuvel .base.cra_ctxsize = sizeof(struct rk_cipher_ctx), 582ce0183cbSArd Biesheuvel .base.cra_alignmask = 0x07, 583ce0183cbSArd Biesheuvel .base.cra_module = THIS_MODULE, 584ce0183cbSArd Biesheuvel 585a7fa0644SCorentin Labbe .init = rk_cipher_tfm_init, 586a7fa0644SCorentin Labbe .exit = rk_cipher_tfm_exit, 587ce0183cbSArd Biesheuvel .min_keysize = DES3_EDE_KEY_SIZE, 588ce0183cbSArd Biesheuvel .max_keysize = DES3_EDE_KEY_SIZE, 589ce0183cbSArd Biesheuvel .ivsize = DES_BLOCK_SIZE, 590ce0183cbSArd Biesheuvel .setkey = rk_tdes_setkey, 591ce0183cbSArd Biesheuvel .encrypt = rk_des3_ede_cbc_encrypt, 592ce0183cbSArd Biesheuvel .decrypt = rk_des3_ede_cbc_decrypt, 593ce0183cbSArd Biesheuvel } 594ce0183cbSArd Biesheuvel }; 595