1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
21d6b8a6fSTom Lendacky /*
31d6b8a6fSTom Lendacky * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support
41d6b8a6fSTom Lendacky *
5e652399eSGary R Hook * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
61d6b8a6fSTom Lendacky *
7e652399eSGary R Hook * Author: Gary R Hook <gary.hook@amd.com>
81d6b8a6fSTom Lendacky * Author: Tom Lendacky <thomas.lendacky@amd.com>
91d6b8a6fSTom Lendacky */
101d6b8a6fSTom Lendacky
111d6b8a6fSTom Lendacky #include <linux/module.h>
121d6b8a6fSTom Lendacky #include <linux/sched.h>
131d6b8a6fSTom Lendacky #include <linux/delay.h>
141d6b8a6fSTom Lendacky #include <linux/scatterlist.h>
151d6b8a6fSTom Lendacky #include <crypto/aes.h>
1647f27f16SGary R Hook #include <crypto/xts.h>
17241118deSHerbert Xu #include <crypto/internal/skcipher.h>
181d6b8a6fSTom Lendacky #include <crypto/scatterwalk.h>
191d6b8a6fSTom Lendacky
201d6b8a6fSTom Lendacky #include "ccp-crypto.h"
211d6b8a6fSTom Lendacky
221d6b8a6fSTom Lendacky struct ccp_aes_xts_def {
231d6b8a6fSTom Lendacky const char *name;
241d6b8a6fSTom Lendacky const char *drv_name;
251d6b8a6fSTom Lendacky };
261d6b8a6fSTom Lendacky
27be9fe620SArd Biesheuvel static const struct ccp_aes_xts_def aes_xts_algs[] = {
281d6b8a6fSTom Lendacky {
291d6b8a6fSTom Lendacky .name = "xts(aes)",
301d6b8a6fSTom Lendacky .drv_name = "xts-aes-ccp",
311d6b8a6fSTom Lendacky },
321d6b8a6fSTom Lendacky };
331d6b8a6fSTom Lendacky
341d6b8a6fSTom Lendacky struct ccp_unit_size_map {
351d6b8a6fSTom Lendacky unsigned int size;
361d6b8a6fSTom Lendacky u32 value;
371d6b8a6fSTom Lendacky };
381d6b8a6fSTom Lendacky
397f7216cfSGary R Hook static struct ccp_unit_size_map xts_unit_sizes[] = {
401d6b8a6fSTom Lendacky {
417f7216cfSGary R Hook .size = 16,
427f7216cfSGary R Hook .value = CCP_XTS_AES_UNIT_SIZE_16,
431d6b8a6fSTom Lendacky },
441d6b8a6fSTom Lendacky {
451d6b8a6fSTom Lendacky .size = 512,
461d6b8a6fSTom Lendacky .value = CCP_XTS_AES_UNIT_SIZE_512,
471d6b8a6fSTom Lendacky },
481d6b8a6fSTom Lendacky {
497f7216cfSGary R Hook .size = 1024,
507f7216cfSGary R Hook .value = CCP_XTS_AES_UNIT_SIZE_1024,
511d6b8a6fSTom Lendacky },
521d6b8a6fSTom Lendacky {
537f7216cfSGary R Hook .size = 2048,
547f7216cfSGary R Hook .value = CCP_XTS_AES_UNIT_SIZE_2048,
551d6b8a6fSTom Lendacky },
561d6b8a6fSTom Lendacky {
577f7216cfSGary R Hook .size = 4096,
587f7216cfSGary R Hook .value = CCP_XTS_AES_UNIT_SIZE_4096,
591d6b8a6fSTom Lendacky },
601d6b8a6fSTom Lendacky };
611d6b8a6fSTom Lendacky
ccp_aes_xts_complete(struct crypto_async_request * async_req,int ret)621d6b8a6fSTom Lendacky static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)
631d6b8a6fSTom Lendacky {
64be9fe620SArd Biesheuvel struct skcipher_request *req = skcipher_request_cast(async_req);
65*99c6b20eSHerbert Xu struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
661d6b8a6fSTom Lendacky
671d6b8a6fSTom Lendacky if (ret)
681d6b8a6fSTom Lendacky return ret;
691d6b8a6fSTom Lendacky
70be9fe620SArd Biesheuvel memcpy(req->iv, rctx->iv, AES_BLOCK_SIZE);
711d6b8a6fSTom Lendacky
721d6b8a6fSTom Lendacky return 0;
731d6b8a6fSTom Lendacky }
741d6b8a6fSTom Lendacky
ccp_aes_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)75be9fe620SArd Biesheuvel static int ccp_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
761d6b8a6fSTom Lendacky unsigned int key_len)
771d6b8a6fSTom Lendacky {
78*99c6b20eSHerbert Xu struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
795060ffc9SGary R Hook unsigned int ccpversion = ccp_version();
8047f27f16SGary R Hook int ret;
8147f27f16SGary R Hook
82be9fe620SArd Biesheuvel ret = xts_verify_key(tfm, key, key_len);
8347f27f16SGary R Hook if (ret)
8447f27f16SGary R Hook return ret;
851d6b8a6fSTom Lendacky
865060ffc9SGary R Hook /* Version 3 devices support 128-bit keys; version 5 devices can
875060ffc9SGary R Hook * accommodate 128- and 256-bit keys.
881d6b8a6fSTom Lendacky */
891d6b8a6fSTom Lendacky switch (key_len) {
901d6b8a6fSTom Lendacky case AES_KEYSIZE_128 * 2:
911d6b8a6fSTom Lendacky memcpy(ctx->u.aes.key, key, key_len);
921d6b8a6fSTom Lendacky break;
935060ffc9SGary R Hook case AES_KEYSIZE_256 * 2:
945060ffc9SGary R Hook if (ccpversion > CCP_VERSION(3, 0))
955060ffc9SGary R Hook memcpy(ctx->u.aes.key, key, key_len);
965060ffc9SGary R Hook break;
971d6b8a6fSTom Lendacky }
981d6b8a6fSTom Lendacky ctx->u.aes.key_len = key_len / 2;
991d6b8a6fSTom Lendacky sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
1001d6b8a6fSTom Lendacky
101413b61ceSArd Biesheuvel return crypto_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len);
1021d6b8a6fSTom Lendacky }
1031d6b8a6fSTom Lendacky
ccp_aes_xts_crypt(struct skcipher_request * req,unsigned int encrypt)104be9fe620SArd Biesheuvel static int ccp_aes_xts_crypt(struct skcipher_request *req,
1051d6b8a6fSTom Lendacky unsigned int encrypt)
1061d6b8a6fSTom Lendacky {
107be9fe620SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
108*99c6b20eSHerbert Xu struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
109*99c6b20eSHerbert Xu struct ccp_aes_req_ctx *rctx = skcipher_request_ctx_dma(req);
1105060ffc9SGary R Hook unsigned int ccpversion = ccp_version();
1115060ffc9SGary R Hook unsigned int fallback = 0;
1121d6b8a6fSTom Lendacky unsigned int unit;
113ab6a11a7STom Lendacky u32 unit_size;
1141d6b8a6fSTom Lendacky int ret;
1151d6b8a6fSTom Lendacky
116369f3dabSTom Lendacky if (!ctx->u.aes.key_len)
1171d6b8a6fSTom Lendacky return -EINVAL;
1181d6b8a6fSTom Lendacky
119be9fe620SArd Biesheuvel if (!req->iv)
1201d6b8a6fSTom Lendacky return -EINVAL;
1211d6b8a6fSTom Lendacky
1227f7216cfSGary R Hook /* Check conditions under which the CCP can fulfill a request. The
1237f7216cfSGary R Hook * device can handle input plaintext of a length that is a multiple
1247f7216cfSGary R Hook * of the unit_size, bug the crypto implementation only supports
1257f7216cfSGary R Hook * the unit_size being equal to the input length. This limits the
1267f7216cfSGary R Hook * number of scenarios we can handle.
1277f7216cfSGary R Hook */
128ab6a11a7STom Lendacky unit_size = CCP_XTS_AES_UNIT_SIZE__LAST;
1297f7216cfSGary R Hook for (unit = 0; unit < ARRAY_SIZE(xts_unit_sizes); unit++) {
130be9fe620SArd Biesheuvel if (req->cryptlen == xts_unit_sizes[unit].size) {
1317f7216cfSGary R Hook unit_size = unit;
1321d6b8a6fSTom Lendacky break;
133ab6a11a7STom Lendacky }
134ab6a11a7STom Lendacky }
1355060ffc9SGary R Hook /* The CCP has restrictions on block sizes. Also, a version 3 device
1365060ffc9SGary R Hook * only supports AES-128 operations; version 5 CCPs support both
1375060ffc9SGary R Hook * AES-128 and -256 operations.
1385060ffc9SGary R Hook */
1395060ffc9SGary R Hook if (unit_size == CCP_XTS_AES_UNIT_SIZE__LAST)
1405060ffc9SGary R Hook fallback = 1;
1415060ffc9SGary R Hook if ((ccpversion < CCP_VERSION(5, 0)) &&
1425060ffc9SGary R Hook (ctx->u.aes.key_len != AES_KEYSIZE_128))
1435060ffc9SGary R Hook fallback = 1;
1445060ffc9SGary R Hook if ((ctx->u.aes.key_len != AES_KEYSIZE_128) &&
1455060ffc9SGary R Hook (ctx->u.aes.key_len != AES_KEYSIZE_256))
1465060ffc9SGary R Hook fallback = 1;
1475060ffc9SGary R Hook if (fallback) {
1481d6b8a6fSTom Lendacky /* Use the fallback to process the request for any
1491d6b8a6fSTom Lendacky * unsupported unit sizes or key sizes
1501d6b8a6fSTom Lendacky */
151413b61ceSArd Biesheuvel skcipher_request_set_tfm(&rctx->fallback_req,
152413b61ceSArd Biesheuvel ctx->u.aes.tfm_skcipher);
153413b61ceSArd Biesheuvel skcipher_request_set_callback(&rctx->fallback_req,
154413b61ceSArd Biesheuvel req->base.flags,
155413b61ceSArd Biesheuvel req->base.complete,
156413b61ceSArd Biesheuvel req->base.data);
157413b61ceSArd Biesheuvel skcipher_request_set_crypt(&rctx->fallback_req, req->src,
158413b61ceSArd Biesheuvel req->dst, req->cryptlen, req->iv);
159413b61ceSArd Biesheuvel ret = encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
160413b61ceSArd Biesheuvel crypto_skcipher_decrypt(&rctx->fallback_req);
1611d6b8a6fSTom Lendacky return ret;
1621d6b8a6fSTom Lendacky }
1631d6b8a6fSTom Lendacky
164be9fe620SArd Biesheuvel memcpy(rctx->iv, req->iv, AES_BLOCK_SIZE);
1651d6b8a6fSTom Lendacky sg_init_one(&rctx->iv_sg, rctx->iv, AES_BLOCK_SIZE);
1661d6b8a6fSTom Lendacky
1671d6b8a6fSTom Lendacky memset(&rctx->cmd, 0, sizeof(rctx->cmd));
1681d6b8a6fSTom Lendacky INIT_LIST_HEAD(&rctx->cmd.entry);
1691d6b8a6fSTom Lendacky rctx->cmd.engine = CCP_ENGINE_XTS_AES_128;
170e652399eSGary R Hook rctx->cmd.u.xts.type = CCP_AES_TYPE_128;
1711d6b8a6fSTom Lendacky rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT
1721d6b8a6fSTom Lendacky : CCP_AES_ACTION_DECRYPT;
173ab6a11a7STom Lendacky rctx->cmd.u.xts.unit_size = unit_size;
1741d6b8a6fSTom Lendacky rctx->cmd.u.xts.key = &ctx->u.aes.key_sg;
1751d6b8a6fSTom Lendacky rctx->cmd.u.xts.key_len = ctx->u.aes.key_len;
1761d6b8a6fSTom Lendacky rctx->cmd.u.xts.iv = &rctx->iv_sg;
1771d6b8a6fSTom Lendacky rctx->cmd.u.xts.iv_len = AES_BLOCK_SIZE;
1781d6b8a6fSTom Lendacky rctx->cmd.u.xts.src = req->src;
179be9fe620SArd Biesheuvel rctx->cmd.u.xts.src_len = req->cryptlen;
1801d6b8a6fSTom Lendacky rctx->cmd.u.xts.dst = req->dst;
1811d6b8a6fSTom Lendacky
1821d6b8a6fSTom Lendacky ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
1831d6b8a6fSTom Lendacky
1841d6b8a6fSTom Lendacky return ret;
1851d6b8a6fSTom Lendacky }
1861d6b8a6fSTom Lendacky
ccp_aes_xts_encrypt(struct skcipher_request * req)187be9fe620SArd Biesheuvel static int ccp_aes_xts_encrypt(struct skcipher_request *req)
1881d6b8a6fSTom Lendacky {
1891d6b8a6fSTom Lendacky return ccp_aes_xts_crypt(req, 1);
1901d6b8a6fSTom Lendacky }
1911d6b8a6fSTom Lendacky
ccp_aes_xts_decrypt(struct skcipher_request * req)192be9fe620SArd Biesheuvel static int ccp_aes_xts_decrypt(struct skcipher_request *req)
1931d6b8a6fSTom Lendacky {
1941d6b8a6fSTom Lendacky return ccp_aes_xts_crypt(req, 0);
1951d6b8a6fSTom Lendacky }
1961d6b8a6fSTom Lendacky
ccp_aes_xts_init_tfm(struct crypto_skcipher * tfm)197be9fe620SArd Biesheuvel static int ccp_aes_xts_init_tfm(struct crypto_skcipher *tfm)
1981d6b8a6fSTom Lendacky {
199*99c6b20eSHerbert Xu struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
200413b61ceSArd Biesheuvel struct crypto_skcipher *fallback_tfm;
2011d6b8a6fSTom Lendacky
2021d6b8a6fSTom Lendacky ctx->complete = ccp_aes_xts_complete;
2031d6b8a6fSTom Lendacky ctx->u.aes.key_len = 0;
2041d6b8a6fSTom Lendacky
205413b61ceSArd Biesheuvel fallback_tfm = crypto_alloc_skcipher("xts(aes)", 0,
2061d6b8a6fSTom Lendacky CRYPTO_ALG_NEED_FALLBACK);
2071d6b8a6fSTom Lendacky if (IS_ERR(fallback_tfm)) {
208241118deSHerbert Xu pr_warn("could not load fallback driver xts(aes)\n");
2091d6b8a6fSTom Lendacky return PTR_ERR(fallback_tfm);
2101d6b8a6fSTom Lendacky }
211241118deSHerbert Xu ctx->u.aes.tfm_skcipher = fallback_tfm;
2121d6b8a6fSTom Lendacky
213*99c6b20eSHerbert Xu crypto_skcipher_set_reqsize_dma(tfm,
214*99c6b20eSHerbert Xu sizeof(struct ccp_aes_req_ctx) +
215413b61ceSArd Biesheuvel crypto_skcipher_reqsize(fallback_tfm));
2161d6b8a6fSTom Lendacky
2171d6b8a6fSTom Lendacky return 0;
2181d6b8a6fSTom Lendacky }
2191d6b8a6fSTom Lendacky
ccp_aes_xts_exit_tfm(struct crypto_skcipher * tfm)220be9fe620SArd Biesheuvel static void ccp_aes_xts_exit_tfm(struct crypto_skcipher *tfm)
2211d6b8a6fSTom Lendacky {
222*99c6b20eSHerbert Xu struct ccp_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
2231d6b8a6fSTom Lendacky
224413b61ceSArd Biesheuvel crypto_free_skcipher(ctx->u.aes.tfm_skcipher);
2251d6b8a6fSTom Lendacky }
2261d6b8a6fSTom Lendacky
ccp_register_aes_xts_alg(struct list_head * head,const struct ccp_aes_xts_def * def)2271d6b8a6fSTom Lendacky static int ccp_register_aes_xts_alg(struct list_head *head,
2281d6b8a6fSTom Lendacky const struct ccp_aes_xts_def *def)
2291d6b8a6fSTom Lendacky {
230be9fe620SArd Biesheuvel struct ccp_crypto_skcipher_alg *ccp_alg;
231be9fe620SArd Biesheuvel struct skcipher_alg *alg;
2321d6b8a6fSTom Lendacky int ret;
2331d6b8a6fSTom Lendacky
2341d6b8a6fSTom Lendacky ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
2351d6b8a6fSTom Lendacky if (!ccp_alg)
2361d6b8a6fSTom Lendacky return -ENOMEM;
2371d6b8a6fSTom Lendacky
2381d6b8a6fSTom Lendacky INIT_LIST_HEAD(&ccp_alg->entry);
2391d6b8a6fSTom Lendacky
2401d6b8a6fSTom Lendacky alg = &ccp_alg->alg;
2411d6b8a6fSTom Lendacky
242be9fe620SArd Biesheuvel snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
243be9fe620SArd Biesheuvel snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2441d6b8a6fSTom Lendacky def->drv_name);
245be9fe620SArd Biesheuvel alg->base.cra_flags = CRYPTO_ALG_ASYNC |
246b8aa7dc5SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY |
2471d6b8a6fSTom Lendacky CRYPTO_ALG_KERN_DRIVER_ONLY |
2481d6b8a6fSTom Lendacky CRYPTO_ALG_NEED_FALLBACK;
249be9fe620SArd Biesheuvel alg->base.cra_blocksize = AES_BLOCK_SIZE;
250*99c6b20eSHerbert Xu alg->base.cra_ctxsize = sizeof(struct ccp_ctx) +
251*99c6b20eSHerbert Xu crypto_dma_padding();
252be9fe620SArd Biesheuvel alg->base.cra_priority = CCP_CRA_PRIORITY;
253be9fe620SArd Biesheuvel alg->base.cra_module = THIS_MODULE;
2541d6b8a6fSTom Lendacky
255be9fe620SArd Biesheuvel alg->setkey = ccp_aes_xts_setkey;
256be9fe620SArd Biesheuvel alg->encrypt = ccp_aes_xts_encrypt;
257be9fe620SArd Biesheuvel alg->decrypt = ccp_aes_xts_decrypt;
258be9fe620SArd Biesheuvel alg->min_keysize = AES_MIN_KEY_SIZE * 2;
259be9fe620SArd Biesheuvel alg->max_keysize = AES_MAX_KEY_SIZE * 2;
260be9fe620SArd Biesheuvel alg->ivsize = AES_BLOCK_SIZE;
261be9fe620SArd Biesheuvel alg->init = ccp_aes_xts_init_tfm;
262be9fe620SArd Biesheuvel alg->exit = ccp_aes_xts_exit_tfm;
263be9fe620SArd Biesheuvel
264be9fe620SArd Biesheuvel ret = crypto_register_skcipher(alg);
2651d6b8a6fSTom Lendacky if (ret) {
266be9fe620SArd Biesheuvel pr_err("%s skcipher algorithm registration error (%d)\n",
267be9fe620SArd Biesheuvel alg->base.cra_name, ret);
2681d6b8a6fSTom Lendacky kfree(ccp_alg);
2691d6b8a6fSTom Lendacky return ret;
2701d6b8a6fSTom Lendacky }
2711d6b8a6fSTom Lendacky
2721d6b8a6fSTom Lendacky list_add(&ccp_alg->entry, head);
2731d6b8a6fSTom Lendacky
2741d6b8a6fSTom Lendacky return 0;
2751d6b8a6fSTom Lendacky }
2761d6b8a6fSTom Lendacky
ccp_register_aes_xts_algs(struct list_head * head)2771d6b8a6fSTom Lendacky int ccp_register_aes_xts_algs(struct list_head *head)
2781d6b8a6fSTom Lendacky {
2791d6b8a6fSTom Lendacky int i, ret;
2801d6b8a6fSTom Lendacky
2811d6b8a6fSTom Lendacky for (i = 0; i < ARRAY_SIZE(aes_xts_algs); i++) {
2821d6b8a6fSTom Lendacky ret = ccp_register_aes_xts_alg(head, &aes_xts_algs[i]);
2831d6b8a6fSTom Lendacky if (ret)
2841d6b8a6fSTom Lendacky return ret;
2851d6b8a6fSTom Lendacky }
2861d6b8a6fSTom Lendacky
2871d6b8a6fSTom Lendacky return 0;
2881d6b8a6fSTom Lendacky }
289