1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cryptographic API. 4 * 5 * Support for OMAP AES GCM HW acceleration. 6 * 7 * Copyright (c) 2016 Texas Instruments Incorporated 8 */ 9 10 #include <linux/errno.h> 11 #include <linux/scatterlist.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/dmaengine.h> 14 #include <linux/omap-dma.h> 15 #include <linux/interrupt.h> 16 #include <linux/pm_runtime.h> 17 #include <crypto/aes.h> 18 #include <crypto/gcm.h> 19 #include <crypto/scatterwalk.h> 20 #include <crypto/skcipher.h> 21 #include <crypto/internal/aead.h> 22 23 #include "omap-crypto.h" 24 #include "omap-aes.h" 25 26 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 27 struct aead_request *req); 28 29 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) 30 { 31 struct aead_request *req = dd->aead_req; 32 33 dd->in_sg = NULL; 34 dd->out_sg = NULL; 35 36 crypto_finalize_aead_request(dd->engine, req, ret); 37 38 pm_runtime_mark_last_busy(dd->dev); 39 pm_runtime_put_autosuspend(dd->dev); 40 } 41 42 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) 43 { 44 u8 *tag; 45 int alen, clen, i, ret = 0, nsg; 46 struct omap_aes_reqctx *rctx; 47 48 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); 49 clen = ALIGN(dd->total, AES_BLOCK_SIZE); 50 rctx = aead_request_ctx(dd->aead_req); 51 52 nsg = !!(dd->assoc_len && dd->total); 53 54 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, 55 DMA_FROM_DEVICE); 56 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); 57 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); 58 omap_aes_crypt_dma_stop(dd); 59 60 omap_crypto_cleanup(dd->out_sg, dd->orig_out, 61 dd->aead_req->assoclen, dd->total, 62 FLAGS_OUT_DATA_ST_SHIFT, dd->flags); 63 64 if (dd->flags & FLAGS_ENCRYPT) 65 scatterwalk_map_and_copy(rctx->auth_tag, 66 dd->aead_req->dst, 67 dd->total + dd->aead_req->assoclen, 68 dd->authsize, 1); 69 70 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, 71 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); 72 73 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, 74 FLAGS_IN_DATA_ST_SHIFT, dd->flags); 75 76 if (!(dd->flags & FLAGS_ENCRYPT)) { 77 tag = (u8 *)rctx->auth_tag; 78 for (i = 0; i < dd->authsize; i++) { 79 if (tag[i]) { 80 ret = -EBADMSG; 81 } 82 } 83 } 84 85 omap_aes_gcm_finish_req(dd, ret); 86 } 87 88 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, 89 struct aead_request *req) 90 { 91 int alen, clen, cryptlen, assoclen, ret; 92 struct crypto_aead *aead = crypto_aead_reqtfm(req); 93 unsigned int authlen = crypto_aead_authsize(aead); 94 struct scatterlist *tmp, sg_arr[2]; 95 int nsg; 96 u16 flags; 97 98 assoclen = req->assoclen; 99 cryptlen = req->cryptlen; 100 101 if (dd->flags & FLAGS_RFC4106_GCM) 102 assoclen -= 8; 103 104 if (!(dd->flags & FLAGS_ENCRYPT)) 105 cryptlen -= authlen; 106 107 alen = ALIGN(assoclen, AES_BLOCK_SIZE); 108 clen = ALIGN(cryptlen, AES_BLOCK_SIZE); 109 110 nsg = !!(assoclen && cryptlen); 111 112 omap_aes_clear_copy_flags(dd); 113 114 sg_init_table(dd->in_sgl, nsg + 1); 115 if (assoclen) { 116 tmp = req->src; 117 ret = omap_crypto_align_sg(&tmp, assoclen, 118 AES_BLOCK_SIZE, dd->in_sgl, 119 OMAP_CRYPTO_COPY_DATA | 120 OMAP_CRYPTO_ZERO_BUF | 121 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 122 FLAGS_ASSOC_DATA_ST_SHIFT, 123 &dd->flags); 124 if (ret) 125 return ret; 126 } 127 128 if (cryptlen) { 129 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); 130 131 if (nsg) 132 sg_unmark_end(dd->in_sgl); 133 134 ret = omap_crypto_align_sg(&tmp, cryptlen, 135 AES_BLOCK_SIZE, &dd->in_sgl[nsg], 136 OMAP_CRYPTO_COPY_DATA | 137 OMAP_CRYPTO_ZERO_BUF | 138 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 139 FLAGS_IN_DATA_ST_SHIFT, 140 &dd->flags); 141 if (ret) 142 return ret; 143 } 144 145 dd->in_sg = dd->in_sgl; 146 dd->total = cryptlen; 147 dd->assoc_len = assoclen; 148 dd->authsize = authlen; 149 150 dd->out_sg = req->dst; 151 dd->orig_out = req->dst; 152 153 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen); 154 155 flags = 0; 156 if (req->src == req->dst || dd->out_sg == sg_arr) 157 flags |= OMAP_CRYPTO_FORCE_COPY; 158 159 if (cryptlen) { 160 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, 161 AES_BLOCK_SIZE, &dd->out_sgl, 162 flags, 163 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); 164 if (ret) 165 return ret; 166 } 167 168 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); 169 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); 170 171 return 0; 172 } 173 174 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) 175 { 176 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 177 178 aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv); 179 return 0; 180 } 181 182 void omap_aes_gcm_dma_out_callback(void *data) 183 { 184 struct omap_aes_dev *dd = data; 185 struct omap_aes_reqctx *rctx; 186 int i, val; 187 u32 *auth_tag, tag[4]; 188 189 if (!(dd->flags & FLAGS_ENCRYPT)) 190 scatterwalk_map_and_copy(tag, dd->aead_req->src, 191 dd->total + dd->aead_req->assoclen, 192 dd->authsize, 0); 193 194 rctx = aead_request_ctx(dd->aead_req); 195 auth_tag = (u32 *)rctx->auth_tag; 196 for (i = 0; i < 4; i++) { 197 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); 198 auth_tag[i] = val ^ auth_tag[i]; 199 if (!(dd->flags & FLAGS_ENCRYPT)) 200 auth_tag[i] = auth_tag[i] ^ tag[i]; 201 } 202 203 omap_aes_gcm_done_task(dd); 204 } 205 206 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 207 struct aead_request *req) 208 { 209 if (req) 210 return crypto_transfer_aead_request_to_engine(dd->engine, req); 211 212 return 0; 213 } 214 215 static int omap_aes_gcm_prepare_req(struct aead_request *req, 216 struct omap_aes_dev *dd) 217 { 218 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 219 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 220 int err; 221 222 dd->aead_req = req; 223 224 rctx->mode &= FLAGS_MODE_MASK; 225 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; 226 227 err = omap_aes_gcm_copy_buffers(dd, req); 228 if (err) 229 return err; 230 231 dd->ctx = &ctx->octx; 232 233 return omap_aes_write_ctrl(dd); 234 } 235 236 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) 237 { 238 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 239 struct crypto_aead *aead = crypto_aead_reqtfm(req); 240 unsigned int authlen = crypto_aead_authsize(aead); 241 struct omap_aes_dev *dd; 242 __be32 counter = cpu_to_be32(1); 243 int err, assoclen; 244 245 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); 246 memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); 247 248 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); 249 if (err) 250 return err; 251 252 if (mode & FLAGS_RFC4106_GCM) 253 assoclen = req->assoclen - 8; 254 else 255 assoclen = req->assoclen; 256 if (assoclen + req->cryptlen == 0) { 257 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, 258 1); 259 return 0; 260 } 261 262 dd = omap_aes_find_dev(rctx); 263 if (!dd) 264 return -ENODEV; 265 rctx->mode = mode; 266 267 return omap_aes_gcm_handle_queue(dd, req); 268 } 269 270 int omap_aes_gcm_encrypt(struct aead_request *req) 271 { 272 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 273 274 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 275 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); 276 } 277 278 int omap_aes_gcm_decrypt(struct aead_request *req) 279 { 280 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 281 282 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 283 return omap_aes_gcm_crypt(req, FLAGS_GCM); 284 } 285 286 int omap_aes_4106gcm_encrypt(struct aead_request *req) 287 { 288 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 289 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 290 291 memcpy(rctx->iv, ctx->octx.nonce, 4); 292 memcpy(rctx->iv + 4, req->iv, 8); 293 return crypto_ipsec_check_assoclen(req->assoclen) ?: 294 omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | 295 FLAGS_RFC4106_GCM); 296 } 297 298 int omap_aes_4106gcm_decrypt(struct aead_request *req) 299 { 300 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 301 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 302 303 memcpy(rctx->iv, ctx->octx.nonce, 4); 304 memcpy(rctx->iv + 4, req->iv, 8); 305 return crypto_ipsec_check_assoclen(req->assoclen) ?: 306 omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); 307 } 308 309 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 310 unsigned int keylen) 311 { 312 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 313 int ret; 314 315 ret = aes_expandkey(&ctx->actx, key, keylen); 316 if (ret) 317 return ret; 318 319 memcpy(ctx->octx.key, key, keylen); 320 ctx->octx.keylen = keylen; 321 322 return 0; 323 } 324 325 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, 326 unsigned int keylen) 327 { 328 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 329 int ret; 330 331 if (keylen < 4) 332 return -EINVAL; 333 keylen -= 4; 334 335 ret = aes_expandkey(&ctx->actx, key, keylen); 336 if (ret) 337 return ret; 338 339 memcpy(ctx->octx.key, key, keylen); 340 memcpy(ctx->octx.nonce, key + keylen, 4); 341 ctx->octx.keylen = keylen; 342 343 return 0; 344 } 345 346 int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 347 { 348 return crypto_gcm_check_authsize(authsize); 349 } 350 351 int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, 352 unsigned int authsize) 353 { 354 return crypto_rfc4106_check_authsize(authsize); 355 } 356 357 static int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq) 358 { 359 struct aead_request *req = container_of(areq, struct aead_request, 360 base); 361 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 362 struct omap_aes_dev *dd = rctx->dd; 363 int ret; 364 365 if (!dd) 366 return -ENODEV; 367 368 ret = omap_aes_gcm_prepare_req(req, dd); 369 if (ret) 370 return ret; 371 372 if (dd->in_sg_len) 373 ret = omap_aes_crypt_dma_start(dd); 374 else 375 omap_aes_gcm_dma_out_callback(dd); 376 377 return ret; 378 } 379 380 int omap_aes_gcm_cra_init(struct crypto_aead *tfm) 381 { 382 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); 383 384 ctx->enginectx.op.do_one_request = omap_aes_gcm_crypt_req; 385 386 crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx)); 387 388 return 0; 389 } 390