1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cryptographic API. 4 * 5 * Support for OMAP AES GCM HW acceleration. 6 * 7 * Copyright (c) 2016 Texas Instruments Incorporated 8 */ 9 10 #include <linux/errno.h> 11 #include <linux/scatterlist.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/dmaengine.h> 14 #include <linux/omap-dma.h> 15 #include <linux/interrupt.h> 16 #include <linux/pm_runtime.h> 17 #include <crypto/aes.h> 18 #include <crypto/gcm.h> 19 #include <crypto/scatterwalk.h> 20 #include <crypto/skcipher.h> 21 #include <crypto/internal/aead.h> 22 23 #include "omap-crypto.h" 24 #include "omap-aes.h" 25 26 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 27 struct aead_request *req); 28 29 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) 30 { 31 struct aead_request *req = dd->aead_req; 32 33 dd->in_sg = NULL; 34 dd->out_sg = NULL; 35 36 crypto_finalize_aead_request(dd->engine, req, ret); 37 38 pm_runtime_mark_last_busy(dd->dev); 39 pm_runtime_put_autosuspend(dd->dev); 40 } 41 42 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) 43 { 44 u8 *tag; 45 int alen, clen, i, ret = 0, nsg; 46 struct omap_aes_reqctx *rctx; 47 48 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); 49 clen = ALIGN(dd->total, AES_BLOCK_SIZE); 50 rctx = aead_request_ctx(dd->aead_req); 51 52 nsg = !!(dd->assoc_len && dd->total); 53 54 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, 55 DMA_FROM_DEVICE); 56 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); 57 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); 58 omap_aes_crypt_dma_stop(dd); 59 60 omap_crypto_cleanup(dd->out_sg, dd->orig_out, 61 dd->aead_req->assoclen, dd->total, 62 FLAGS_OUT_DATA_ST_SHIFT, dd->flags); 63 64 if (dd->flags & FLAGS_ENCRYPT) 65 scatterwalk_map_and_copy(rctx->auth_tag, 66 dd->aead_req->dst, 67 dd->total + dd->aead_req->assoclen, 68 dd->authsize, 1); 69 70 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, 71 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); 72 73 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, 74 FLAGS_IN_DATA_ST_SHIFT, dd->flags); 75 76 if (!(dd->flags & FLAGS_ENCRYPT)) { 77 tag = (u8 *)rctx->auth_tag; 78 for (i = 0; i < dd->authsize; i++) { 79 if (tag[i]) { 80 dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n"); 81 ret = -EBADMSG; 82 } 83 } 84 } 85 86 omap_aes_gcm_finish_req(dd, ret); 87 } 88 89 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, 90 struct aead_request *req) 91 { 92 int alen, clen, cryptlen, assoclen, ret; 93 struct crypto_aead *aead = crypto_aead_reqtfm(req); 94 unsigned int authlen = crypto_aead_authsize(aead); 95 struct scatterlist *tmp, sg_arr[2]; 96 int nsg; 97 u16 flags; 98 99 assoclen = req->assoclen; 100 cryptlen = req->cryptlen; 101 102 if (dd->flags & FLAGS_RFC4106_GCM) 103 assoclen -= 8; 104 105 if (!(dd->flags & FLAGS_ENCRYPT)) 106 cryptlen -= authlen; 107 108 alen = ALIGN(assoclen, AES_BLOCK_SIZE); 109 clen = ALIGN(cryptlen, AES_BLOCK_SIZE); 110 111 nsg = !!(assoclen && cryptlen); 112 113 omap_aes_clear_copy_flags(dd); 114 115 sg_init_table(dd->in_sgl, nsg + 1); 116 if (assoclen) { 117 tmp = req->src; 118 ret = omap_crypto_align_sg(&tmp, assoclen, 119 AES_BLOCK_SIZE, dd->in_sgl, 120 OMAP_CRYPTO_COPY_DATA | 121 OMAP_CRYPTO_ZERO_BUF | 122 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 123 FLAGS_ASSOC_DATA_ST_SHIFT, 124 &dd->flags); 125 if (ret) 126 return ret; 127 } 128 129 if (cryptlen) { 130 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); 131 132 if (nsg) 133 sg_unmark_end(dd->in_sgl); 134 135 ret = omap_crypto_align_sg(&tmp, cryptlen, 136 AES_BLOCK_SIZE, &dd->in_sgl[nsg], 137 OMAP_CRYPTO_COPY_DATA | 138 OMAP_CRYPTO_ZERO_BUF | 139 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 140 FLAGS_IN_DATA_ST_SHIFT, 141 &dd->flags); 142 if (ret) 143 return ret; 144 } 145 146 dd->in_sg = dd->in_sgl; 147 dd->total = cryptlen; 148 dd->assoc_len = assoclen; 149 dd->authsize = authlen; 150 151 dd->out_sg = req->dst; 152 dd->orig_out = req->dst; 153 154 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen); 155 156 flags = 0; 157 if (req->src == req->dst || dd->out_sg == sg_arr) 158 flags |= OMAP_CRYPTO_FORCE_COPY; 159 160 if (cryptlen) { 161 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, 162 AES_BLOCK_SIZE, &dd->out_sgl, 163 flags, 164 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); 165 if (ret) 166 return ret; 167 } 168 169 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); 170 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); 171 172 return 0; 173 } 174 175 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) 176 { 177 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 178 179 aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv); 180 return 0; 181 } 182 183 void omap_aes_gcm_dma_out_callback(void *data) 184 { 185 struct omap_aes_dev *dd = data; 186 struct omap_aes_reqctx *rctx; 187 int i, val; 188 u32 *auth_tag, tag[4]; 189 190 if (!(dd->flags & FLAGS_ENCRYPT)) 191 scatterwalk_map_and_copy(tag, dd->aead_req->src, 192 dd->total + dd->aead_req->assoclen, 193 dd->authsize, 0); 194 195 rctx = aead_request_ctx(dd->aead_req); 196 auth_tag = (u32 *)rctx->auth_tag; 197 for (i = 0; i < 4; i++) { 198 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); 199 auth_tag[i] = val ^ auth_tag[i]; 200 if (!(dd->flags & FLAGS_ENCRYPT)) 201 auth_tag[i] = auth_tag[i] ^ tag[i]; 202 } 203 204 omap_aes_gcm_done_task(dd); 205 } 206 207 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 208 struct aead_request *req) 209 { 210 if (req) 211 return crypto_transfer_aead_request_to_engine(dd->engine, req); 212 213 return 0; 214 } 215 216 static int omap_aes_gcm_prepare_req(struct crypto_engine *engine, void *areq) 217 { 218 struct aead_request *req = container_of(areq, struct aead_request, 219 base); 220 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 221 struct omap_aes_dev *dd = rctx->dd; 222 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 223 int err; 224 225 dd->aead_req = req; 226 227 rctx->mode &= FLAGS_MODE_MASK; 228 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; 229 230 err = omap_aes_gcm_copy_buffers(dd, req); 231 if (err) 232 return err; 233 234 dd->ctx = &ctx->octx; 235 236 return omap_aes_write_ctrl(dd); 237 } 238 239 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) 240 { 241 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 242 struct crypto_aead *aead = crypto_aead_reqtfm(req); 243 unsigned int authlen = crypto_aead_authsize(aead); 244 struct omap_aes_dev *dd; 245 __be32 counter = cpu_to_be32(1); 246 int err, assoclen; 247 248 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); 249 memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); 250 251 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); 252 if (err) 253 return err; 254 255 if (mode & FLAGS_RFC4106_GCM) 256 assoclen = req->assoclen - 8; 257 else 258 assoclen = req->assoclen; 259 if (assoclen + req->cryptlen == 0) { 260 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, 261 1); 262 return 0; 263 } 264 265 dd = omap_aes_find_dev(rctx); 266 if (!dd) 267 return -ENODEV; 268 rctx->mode = mode; 269 270 return omap_aes_gcm_handle_queue(dd, req); 271 } 272 273 int omap_aes_gcm_encrypt(struct aead_request *req) 274 { 275 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 276 277 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 278 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); 279 } 280 281 int omap_aes_gcm_decrypt(struct aead_request *req) 282 { 283 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 284 285 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); 286 return omap_aes_gcm_crypt(req, FLAGS_GCM); 287 } 288 289 int omap_aes_4106gcm_encrypt(struct aead_request *req) 290 { 291 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 292 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 293 294 memcpy(rctx->iv, ctx->octx.nonce, 4); 295 memcpy(rctx->iv + 4, req->iv, 8); 296 return crypto_ipsec_check_assoclen(req->assoclen) ?: 297 omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | 298 FLAGS_RFC4106_GCM); 299 } 300 301 int omap_aes_4106gcm_decrypt(struct aead_request *req) 302 { 303 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 304 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 305 306 memcpy(rctx->iv, ctx->octx.nonce, 4); 307 memcpy(rctx->iv + 4, req->iv, 8); 308 return crypto_ipsec_check_assoclen(req->assoclen) ?: 309 omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); 310 } 311 312 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 313 unsigned int keylen) 314 { 315 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 316 int ret; 317 318 ret = aes_expandkey(&ctx->actx, key, keylen); 319 if (ret) 320 return ret; 321 322 memcpy(ctx->octx.key, key, keylen); 323 ctx->octx.keylen = keylen; 324 325 return 0; 326 } 327 328 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, 329 unsigned int keylen) 330 { 331 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 332 int ret; 333 334 if (keylen < 4) 335 return -EINVAL; 336 keylen -= 4; 337 338 ret = aes_expandkey(&ctx->actx, key, keylen); 339 if (ret) 340 return ret; 341 342 memcpy(ctx->octx.key, key, keylen); 343 memcpy(ctx->octx.nonce, key + keylen, 4); 344 ctx->octx.keylen = keylen; 345 346 return 0; 347 } 348 349 int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 350 { 351 return crypto_gcm_check_authsize(authsize); 352 } 353 354 int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, 355 unsigned int authsize) 356 { 357 return crypto_rfc4106_check_authsize(authsize); 358 } 359 360 static int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq) 361 { 362 struct aead_request *req = container_of(areq, struct aead_request, 363 base); 364 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 365 struct omap_aes_dev *dd = rctx->dd; 366 int ret = 0; 367 368 if (!dd) 369 return -ENODEV; 370 371 if (dd->in_sg_len) 372 ret = omap_aes_crypt_dma_start(dd); 373 else 374 omap_aes_gcm_dma_out_callback(dd); 375 376 return ret; 377 } 378 379 int omap_aes_gcm_cra_init(struct crypto_aead *tfm) 380 { 381 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); 382 383 ctx->enginectx.op.prepare_request = omap_aes_gcm_prepare_req; 384 ctx->enginectx.op.unprepare_request = NULL; 385 ctx->enginectx.op.do_one_request = omap_aes_gcm_crypt_req; 386 387 crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx)); 388 389 return 0; 390 } 391