1 /* 2 * Cryptographic API. 3 * 4 * Support for OMAP AES GCM HW acceleration. 5 * 6 * Copyright (c) 2016 Texas Instruments Incorporated 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as published 10 * by the Free Software Foundation. 11 * 12 */ 13 14 #include <linux/errno.h> 15 #include <linux/scatterlist.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/dmaengine.h> 18 #include <linux/omap-dma.h> 19 #include <linux/interrupt.h> 20 #include <crypto/aes.h> 21 #include <crypto/scatterwalk.h> 22 #include <crypto/skcipher.h> 23 #include <crypto/internal/aead.h> 24 25 #include "omap-crypto.h" 26 #include "omap-aes.h" 27 28 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 29 struct aead_request *req); 30 31 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) 32 { 33 struct aead_request *req = dd->aead_req; 34 35 dd->flags &= ~FLAGS_BUSY; 36 dd->in_sg = NULL; 37 dd->out_sg = NULL; 38 39 req->base.complete(&req->base, ret); 40 } 41 42 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) 43 { 44 u8 *tag; 45 int alen, clen, i, ret = 0, nsg; 46 struct omap_aes_reqctx *rctx; 47 48 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); 49 clen = ALIGN(dd->total, AES_BLOCK_SIZE); 50 rctx = aead_request_ctx(dd->aead_req); 51 52 nsg = !!(dd->assoc_len && dd->total); 53 54 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, 55 DMA_FROM_DEVICE); 56 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); 57 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); 58 omap_aes_crypt_dma_stop(dd); 59 60 omap_crypto_cleanup(dd->out_sg, dd->orig_out, 61 dd->aead_req->assoclen, dd->total, 62 FLAGS_OUT_DATA_ST_SHIFT, dd->flags); 63 64 if (dd->flags & FLAGS_ENCRYPT) 65 scatterwalk_map_and_copy(rctx->auth_tag, 66 dd->aead_req->dst, 67 dd->total + dd->aead_req->assoclen, 68 dd->authsize, 1); 69 70 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, 71 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); 72 73 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, 74 FLAGS_IN_DATA_ST_SHIFT, dd->flags); 75 76 if (!(dd->flags & FLAGS_ENCRYPT)) { 77 tag = (u8 *)rctx->auth_tag; 78 for (i = 0; i < dd->authsize; i++) { 79 if (tag[i]) { 80 dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n"); 81 ret = -EBADMSG; 82 } 83 } 84 } 85 86 omap_aes_gcm_finish_req(dd, ret); 87 omap_aes_gcm_handle_queue(dd, NULL); 88 } 89 90 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, 91 struct aead_request *req) 92 { 93 int alen, clen, cryptlen, assoclen, ret; 94 struct crypto_aead *aead = crypto_aead_reqtfm(req); 95 unsigned int authlen = crypto_aead_authsize(aead); 96 struct scatterlist *tmp, sg_arr[2]; 97 int nsg; 98 u16 flags; 99 100 assoclen = req->assoclen; 101 cryptlen = req->cryptlen; 102 103 if (dd->flags & FLAGS_RFC4106_GCM) 104 assoclen -= 8; 105 106 if (!(dd->flags & FLAGS_ENCRYPT)) 107 cryptlen -= authlen; 108 109 alen = ALIGN(assoclen, AES_BLOCK_SIZE); 110 clen = ALIGN(cryptlen, AES_BLOCK_SIZE); 111 112 nsg = !!(assoclen && cryptlen); 113 114 omap_aes_clear_copy_flags(dd); 115 116 sg_init_table(dd->in_sgl, nsg + 1); 117 if (assoclen) { 118 tmp = req->src; 119 ret = omap_crypto_align_sg(&tmp, assoclen, 120 AES_BLOCK_SIZE, dd->in_sgl, 121 OMAP_CRYPTO_COPY_DATA | 122 OMAP_CRYPTO_ZERO_BUF | 123 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 124 FLAGS_ASSOC_DATA_ST_SHIFT, 125 &dd->flags); 126 } 127 128 if (cryptlen) { 129 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); 130 131 ret = omap_crypto_align_sg(&tmp, cryptlen, 132 AES_BLOCK_SIZE, &dd->in_sgl[nsg], 133 OMAP_CRYPTO_COPY_DATA | 134 OMAP_CRYPTO_ZERO_BUF | 135 OMAP_CRYPTO_FORCE_SINGLE_ENTRY, 136 FLAGS_IN_DATA_ST_SHIFT, 137 &dd->flags); 138 } 139 140 dd->in_sg = dd->in_sgl; 141 dd->total = cryptlen; 142 dd->assoc_len = assoclen; 143 dd->authsize = authlen; 144 145 dd->out_sg = req->dst; 146 dd->orig_out = req->dst; 147 148 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen); 149 150 flags = 0; 151 if (req->src == req->dst || dd->out_sg == sg_arr) 152 flags |= OMAP_CRYPTO_FORCE_COPY; 153 154 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, 155 AES_BLOCK_SIZE, &dd->out_sgl, 156 flags, 157 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); 158 if (ret) 159 return ret; 160 161 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); 162 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); 163 164 return 0; 165 } 166 167 static void omap_aes_gcm_complete(struct crypto_async_request *req, int err) 168 { 169 struct omap_aes_gcm_result *res = req->data; 170 171 if (err == -EINPROGRESS) 172 return; 173 174 res->err = err; 175 complete(&res->completion); 176 } 177 178 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) 179 { 180 struct scatterlist iv_sg, tag_sg; 181 struct skcipher_request *sk_req; 182 struct omap_aes_gcm_result result; 183 struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 184 int ret = 0; 185 186 sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL); 187 if (!sk_req) { 188 pr_err("skcipher: Failed to allocate request\n"); 189 return -1; 190 } 191 192 init_completion(&result.completion); 193 194 sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE); 195 sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE); 196 skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 197 omap_aes_gcm_complete, &result); 198 ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen); 199 skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE, 200 NULL); 201 ret = crypto_skcipher_encrypt(sk_req); 202 switch (ret) { 203 case 0: 204 break; 205 case -EINPROGRESS: 206 case -EBUSY: 207 ret = wait_for_completion_interruptible(&result.completion); 208 if (!ret) { 209 ret = result.err; 210 if (!ret) { 211 reinit_completion(&result.completion); 212 break; 213 } 214 } 215 /* fall through */ 216 default: 217 pr_err("Encryption of IV failed for GCM mode"); 218 break; 219 } 220 221 skcipher_request_free(sk_req); 222 return ret; 223 } 224 225 void omap_aes_gcm_dma_out_callback(void *data) 226 { 227 struct omap_aes_dev *dd = data; 228 struct omap_aes_reqctx *rctx; 229 int i, val; 230 u32 *auth_tag, tag[4]; 231 232 if (!(dd->flags & FLAGS_ENCRYPT)) 233 scatterwalk_map_and_copy(tag, dd->aead_req->src, 234 dd->total + dd->aead_req->assoclen, 235 dd->authsize, 0); 236 237 rctx = aead_request_ctx(dd->aead_req); 238 auth_tag = (u32 *)rctx->auth_tag; 239 for (i = 0; i < 4; i++) { 240 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); 241 auth_tag[i] = val ^ auth_tag[i]; 242 if (!(dd->flags & FLAGS_ENCRYPT)) 243 auth_tag[i] = auth_tag[i] ^ tag[i]; 244 } 245 246 omap_aes_gcm_done_task(dd); 247 } 248 249 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, 250 struct aead_request *req) 251 { 252 struct omap_aes_ctx *ctx; 253 struct aead_request *backlog; 254 struct omap_aes_reqctx *rctx; 255 unsigned long flags; 256 int err, ret = 0; 257 258 spin_lock_irqsave(&dd->lock, flags); 259 if (req) 260 ret = aead_enqueue_request(&dd->aead_queue, req); 261 if (dd->flags & FLAGS_BUSY) { 262 spin_unlock_irqrestore(&dd->lock, flags); 263 return ret; 264 } 265 266 backlog = aead_get_backlog(&dd->aead_queue); 267 req = aead_dequeue_request(&dd->aead_queue); 268 if (req) 269 dd->flags |= FLAGS_BUSY; 270 spin_unlock_irqrestore(&dd->lock, flags); 271 272 if (!req) 273 return ret; 274 275 if (backlog) 276 backlog->base.complete(&backlog->base, -EINPROGRESS); 277 278 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 279 rctx = aead_request_ctx(req); 280 281 dd->ctx = ctx; 282 rctx->dd = dd; 283 dd->aead_req = req; 284 285 rctx->mode &= FLAGS_MODE_MASK; 286 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; 287 288 err = omap_aes_gcm_copy_buffers(dd, req); 289 if (err) 290 return err; 291 292 err = omap_aes_write_ctrl(dd); 293 if (!err) 294 err = omap_aes_crypt_dma_start(dd); 295 296 if (err) { 297 omap_aes_gcm_finish_req(dd, err); 298 omap_aes_gcm_handle_queue(dd, NULL); 299 } 300 301 return ret; 302 } 303 304 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) 305 { 306 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 307 struct crypto_aead *aead = crypto_aead_reqtfm(req); 308 unsigned int authlen = crypto_aead_authsize(aead); 309 struct omap_aes_dev *dd; 310 __be32 counter = cpu_to_be32(1); 311 int err, assoclen; 312 313 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); 314 memcpy(rctx->iv + 12, &counter, 4); 315 316 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); 317 if (err) 318 return err; 319 320 if (mode & FLAGS_RFC4106_GCM) 321 assoclen = req->assoclen - 8; 322 else 323 assoclen = req->assoclen; 324 if (assoclen + req->cryptlen == 0) { 325 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, 326 1); 327 return 0; 328 } 329 330 dd = omap_aes_find_dev(rctx); 331 if (!dd) 332 return -ENODEV; 333 rctx->mode = mode; 334 335 return omap_aes_gcm_handle_queue(dd, req); 336 } 337 338 int omap_aes_gcm_encrypt(struct aead_request *req) 339 { 340 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 341 342 memcpy(rctx->iv, req->iv, 12); 343 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); 344 } 345 346 int omap_aes_gcm_decrypt(struct aead_request *req) 347 { 348 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 349 350 memcpy(rctx->iv, req->iv, 12); 351 return omap_aes_gcm_crypt(req, FLAGS_GCM); 352 } 353 354 int omap_aes_4106gcm_encrypt(struct aead_request *req) 355 { 356 struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 357 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 358 359 memcpy(rctx->iv, ctx->nonce, 4); 360 memcpy(rctx->iv + 4, req->iv, 8); 361 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | 362 FLAGS_RFC4106_GCM); 363 } 364 365 int omap_aes_4106gcm_decrypt(struct aead_request *req) 366 { 367 struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 368 struct omap_aes_reqctx *rctx = aead_request_ctx(req); 369 370 memcpy(rctx->iv, ctx->nonce, 4); 371 memcpy(rctx->iv + 4, req->iv, 8); 372 return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); 373 } 374 375 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 376 unsigned int keylen) 377 { 378 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); 379 380 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 381 keylen != AES_KEYSIZE_256) 382 return -EINVAL; 383 384 memcpy(ctx->key, key, keylen); 385 ctx->keylen = keylen; 386 387 return 0; 388 } 389 390 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, 391 unsigned int keylen) 392 { 393 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); 394 395 if (keylen < 4) 396 return -EINVAL; 397 398 keylen -= 4; 399 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 400 keylen != AES_KEYSIZE_256) 401 return -EINVAL; 402 403 memcpy(ctx->key, key, keylen); 404 memcpy(ctx->nonce, key + keylen, 4); 405 ctx->keylen = keylen; 406 407 return 0; 408 } 409