1 /* 2 * Freescale FSL CAAM support for crypto API over QI backend. 3 * Based on caamalg.c 4 * 5 * Copyright 2013-2016 Freescale Semiconductor, Inc. 6 * Copyright 2016-2017 NXP 7 */ 8 9 #include "compat.h" 10 #include "ctrl.h" 11 #include "regs.h" 12 #include "intern.h" 13 #include "desc_constr.h" 14 #include "error.h" 15 #include "sg_sw_qm.h" 16 #include "key_gen.h" 17 #include "qi.h" 18 #include "jr.h" 19 #include "caamalg_desc.h" 20 21 /* 22 * crypto alg 23 */ 24 #define CAAM_CRA_PRIORITY 2000 25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 26 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 27 SHA512_DIGEST_SIZE * 2) 28 29 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 30 CAAM_MAX_KEY_SIZE) 31 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 32 33 struct caam_alg_entry { 34 int class1_alg_type; 35 int class2_alg_type; 36 bool rfc3686; 37 bool geniv; 38 }; 39 40 struct caam_aead_alg { 41 struct aead_alg aead; 42 struct caam_alg_entry caam; 43 bool registered; 44 }; 45 46 /* 47 * per-session context 48 */ 49 struct caam_ctx { 50 struct device *jrdev; 51 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 52 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 53 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 54 u8 key[CAAM_MAX_KEY_SIZE]; 55 dma_addr_t key_dma; 56 enum dma_data_direction dir; 57 struct alginfo adata; 58 struct alginfo cdata; 59 unsigned int authsize; 60 struct device *qidev; 61 spinlock_t lock; /* Protects multiple init of driver context */ 62 struct caam_drv_ctx *drv_ctx[NUM_OP]; 63 }; 64 65 static int aead_set_sh_desc(struct crypto_aead *aead) 66 { 67 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 68 typeof(*alg), aead); 69 struct caam_ctx *ctx = crypto_aead_ctx(aead); 70 unsigned int ivsize = crypto_aead_ivsize(aead); 71 u32 ctx1_iv_off = 0; 72 u32 *nonce = NULL; 73 unsigned int data_len[2]; 74 u32 inl_mask; 75 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 76 OP_ALG_AAI_CTR_MOD128); 77 const bool is_rfc3686 = alg->caam.rfc3686; 78 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 79 80 if (!ctx->cdata.keylen || !ctx->authsize) 81 return 0; 82 83 /* 84 * AES-CTR needs to load IV in CONTEXT1 reg 85 * at an offset of 128bits (16bytes) 86 * CONTEXT1[255:128] = IV 87 */ 88 if (ctr_mode) 89 ctx1_iv_off = 16; 90 91 /* 92 * RFC3686 specific: 93 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 94 */ 95 if (is_rfc3686) { 96 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 97 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 98 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 99 } 100 101 data_len[0] = ctx->adata.keylen_pad; 102 data_len[1] = ctx->cdata.keylen; 103 104 if (alg->caam.geniv) 105 goto skip_enc; 106 107 /* aead_encrypt shared descriptor */ 108 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 109 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 110 DESC_JOB_IO_LEN, data_len, &inl_mask, 111 ARRAY_SIZE(data_len)) < 0) 112 return -EINVAL; 113 114 if (inl_mask & 1) 115 ctx->adata.key_virt = ctx->key; 116 else 117 ctx->adata.key_dma = ctx->key_dma; 118 119 if (inl_mask & 2) 120 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 121 else 122 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 123 124 ctx->adata.key_inline = !!(inl_mask & 1); 125 ctx->cdata.key_inline = !!(inl_mask & 2); 126 127 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 128 ivsize, ctx->authsize, is_rfc3686, nonce, 129 ctx1_iv_off, true, ctrlpriv->era); 130 131 skip_enc: 132 /* aead_decrypt shared descriptor */ 133 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 134 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 135 DESC_JOB_IO_LEN, data_len, &inl_mask, 136 ARRAY_SIZE(data_len)) < 0) 137 return -EINVAL; 138 139 if (inl_mask & 1) 140 ctx->adata.key_virt = ctx->key; 141 else 142 ctx->adata.key_dma = ctx->key_dma; 143 144 if (inl_mask & 2) 145 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 146 else 147 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 148 149 ctx->adata.key_inline = !!(inl_mask & 1); 150 ctx->cdata.key_inline = !!(inl_mask & 2); 151 152 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 153 ivsize, ctx->authsize, alg->caam.geniv, 154 is_rfc3686, nonce, ctx1_iv_off, true, 155 ctrlpriv->era); 156 157 if (!alg->caam.geniv) 158 goto skip_givenc; 159 160 /* aead_givencrypt shared descriptor */ 161 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 162 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 163 DESC_JOB_IO_LEN, data_len, &inl_mask, 164 ARRAY_SIZE(data_len)) < 0) 165 return -EINVAL; 166 167 if (inl_mask & 1) 168 ctx->adata.key_virt = ctx->key; 169 else 170 ctx->adata.key_dma = ctx->key_dma; 171 172 if (inl_mask & 2) 173 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 174 else 175 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 176 177 ctx->adata.key_inline = !!(inl_mask & 1); 178 ctx->cdata.key_inline = !!(inl_mask & 2); 179 180 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 181 ivsize, ctx->authsize, is_rfc3686, nonce, 182 ctx1_iv_off, true, ctrlpriv->era); 183 184 skip_givenc: 185 return 0; 186 } 187 188 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 189 { 190 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 191 192 ctx->authsize = authsize; 193 aead_set_sh_desc(authenc); 194 195 return 0; 196 } 197 198 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 199 unsigned int keylen) 200 { 201 struct caam_ctx *ctx = crypto_aead_ctx(aead); 202 struct device *jrdev = ctx->jrdev; 203 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 204 struct crypto_authenc_keys keys; 205 int ret = 0; 206 207 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 208 goto badkey; 209 210 #ifdef DEBUG 211 dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 212 keys.authkeylen + keys.enckeylen, keys.enckeylen, 213 keys.authkeylen); 214 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 215 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 216 #endif 217 218 /* 219 * If DKP is supported, use it in the shared descriptor to generate 220 * the split key. 221 */ 222 if (ctrlpriv->era >= 6) { 223 ctx->adata.keylen = keys.authkeylen; 224 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 225 OP_ALG_ALGSEL_MASK); 226 227 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 228 goto badkey; 229 230 memcpy(ctx->key, keys.authkey, keys.authkeylen); 231 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 232 keys.enckeylen); 233 dma_sync_single_for_device(jrdev, ctx->key_dma, 234 ctx->adata.keylen_pad + 235 keys.enckeylen, ctx->dir); 236 goto skip_split_key; 237 } 238 239 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 240 keys.authkeylen, CAAM_MAX_KEY_SIZE - 241 keys.enckeylen); 242 if (ret) 243 goto badkey; 244 245 /* postpend encryption key to auth split key */ 246 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 247 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 248 keys.enckeylen, ctx->dir); 249 #ifdef DEBUG 250 print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", 251 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 252 ctx->adata.keylen_pad + keys.enckeylen, 1); 253 #endif 254 255 skip_split_key: 256 ctx->cdata.keylen = keys.enckeylen; 257 258 ret = aead_set_sh_desc(aead); 259 if (ret) 260 goto badkey; 261 262 /* Now update the driver contexts with the new shared descriptor */ 263 if (ctx->drv_ctx[ENCRYPT]) { 264 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 265 ctx->sh_desc_enc); 266 if (ret) { 267 dev_err(jrdev, "driver enc context update failed\n"); 268 goto badkey; 269 } 270 } 271 272 if (ctx->drv_ctx[DECRYPT]) { 273 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 274 ctx->sh_desc_dec); 275 if (ret) { 276 dev_err(jrdev, "driver dec context update failed\n"); 277 goto badkey; 278 } 279 } 280 281 return ret; 282 badkey: 283 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 284 return -EINVAL; 285 } 286 287 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 288 const u8 *key, unsigned int keylen) 289 { 290 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 291 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 292 const char *alg_name = crypto_tfm_alg_name(tfm); 293 struct device *jrdev = ctx->jrdev; 294 unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 295 u32 ctx1_iv_off = 0; 296 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 297 OP_ALG_AAI_CTR_MOD128); 298 const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); 299 int ret = 0; 300 301 #ifdef DEBUG 302 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 303 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 304 #endif 305 /* 306 * AES-CTR needs to load IV in CONTEXT1 reg 307 * at an offset of 128bits (16bytes) 308 * CONTEXT1[255:128] = IV 309 */ 310 if (ctr_mode) 311 ctx1_iv_off = 16; 312 313 /* 314 * RFC3686 specific: 315 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 316 * | *key = {KEY, NONCE} 317 */ 318 if (is_rfc3686) { 319 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 320 keylen -= CTR_RFC3686_NONCE_SIZE; 321 } 322 323 ctx->cdata.keylen = keylen; 324 ctx->cdata.key_virt = key; 325 ctx->cdata.key_inline = true; 326 327 /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ 328 cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 329 is_rfc3686, ctx1_iv_off); 330 cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 331 is_rfc3686, ctx1_iv_off); 332 cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata, 333 ivsize, is_rfc3686, ctx1_iv_off); 334 335 /* Now update the driver contexts with the new shared descriptor */ 336 if (ctx->drv_ctx[ENCRYPT]) { 337 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 338 ctx->sh_desc_enc); 339 if (ret) { 340 dev_err(jrdev, "driver enc context update failed\n"); 341 goto badkey; 342 } 343 } 344 345 if (ctx->drv_ctx[DECRYPT]) { 346 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 347 ctx->sh_desc_dec); 348 if (ret) { 349 dev_err(jrdev, "driver dec context update failed\n"); 350 goto badkey; 351 } 352 } 353 354 if (ctx->drv_ctx[GIVENCRYPT]) { 355 ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT], 356 ctx->sh_desc_givenc); 357 if (ret) { 358 dev_err(jrdev, "driver givenc context update failed\n"); 359 goto badkey; 360 } 361 } 362 363 return ret; 364 badkey: 365 crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 366 return -EINVAL; 367 } 368 369 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 370 const u8 *key, unsigned int keylen) 371 { 372 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 373 struct device *jrdev = ctx->jrdev; 374 int ret = 0; 375 376 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 377 crypto_ablkcipher_set_flags(ablkcipher, 378 CRYPTO_TFM_RES_BAD_KEY_LEN); 379 dev_err(jrdev, "key size mismatch\n"); 380 return -EINVAL; 381 } 382 383 ctx->cdata.keylen = keylen; 384 ctx->cdata.key_virt = key; 385 ctx->cdata.key_inline = true; 386 387 /* xts ablkcipher encrypt, decrypt shared descriptors */ 388 cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 389 cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 390 391 /* Now update the driver contexts with the new shared descriptor */ 392 if (ctx->drv_ctx[ENCRYPT]) { 393 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 394 ctx->sh_desc_enc); 395 if (ret) { 396 dev_err(jrdev, "driver enc context update failed\n"); 397 goto badkey; 398 } 399 } 400 401 if (ctx->drv_ctx[DECRYPT]) { 402 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 403 ctx->sh_desc_dec); 404 if (ret) { 405 dev_err(jrdev, "driver dec context update failed\n"); 406 goto badkey; 407 } 408 } 409 410 return ret; 411 badkey: 412 crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 413 return 0; 414 } 415 416 /* 417 * aead_edesc - s/w-extended aead descriptor 418 * @src_nents: number of segments in input scatterlist 419 * @dst_nents: number of segments in output scatterlist 420 * @iv_dma: dma address of iv for checking continuity and link table 421 * @qm_sg_bytes: length of dma mapped h/w link table 422 * @qm_sg_dma: bus physical mapped address of h/w link table 423 * @assoclen: associated data length, in CAAM endianness 424 * @assoclen_dma: bus physical mapped address of req->assoclen 425 * @drv_req: driver-specific request structure 426 * @sgt: the h/w link table 427 */ 428 struct aead_edesc { 429 int src_nents; 430 int dst_nents; 431 dma_addr_t iv_dma; 432 int qm_sg_bytes; 433 dma_addr_t qm_sg_dma; 434 unsigned int assoclen; 435 dma_addr_t assoclen_dma; 436 struct caam_drv_req drv_req; 437 #define CAAM_QI_MAX_AEAD_SG \ 438 ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \ 439 sizeof(struct qm_sg_entry)) 440 struct qm_sg_entry sgt[0]; 441 }; 442 443 /* 444 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 445 * @src_nents: number of segments in input scatterlist 446 * @dst_nents: number of segments in output scatterlist 447 * @iv_dma: dma address of iv for checking continuity and link table 448 * @qm_sg_bytes: length of dma mapped h/w link table 449 * @qm_sg_dma: bus physical mapped address of h/w link table 450 * @drv_req: driver-specific request structure 451 * @sgt: the h/w link table 452 */ 453 struct ablkcipher_edesc { 454 int src_nents; 455 int dst_nents; 456 dma_addr_t iv_dma; 457 int qm_sg_bytes; 458 dma_addr_t qm_sg_dma; 459 struct caam_drv_req drv_req; 460 #define CAAM_QI_MAX_ABLKCIPHER_SG \ 461 ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \ 462 sizeof(struct qm_sg_entry)) 463 struct qm_sg_entry sgt[0]; 464 }; 465 466 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 467 enum optype type) 468 { 469 /* 470 * This function is called on the fast path with values of 'type' 471 * known at compile time. Invalid arguments are not expected and 472 * thus no checks are made. 473 */ 474 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 475 u32 *desc; 476 477 if (unlikely(!drv_ctx)) { 478 spin_lock(&ctx->lock); 479 480 /* Read again to check if some other core init drv_ctx */ 481 drv_ctx = ctx->drv_ctx[type]; 482 if (!drv_ctx) { 483 int cpu; 484 485 if (type == ENCRYPT) 486 desc = ctx->sh_desc_enc; 487 else if (type == DECRYPT) 488 desc = ctx->sh_desc_dec; 489 else /* (type == GIVENCRYPT) */ 490 desc = ctx->sh_desc_givenc; 491 492 cpu = smp_processor_id(); 493 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 494 if (likely(!IS_ERR_OR_NULL(drv_ctx))) 495 drv_ctx->op_type = type; 496 497 ctx->drv_ctx[type] = drv_ctx; 498 } 499 500 spin_unlock(&ctx->lock); 501 } 502 503 return drv_ctx; 504 } 505 506 static void caam_unmap(struct device *dev, struct scatterlist *src, 507 struct scatterlist *dst, int src_nents, 508 int dst_nents, dma_addr_t iv_dma, int ivsize, 509 enum optype op_type, dma_addr_t qm_sg_dma, 510 int qm_sg_bytes) 511 { 512 if (dst != src) { 513 if (src_nents) 514 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 515 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 516 } else { 517 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 518 } 519 520 if (iv_dma) 521 dma_unmap_single(dev, iv_dma, ivsize, 522 op_type == GIVENCRYPT ? DMA_FROM_DEVICE : 523 DMA_TO_DEVICE); 524 if (qm_sg_bytes) 525 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 526 } 527 528 static void aead_unmap(struct device *dev, 529 struct aead_edesc *edesc, 530 struct aead_request *req) 531 { 532 struct crypto_aead *aead = crypto_aead_reqtfm(req); 533 int ivsize = crypto_aead_ivsize(aead); 534 535 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 536 edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, 537 edesc->qm_sg_dma, edesc->qm_sg_bytes); 538 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 539 } 540 541 static void ablkcipher_unmap(struct device *dev, 542 struct ablkcipher_edesc *edesc, 543 struct ablkcipher_request *req) 544 { 545 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 546 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 547 548 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 549 edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, 550 edesc->qm_sg_dma, edesc->qm_sg_bytes); 551 } 552 553 static void aead_done(struct caam_drv_req *drv_req, u32 status) 554 { 555 struct device *qidev; 556 struct aead_edesc *edesc; 557 struct aead_request *aead_req = drv_req->app_ctx; 558 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 559 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); 560 int ecode = 0; 561 562 qidev = caam_ctx->qidev; 563 564 if (unlikely(status)) { 565 caam_jr_strstatus(qidev, status); 566 ecode = -EIO; 567 } 568 569 edesc = container_of(drv_req, typeof(*edesc), drv_req); 570 aead_unmap(qidev, edesc, aead_req); 571 572 aead_request_complete(aead_req, ecode); 573 qi_cache_free(edesc); 574 } 575 576 /* 577 * allocate and map the aead extended descriptor 578 */ 579 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 580 bool encrypt) 581 { 582 struct crypto_aead *aead = crypto_aead_reqtfm(req); 583 struct caam_ctx *ctx = crypto_aead_ctx(aead); 584 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 585 typeof(*alg), aead); 586 struct device *qidev = ctx->qidev; 587 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 588 GFP_KERNEL : GFP_ATOMIC; 589 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 590 struct aead_edesc *edesc; 591 dma_addr_t qm_sg_dma, iv_dma = 0; 592 int ivsize = 0; 593 unsigned int authsize = ctx->authsize; 594 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 595 int in_len, out_len; 596 struct qm_sg_entry *sg_table, *fd_sgt; 597 struct caam_drv_ctx *drv_ctx; 598 enum optype op_type = encrypt ? ENCRYPT : DECRYPT; 599 600 drv_ctx = get_drv_ctx(ctx, op_type); 601 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 602 return (struct aead_edesc *)drv_ctx; 603 604 /* allocate space for base edesc and hw desc commands, link tables */ 605 edesc = qi_cache_alloc(GFP_DMA | flags); 606 if (unlikely(!edesc)) { 607 dev_err(qidev, "could not allocate extended descriptor\n"); 608 return ERR_PTR(-ENOMEM); 609 } 610 611 if (likely(req->src == req->dst)) { 612 src_nents = sg_nents_for_len(req->src, req->assoclen + 613 req->cryptlen + 614 (encrypt ? authsize : 0)); 615 if (unlikely(src_nents < 0)) { 616 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 617 req->assoclen + req->cryptlen + 618 (encrypt ? authsize : 0)); 619 qi_cache_free(edesc); 620 return ERR_PTR(src_nents); 621 } 622 623 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 624 DMA_BIDIRECTIONAL); 625 if (unlikely(!mapped_src_nents)) { 626 dev_err(qidev, "unable to map source\n"); 627 qi_cache_free(edesc); 628 return ERR_PTR(-ENOMEM); 629 } 630 } else { 631 src_nents = sg_nents_for_len(req->src, req->assoclen + 632 req->cryptlen); 633 if (unlikely(src_nents < 0)) { 634 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 635 req->assoclen + req->cryptlen); 636 qi_cache_free(edesc); 637 return ERR_PTR(src_nents); 638 } 639 640 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 641 req->cryptlen + 642 (encrypt ? authsize : 643 (-authsize))); 644 if (unlikely(dst_nents < 0)) { 645 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 646 req->assoclen + req->cryptlen + 647 (encrypt ? authsize : (-authsize))); 648 qi_cache_free(edesc); 649 return ERR_PTR(dst_nents); 650 } 651 652 if (src_nents) { 653 mapped_src_nents = dma_map_sg(qidev, req->src, 654 src_nents, DMA_TO_DEVICE); 655 if (unlikely(!mapped_src_nents)) { 656 dev_err(qidev, "unable to map source\n"); 657 qi_cache_free(edesc); 658 return ERR_PTR(-ENOMEM); 659 } 660 } else { 661 mapped_src_nents = 0; 662 } 663 664 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 665 DMA_FROM_DEVICE); 666 if (unlikely(!mapped_dst_nents)) { 667 dev_err(qidev, "unable to map destination\n"); 668 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 669 qi_cache_free(edesc); 670 return ERR_PTR(-ENOMEM); 671 } 672 } 673 674 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) { 675 ivsize = crypto_aead_ivsize(aead); 676 iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE); 677 if (dma_mapping_error(qidev, iv_dma)) { 678 dev_err(qidev, "unable to map IV\n"); 679 caam_unmap(qidev, req->src, req->dst, src_nents, 680 dst_nents, 0, 0, op_type, 0, 0); 681 qi_cache_free(edesc); 682 return ERR_PTR(-ENOMEM); 683 } 684 } 685 686 /* 687 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 688 * Input is not contiguous. 689 */ 690 qm_sg_ents = 1 + !!ivsize + mapped_src_nents + 691 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 692 if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) { 693 dev_err(qidev, "Insufficient S/G entries: %d > %zu\n", 694 qm_sg_ents, CAAM_QI_MAX_AEAD_SG); 695 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 696 iv_dma, ivsize, op_type, 0, 0); 697 qi_cache_free(edesc); 698 return ERR_PTR(-ENOMEM); 699 } 700 sg_table = &edesc->sgt[0]; 701 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 702 703 edesc->src_nents = src_nents; 704 edesc->dst_nents = dst_nents; 705 edesc->iv_dma = iv_dma; 706 edesc->drv_req.app_ctx = req; 707 edesc->drv_req.cbk = aead_done; 708 edesc->drv_req.drv_ctx = drv_ctx; 709 710 edesc->assoclen = cpu_to_caam32(req->assoclen); 711 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 712 DMA_TO_DEVICE); 713 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 714 dev_err(qidev, "unable to map assoclen\n"); 715 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 716 iv_dma, ivsize, op_type, 0, 0); 717 qi_cache_free(edesc); 718 return ERR_PTR(-ENOMEM); 719 } 720 721 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 722 qm_sg_index++; 723 if (ivsize) { 724 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 725 qm_sg_index++; 726 } 727 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); 728 qm_sg_index += mapped_src_nents; 729 730 if (mapped_dst_nents > 1) 731 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 732 qm_sg_index, 0); 733 734 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 735 if (dma_mapping_error(qidev, qm_sg_dma)) { 736 dev_err(qidev, "unable to map S/G table\n"); 737 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 738 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 739 iv_dma, ivsize, op_type, 0, 0); 740 qi_cache_free(edesc); 741 return ERR_PTR(-ENOMEM); 742 } 743 744 edesc->qm_sg_dma = qm_sg_dma; 745 edesc->qm_sg_bytes = qm_sg_bytes; 746 747 out_len = req->assoclen + req->cryptlen + 748 (encrypt ? ctx->authsize : (-ctx->authsize)); 749 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 750 751 fd_sgt = &edesc->drv_req.fd_sgt[0]; 752 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 753 754 if (req->dst == req->src) { 755 if (mapped_src_nents == 1) 756 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 757 out_len, 0); 758 else 759 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 760 (1 + !!ivsize) * sizeof(*sg_table), 761 out_len, 0); 762 } else if (mapped_dst_nents == 1) { 763 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 764 0); 765 } else { 766 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 767 qm_sg_index, out_len, 0); 768 } 769 770 return edesc; 771 } 772 773 static inline int aead_crypt(struct aead_request *req, bool encrypt) 774 { 775 struct aead_edesc *edesc; 776 struct crypto_aead *aead = crypto_aead_reqtfm(req); 777 struct caam_ctx *ctx = crypto_aead_ctx(aead); 778 int ret; 779 780 if (unlikely(caam_congested)) 781 return -EAGAIN; 782 783 /* allocate extended descriptor */ 784 edesc = aead_edesc_alloc(req, encrypt); 785 if (IS_ERR_OR_NULL(edesc)) 786 return PTR_ERR(edesc); 787 788 /* Create and submit job descriptor */ 789 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 790 if (!ret) { 791 ret = -EINPROGRESS; 792 } else { 793 aead_unmap(ctx->qidev, edesc, req); 794 qi_cache_free(edesc); 795 } 796 797 return ret; 798 } 799 800 static int aead_encrypt(struct aead_request *req) 801 { 802 return aead_crypt(req, true); 803 } 804 805 static int aead_decrypt(struct aead_request *req) 806 { 807 return aead_crypt(req, false); 808 } 809 810 static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) 811 { 812 struct ablkcipher_edesc *edesc; 813 struct ablkcipher_request *req = drv_req->app_ctx; 814 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 815 struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); 816 struct device *qidev = caam_ctx->qidev; 817 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 818 819 #ifdef DEBUG 820 dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 821 #endif 822 823 edesc = container_of(drv_req, typeof(*edesc), drv_req); 824 825 if (status) 826 caam_jr_strstatus(qidev, status); 827 828 #ifdef DEBUG 829 print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", 830 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 831 edesc->src_nents > 1 ? 100 : ivsize, 1); 832 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 833 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 834 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 835 #endif 836 837 ablkcipher_unmap(qidev, edesc, req); 838 qi_cache_free(edesc); 839 840 /* 841 * The crypto API expects us to set the IV (req->info) to the last 842 * ciphertext block. This is used e.g. by the CTS mode. 843 */ 844 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, 845 ivsize, 0); 846 847 ablkcipher_request_complete(req, status); 848 } 849 850 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 851 *req, bool encrypt) 852 { 853 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 854 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 855 struct device *qidev = ctx->qidev; 856 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 857 GFP_KERNEL : GFP_ATOMIC; 858 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 859 struct ablkcipher_edesc *edesc; 860 dma_addr_t iv_dma; 861 bool in_contig; 862 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 863 int dst_sg_idx, qm_sg_ents; 864 struct qm_sg_entry *sg_table, *fd_sgt; 865 struct caam_drv_ctx *drv_ctx; 866 enum optype op_type = encrypt ? ENCRYPT : DECRYPT; 867 868 drv_ctx = get_drv_ctx(ctx, op_type); 869 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 870 return (struct ablkcipher_edesc *)drv_ctx; 871 872 src_nents = sg_nents_for_len(req->src, req->nbytes); 873 if (unlikely(src_nents < 0)) { 874 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 875 req->nbytes); 876 return ERR_PTR(src_nents); 877 } 878 879 if (unlikely(req->src != req->dst)) { 880 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 881 if (unlikely(dst_nents < 0)) { 882 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 883 req->nbytes); 884 return ERR_PTR(dst_nents); 885 } 886 887 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 888 DMA_TO_DEVICE); 889 if (unlikely(!mapped_src_nents)) { 890 dev_err(qidev, "unable to map source\n"); 891 return ERR_PTR(-ENOMEM); 892 } 893 894 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 895 DMA_FROM_DEVICE); 896 if (unlikely(!mapped_dst_nents)) { 897 dev_err(qidev, "unable to map destination\n"); 898 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 899 return ERR_PTR(-ENOMEM); 900 } 901 } else { 902 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 903 DMA_BIDIRECTIONAL); 904 if (unlikely(!mapped_src_nents)) { 905 dev_err(qidev, "unable to map source\n"); 906 return ERR_PTR(-ENOMEM); 907 } 908 } 909 910 iv_dma = dma_map_single(qidev, req->info, ivsize, DMA_TO_DEVICE); 911 if (dma_mapping_error(qidev, iv_dma)) { 912 dev_err(qidev, "unable to map IV\n"); 913 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 914 0, 0, 0, 0); 915 return ERR_PTR(-ENOMEM); 916 } 917 918 if (mapped_src_nents == 1 && 919 iv_dma + ivsize == sg_dma_address(req->src)) { 920 in_contig = true; 921 qm_sg_ents = 0; 922 } else { 923 in_contig = false; 924 qm_sg_ents = 1 + mapped_src_nents; 925 } 926 dst_sg_idx = qm_sg_ents; 927 928 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 929 if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { 930 dev_err(qidev, "Insufficient S/G entries: %d > %zu\n", 931 qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); 932 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 933 iv_dma, ivsize, op_type, 0, 0); 934 return ERR_PTR(-ENOMEM); 935 } 936 937 /* allocate space for base edesc and link tables */ 938 edesc = qi_cache_alloc(GFP_DMA | flags); 939 if (unlikely(!edesc)) { 940 dev_err(qidev, "could not allocate extended descriptor\n"); 941 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 942 iv_dma, ivsize, op_type, 0, 0); 943 return ERR_PTR(-ENOMEM); 944 } 945 946 edesc->src_nents = src_nents; 947 edesc->dst_nents = dst_nents; 948 edesc->iv_dma = iv_dma; 949 sg_table = &edesc->sgt[0]; 950 edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 951 edesc->drv_req.app_ctx = req; 952 edesc->drv_req.cbk = ablkcipher_done; 953 edesc->drv_req.drv_ctx = drv_ctx; 954 955 if (!in_contig) { 956 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 957 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); 958 } 959 960 if (mapped_dst_nents > 1) 961 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 962 dst_sg_idx, 0); 963 964 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 965 DMA_TO_DEVICE); 966 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 967 dev_err(qidev, "unable to map S/G table\n"); 968 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 969 iv_dma, ivsize, op_type, 0, 0); 970 qi_cache_free(edesc); 971 return ERR_PTR(-ENOMEM); 972 } 973 974 fd_sgt = &edesc->drv_req.fd_sgt[0]; 975 976 if (!in_contig) 977 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 978 ivsize + req->nbytes, 0); 979 else 980 dma_to_qm_sg_one_last(&fd_sgt[1], iv_dma, ivsize + req->nbytes, 981 0); 982 983 if (req->src == req->dst) { 984 if (!in_contig) 985 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 986 sizeof(*sg_table), req->nbytes, 0); 987 else 988 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 989 req->nbytes, 0); 990 } else if (mapped_dst_nents > 1) { 991 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 992 sizeof(*sg_table), req->nbytes, 0); 993 } else { 994 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), 995 req->nbytes, 0); 996 } 997 998 return edesc; 999 } 1000 1001 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 1002 struct skcipher_givcrypt_request *creq) 1003 { 1004 struct ablkcipher_request *req = &creq->creq; 1005 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1006 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1007 struct device *qidev = ctx->qidev; 1008 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1009 GFP_KERNEL : GFP_ATOMIC; 1010 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; 1011 struct ablkcipher_edesc *edesc; 1012 dma_addr_t iv_dma; 1013 bool out_contig; 1014 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1015 struct qm_sg_entry *sg_table, *fd_sgt; 1016 int dst_sg_idx, qm_sg_ents; 1017 struct caam_drv_ctx *drv_ctx; 1018 1019 drv_ctx = get_drv_ctx(ctx, GIVENCRYPT); 1020 if (unlikely(IS_ERR_OR_NULL(drv_ctx))) 1021 return (struct ablkcipher_edesc *)drv_ctx; 1022 1023 src_nents = sg_nents_for_len(req->src, req->nbytes); 1024 if (unlikely(src_nents < 0)) { 1025 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1026 req->nbytes); 1027 return ERR_PTR(src_nents); 1028 } 1029 1030 if (unlikely(req->src != req->dst)) { 1031 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1032 if (unlikely(dst_nents < 0)) { 1033 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1034 req->nbytes); 1035 return ERR_PTR(dst_nents); 1036 } 1037 1038 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1039 DMA_TO_DEVICE); 1040 if (unlikely(!mapped_src_nents)) { 1041 dev_err(qidev, "unable to map source\n"); 1042 return ERR_PTR(-ENOMEM); 1043 } 1044 1045 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1046 DMA_FROM_DEVICE); 1047 if (unlikely(!mapped_dst_nents)) { 1048 dev_err(qidev, "unable to map destination\n"); 1049 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1050 return ERR_PTR(-ENOMEM); 1051 } 1052 } else { 1053 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1054 DMA_BIDIRECTIONAL); 1055 if (unlikely(!mapped_src_nents)) { 1056 dev_err(qidev, "unable to map source\n"); 1057 return ERR_PTR(-ENOMEM); 1058 } 1059 1060 dst_nents = src_nents; 1061 mapped_dst_nents = src_nents; 1062 } 1063 1064 iv_dma = dma_map_single(qidev, creq->giv, ivsize, DMA_FROM_DEVICE); 1065 if (dma_mapping_error(qidev, iv_dma)) { 1066 dev_err(qidev, "unable to map IV\n"); 1067 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1068 0, 0, 0, 0); 1069 return ERR_PTR(-ENOMEM); 1070 } 1071 1072 qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; 1073 dst_sg_idx = qm_sg_ents; 1074 if (mapped_dst_nents == 1 && 1075 iv_dma + ivsize == sg_dma_address(req->dst)) { 1076 out_contig = true; 1077 } else { 1078 out_contig = false; 1079 qm_sg_ents += 1 + mapped_dst_nents; 1080 } 1081 1082 if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { 1083 dev_err(qidev, "Insufficient S/G entries: %d > %zu\n", 1084 qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); 1085 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1086 iv_dma, ivsize, GIVENCRYPT, 0, 0); 1087 return ERR_PTR(-ENOMEM); 1088 } 1089 1090 /* allocate space for base edesc and link tables */ 1091 edesc = qi_cache_alloc(GFP_DMA | flags); 1092 if (!edesc) { 1093 dev_err(qidev, "could not allocate extended descriptor\n"); 1094 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1095 iv_dma, ivsize, GIVENCRYPT, 0, 0); 1096 return ERR_PTR(-ENOMEM); 1097 } 1098 1099 edesc->src_nents = src_nents; 1100 edesc->dst_nents = dst_nents; 1101 edesc->iv_dma = iv_dma; 1102 sg_table = &edesc->sgt[0]; 1103 edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 1104 edesc->drv_req.app_ctx = req; 1105 edesc->drv_req.cbk = ablkcipher_done; 1106 edesc->drv_req.drv_ctx = drv_ctx; 1107 1108 if (mapped_src_nents > 1) 1109 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); 1110 1111 if (!out_contig) { 1112 dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); 1113 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1114 dst_sg_idx + 1, 0); 1115 } 1116 1117 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1118 DMA_TO_DEVICE); 1119 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1120 dev_err(qidev, "unable to map S/G table\n"); 1121 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1122 iv_dma, ivsize, GIVENCRYPT, 0, 0); 1123 qi_cache_free(edesc); 1124 return ERR_PTR(-ENOMEM); 1125 } 1126 1127 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1128 1129 if (mapped_src_nents > 1) 1130 dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes, 1131 0); 1132 else 1133 dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src), 1134 req->nbytes, 0); 1135 1136 if (!out_contig) 1137 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1138 sizeof(*sg_table), ivsize + req->nbytes, 1139 0); 1140 else 1141 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), 1142 ivsize + req->nbytes, 0); 1143 1144 return edesc; 1145 } 1146 1147 static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt) 1148 { 1149 struct ablkcipher_edesc *edesc; 1150 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1151 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1152 int ret; 1153 1154 if (unlikely(caam_congested)) 1155 return -EAGAIN; 1156 1157 /* allocate extended descriptor */ 1158 edesc = ablkcipher_edesc_alloc(req, encrypt); 1159 if (IS_ERR(edesc)) 1160 return PTR_ERR(edesc); 1161 1162 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1163 if (!ret) { 1164 ret = -EINPROGRESS; 1165 } else { 1166 ablkcipher_unmap(ctx->qidev, edesc, req); 1167 qi_cache_free(edesc); 1168 } 1169 1170 return ret; 1171 } 1172 1173 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1174 { 1175 return ablkcipher_crypt(req, true); 1176 } 1177 1178 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1179 { 1180 return ablkcipher_crypt(req, false); 1181 } 1182 1183 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 1184 { 1185 struct ablkcipher_request *req = &creq->creq; 1186 struct ablkcipher_edesc *edesc; 1187 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1188 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1189 int ret; 1190 1191 if (unlikely(caam_congested)) 1192 return -EAGAIN; 1193 1194 /* allocate extended descriptor */ 1195 edesc = ablkcipher_giv_edesc_alloc(creq); 1196 if (IS_ERR(edesc)) 1197 return PTR_ERR(edesc); 1198 1199 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1200 if (!ret) { 1201 ret = -EINPROGRESS; 1202 } else { 1203 ablkcipher_unmap(ctx->qidev, edesc, req); 1204 qi_cache_free(edesc); 1205 } 1206 1207 return ret; 1208 } 1209 1210 #define template_ablkcipher template_u.ablkcipher 1211 struct caam_alg_template { 1212 char name[CRYPTO_MAX_ALG_NAME]; 1213 char driver_name[CRYPTO_MAX_ALG_NAME]; 1214 unsigned int blocksize; 1215 u32 type; 1216 union { 1217 struct ablkcipher_alg ablkcipher; 1218 } template_u; 1219 u32 class1_alg_type; 1220 u32 class2_alg_type; 1221 }; 1222 1223 static struct caam_alg_template driver_algs[] = { 1224 /* ablkcipher descriptor */ 1225 { 1226 .name = "cbc(aes)", 1227 .driver_name = "cbc-aes-caam-qi", 1228 .blocksize = AES_BLOCK_SIZE, 1229 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1230 .template_ablkcipher = { 1231 .setkey = ablkcipher_setkey, 1232 .encrypt = ablkcipher_encrypt, 1233 .decrypt = ablkcipher_decrypt, 1234 .givencrypt = ablkcipher_givencrypt, 1235 .geniv = "<built-in>", 1236 .min_keysize = AES_MIN_KEY_SIZE, 1237 .max_keysize = AES_MAX_KEY_SIZE, 1238 .ivsize = AES_BLOCK_SIZE, 1239 }, 1240 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1241 }, 1242 { 1243 .name = "cbc(des3_ede)", 1244 .driver_name = "cbc-3des-caam-qi", 1245 .blocksize = DES3_EDE_BLOCK_SIZE, 1246 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1247 .template_ablkcipher = { 1248 .setkey = ablkcipher_setkey, 1249 .encrypt = ablkcipher_encrypt, 1250 .decrypt = ablkcipher_decrypt, 1251 .givencrypt = ablkcipher_givencrypt, 1252 .geniv = "<built-in>", 1253 .min_keysize = DES3_EDE_KEY_SIZE, 1254 .max_keysize = DES3_EDE_KEY_SIZE, 1255 .ivsize = DES3_EDE_BLOCK_SIZE, 1256 }, 1257 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1258 }, 1259 { 1260 .name = "cbc(des)", 1261 .driver_name = "cbc-des-caam-qi", 1262 .blocksize = DES_BLOCK_SIZE, 1263 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1264 .template_ablkcipher = { 1265 .setkey = ablkcipher_setkey, 1266 .encrypt = ablkcipher_encrypt, 1267 .decrypt = ablkcipher_decrypt, 1268 .givencrypt = ablkcipher_givencrypt, 1269 .geniv = "<built-in>", 1270 .min_keysize = DES_KEY_SIZE, 1271 .max_keysize = DES_KEY_SIZE, 1272 .ivsize = DES_BLOCK_SIZE, 1273 }, 1274 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1275 }, 1276 { 1277 .name = "ctr(aes)", 1278 .driver_name = "ctr-aes-caam-qi", 1279 .blocksize = 1, 1280 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1281 .template_ablkcipher = { 1282 .setkey = ablkcipher_setkey, 1283 .encrypt = ablkcipher_encrypt, 1284 .decrypt = ablkcipher_decrypt, 1285 .geniv = "chainiv", 1286 .min_keysize = AES_MIN_KEY_SIZE, 1287 .max_keysize = AES_MAX_KEY_SIZE, 1288 .ivsize = AES_BLOCK_SIZE, 1289 }, 1290 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1291 }, 1292 { 1293 .name = "rfc3686(ctr(aes))", 1294 .driver_name = "rfc3686-ctr-aes-caam-qi", 1295 .blocksize = 1, 1296 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1297 .template_ablkcipher = { 1298 .setkey = ablkcipher_setkey, 1299 .encrypt = ablkcipher_encrypt, 1300 .decrypt = ablkcipher_decrypt, 1301 .givencrypt = ablkcipher_givencrypt, 1302 .geniv = "<built-in>", 1303 .min_keysize = AES_MIN_KEY_SIZE + 1304 CTR_RFC3686_NONCE_SIZE, 1305 .max_keysize = AES_MAX_KEY_SIZE + 1306 CTR_RFC3686_NONCE_SIZE, 1307 .ivsize = CTR_RFC3686_IV_SIZE, 1308 }, 1309 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1310 }, 1311 { 1312 .name = "xts(aes)", 1313 .driver_name = "xts-aes-caam-qi", 1314 .blocksize = AES_BLOCK_SIZE, 1315 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1316 .template_ablkcipher = { 1317 .setkey = xts_ablkcipher_setkey, 1318 .encrypt = ablkcipher_encrypt, 1319 .decrypt = ablkcipher_decrypt, 1320 .geniv = "eseqiv", 1321 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1322 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1323 .ivsize = AES_BLOCK_SIZE, 1324 }, 1325 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1326 }, 1327 }; 1328 1329 static struct caam_aead_alg driver_aeads[] = { 1330 /* single-pass ipsec_esp descriptor */ 1331 { 1332 .aead = { 1333 .base = { 1334 .cra_name = "authenc(hmac(md5),cbc(aes))", 1335 .cra_driver_name = "authenc-hmac-md5-" 1336 "cbc-aes-caam-qi", 1337 .cra_blocksize = AES_BLOCK_SIZE, 1338 }, 1339 .setkey = aead_setkey, 1340 .setauthsize = aead_setauthsize, 1341 .encrypt = aead_encrypt, 1342 .decrypt = aead_decrypt, 1343 .ivsize = AES_BLOCK_SIZE, 1344 .maxauthsize = MD5_DIGEST_SIZE, 1345 }, 1346 .caam = { 1347 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1348 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1349 OP_ALG_AAI_HMAC_PRECOMP, 1350 } 1351 }, 1352 { 1353 .aead = { 1354 .base = { 1355 .cra_name = "echainiv(authenc(hmac(md5)," 1356 "cbc(aes)))", 1357 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1358 "cbc-aes-caam-qi", 1359 .cra_blocksize = AES_BLOCK_SIZE, 1360 }, 1361 .setkey = aead_setkey, 1362 .setauthsize = aead_setauthsize, 1363 .encrypt = aead_encrypt, 1364 .decrypt = aead_decrypt, 1365 .ivsize = AES_BLOCK_SIZE, 1366 .maxauthsize = MD5_DIGEST_SIZE, 1367 }, 1368 .caam = { 1369 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1370 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1371 OP_ALG_AAI_HMAC_PRECOMP, 1372 .geniv = true, 1373 } 1374 }, 1375 { 1376 .aead = { 1377 .base = { 1378 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1379 .cra_driver_name = "authenc-hmac-sha1-" 1380 "cbc-aes-caam-qi", 1381 .cra_blocksize = AES_BLOCK_SIZE, 1382 }, 1383 .setkey = aead_setkey, 1384 .setauthsize = aead_setauthsize, 1385 .encrypt = aead_encrypt, 1386 .decrypt = aead_decrypt, 1387 .ivsize = AES_BLOCK_SIZE, 1388 .maxauthsize = SHA1_DIGEST_SIZE, 1389 }, 1390 .caam = { 1391 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1392 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1393 OP_ALG_AAI_HMAC_PRECOMP, 1394 } 1395 }, 1396 { 1397 .aead = { 1398 .base = { 1399 .cra_name = "echainiv(authenc(hmac(sha1)," 1400 "cbc(aes)))", 1401 .cra_driver_name = "echainiv-authenc-" 1402 "hmac-sha1-cbc-aes-caam-qi", 1403 .cra_blocksize = AES_BLOCK_SIZE, 1404 }, 1405 .setkey = aead_setkey, 1406 .setauthsize = aead_setauthsize, 1407 .encrypt = aead_encrypt, 1408 .decrypt = aead_decrypt, 1409 .ivsize = AES_BLOCK_SIZE, 1410 .maxauthsize = SHA1_DIGEST_SIZE, 1411 }, 1412 .caam = { 1413 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1414 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1415 OP_ALG_AAI_HMAC_PRECOMP, 1416 .geniv = true, 1417 }, 1418 }, 1419 { 1420 .aead = { 1421 .base = { 1422 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1423 .cra_driver_name = "authenc-hmac-sha224-" 1424 "cbc-aes-caam-qi", 1425 .cra_blocksize = AES_BLOCK_SIZE, 1426 }, 1427 .setkey = aead_setkey, 1428 .setauthsize = aead_setauthsize, 1429 .encrypt = aead_encrypt, 1430 .decrypt = aead_decrypt, 1431 .ivsize = AES_BLOCK_SIZE, 1432 .maxauthsize = SHA224_DIGEST_SIZE, 1433 }, 1434 .caam = { 1435 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1436 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1437 OP_ALG_AAI_HMAC_PRECOMP, 1438 } 1439 }, 1440 { 1441 .aead = { 1442 .base = { 1443 .cra_name = "echainiv(authenc(hmac(sha224)," 1444 "cbc(aes)))", 1445 .cra_driver_name = "echainiv-authenc-" 1446 "hmac-sha224-cbc-aes-caam-qi", 1447 .cra_blocksize = AES_BLOCK_SIZE, 1448 }, 1449 .setkey = aead_setkey, 1450 .setauthsize = aead_setauthsize, 1451 .encrypt = aead_encrypt, 1452 .decrypt = aead_decrypt, 1453 .ivsize = AES_BLOCK_SIZE, 1454 .maxauthsize = SHA224_DIGEST_SIZE, 1455 }, 1456 .caam = { 1457 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1458 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1459 OP_ALG_AAI_HMAC_PRECOMP, 1460 .geniv = true, 1461 } 1462 }, 1463 { 1464 .aead = { 1465 .base = { 1466 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1467 .cra_driver_name = "authenc-hmac-sha256-" 1468 "cbc-aes-caam-qi", 1469 .cra_blocksize = AES_BLOCK_SIZE, 1470 }, 1471 .setkey = aead_setkey, 1472 .setauthsize = aead_setauthsize, 1473 .encrypt = aead_encrypt, 1474 .decrypt = aead_decrypt, 1475 .ivsize = AES_BLOCK_SIZE, 1476 .maxauthsize = SHA256_DIGEST_SIZE, 1477 }, 1478 .caam = { 1479 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1480 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1481 OP_ALG_AAI_HMAC_PRECOMP, 1482 } 1483 }, 1484 { 1485 .aead = { 1486 .base = { 1487 .cra_name = "echainiv(authenc(hmac(sha256)," 1488 "cbc(aes)))", 1489 .cra_driver_name = "echainiv-authenc-" 1490 "hmac-sha256-cbc-aes-" 1491 "caam-qi", 1492 .cra_blocksize = AES_BLOCK_SIZE, 1493 }, 1494 .setkey = aead_setkey, 1495 .setauthsize = aead_setauthsize, 1496 .encrypt = aead_encrypt, 1497 .decrypt = aead_decrypt, 1498 .ivsize = AES_BLOCK_SIZE, 1499 .maxauthsize = SHA256_DIGEST_SIZE, 1500 }, 1501 .caam = { 1502 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1503 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1504 OP_ALG_AAI_HMAC_PRECOMP, 1505 .geniv = true, 1506 } 1507 }, 1508 { 1509 .aead = { 1510 .base = { 1511 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1512 .cra_driver_name = "authenc-hmac-sha384-" 1513 "cbc-aes-caam-qi", 1514 .cra_blocksize = AES_BLOCK_SIZE, 1515 }, 1516 .setkey = aead_setkey, 1517 .setauthsize = aead_setauthsize, 1518 .encrypt = aead_encrypt, 1519 .decrypt = aead_decrypt, 1520 .ivsize = AES_BLOCK_SIZE, 1521 .maxauthsize = SHA384_DIGEST_SIZE, 1522 }, 1523 .caam = { 1524 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1525 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1526 OP_ALG_AAI_HMAC_PRECOMP, 1527 } 1528 }, 1529 { 1530 .aead = { 1531 .base = { 1532 .cra_name = "echainiv(authenc(hmac(sha384)," 1533 "cbc(aes)))", 1534 .cra_driver_name = "echainiv-authenc-" 1535 "hmac-sha384-cbc-aes-" 1536 "caam-qi", 1537 .cra_blocksize = AES_BLOCK_SIZE, 1538 }, 1539 .setkey = aead_setkey, 1540 .setauthsize = aead_setauthsize, 1541 .encrypt = aead_encrypt, 1542 .decrypt = aead_decrypt, 1543 .ivsize = AES_BLOCK_SIZE, 1544 .maxauthsize = SHA384_DIGEST_SIZE, 1545 }, 1546 .caam = { 1547 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1548 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1549 OP_ALG_AAI_HMAC_PRECOMP, 1550 .geniv = true, 1551 } 1552 }, 1553 { 1554 .aead = { 1555 .base = { 1556 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1557 .cra_driver_name = "authenc-hmac-sha512-" 1558 "cbc-aes-caam-qi", 1559 .cra_blocksize = AES_BLOCK_SIZE, 1560 }, 1561 .setkey = aead_setkey, 1562 .setauthsize = aead_setauthsize, 1563 .encrypt = aead_encrypt, 1564 .decrypt = aead_decrypt, 1565 .ivsize = AES_BLOCK_SIZE, 1566 .maxauthsize = SHA512_DIGEST_SIZE, 1567 }, 1568 .caam = { 1569 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1570 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1571 OP_ALG_AAI_HMAC_PRECOMP, 1572 } 1573 }, 1574 { 1575 .aead = { 1576 .base = { 1577 .cra_name = "echainiv(authenc(hmac(sha512)," 1578 "cbc(aes)))", 1579 .cra_driver_name = "echainiv-authenc-" 1580 "hmac-sha512-cbc-aes-" 1581 "caam-qi", 1582 .cra_blocksize = AES_BLOCK_SIZE, 1583 }, 1584 .setkey = aead_setkey, 1585 .setauthsize = aead_setauthsize, 1586 .encrypt = aead_encrypt, 1587 .decrypt = aead_decrypt, 1588 .ivsize = AES_BLOCK_SIZE, 1589 .maxauthsize = SHA512_DIGEST_SIZE, 1590 }, 1591 .caam = { 1592 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1593 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1594 OP_ALG_AAI_HMAC_PRECOMP, 1595 .geniv = true, 1596 } 1597 }, 1598 { 1599 .aead = { 1600 .base = { 1601 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1602 .cra_driver_name = "authenc-hmac-md5-" 1603 "cbc-des3_ede-caam-qi", 1604 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1605 }, 1606 .setkey = aead_setkey, 1607 .setauthsize = aead_setauthsize, 1608 .encrypt = aead_encrypt, 1609 .decrypt = aead_decrypt, 1610 .ivsize = DES3_EDE_BLOCK_SIZE, 1611 .maxauthsize = MD5_DIGEST_SIZE, 1612 }, 1613 .caam = { 1614 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1615 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1616 OP_ALG_AAI_HMAC_PRECOMP, 1617 } 1618 }, 1619 { 1620 .aead = { 1621 .base = { 1622 .cra_name = "echainiv(authenc(hmac(md5)," 1623 "cbc(des3_ede)))", 1624 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1625 "cbc-des3_ede-caam-qi", 1626 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1627 }, 1628 .setkey = aead_setkey, 1629 .setauthsize = aead_setauthsize, 1630 .encrypt = aead_encrypt, 1631 .decrypt = aead_decrypt, 1632 .ivsize = DES3_EDE_BLOCK_SIZE, 1633 .maxauthsize = MD5_DIGEST_SIZE, 1634 }, 1635 .caam = { 1636 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1637 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1638 OP_ALG_AAI_HMAC_PRECOMP, 1639 .geniv = true, 1640 } 1641 }, 1642 { 1643 .aead = { 1644 .base = { 1645 .cra_name = "authenc(hmac(sha1)," 1646 "cbc(des3_ede))", 1647 .cra_driver_name = "authenc-hmac-sha1-" 1648 "cbc-des3_ede-caam-qi", 1649 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1650 }, 1651 .setkey = aead_setkey, 1652 .setauthsize = aead_setauthsize, 1653 .encrypt = aead_encrypt, 1654 .decrypt = aead_decrypt, 1655 .ivsize = DES3_EDE_BLOCK_SIZE, 1656 .maxauthsize = SHA1_DIGEST_SIZE, 1657 }, 1658 .caam = { 1659 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1660 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1661 OP_ALG_AAI_HMAC_PRECOMP, 1662 }, 1663 }, 1664 { 1665 .aead = { 1666 .base = { 1667 .cra_name = "echainiv(authenc(hmac(sha1)," 1668 "cbc(des3_ede)))", 1669 .cra_driver_name = "echainiv-authenc-" 1670 "hmac-sha1-" 1671 "cbc-des3_ede-caam-qi", 1672 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1673 }, 1674 .setkey = aead_setkey, 1675 .setauthsize = aead_setauthsize, 1676 .encrypt = aead_encrypt, 1677 .decrypt = aead_decrypt, 1678 .ivsize = DES3_EDE_BLOCK_SIZE, 1679 .maxauthsize = SHA1_DIGEST_SIZE, 1680 }, 1681 .caam = { 1682 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1683 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1684 OP_ALG_AAI_HMAC_PRECOMP, 1685 .geniv = true, 1686 } 1687 }, 1688 { 1689 .aead = { 1690 .base = { 1691 .cra_name = "authenc(hmac(sha224)," 1692 "cbc(des3_ede))", 1693 .cra_driver_name = "authenc-hmac-sha224-" 1694 "cbc-des3_ede-caam-qi", 1695 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1696 }, 1697 .setkey = aead_setkey, 1698 .setauthsize = aead_setauthsize, 1699 .encrypt = aead_encrypt, 1700 .decrypt = aead_decrypt, 1701 .ivsize = DES3_EDE_BLOCK_SIZE, 1702 .maxauthsize = SHA224_DIGEST_SIZE, 1703 }, 1704 .caam = { 1705 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1706 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1707 OP_ALG_AAI_HMAC_PRECOMP, 1708 }, 1709 }, 1710 { 1711 .aead = { 1712 .base = { 1713 .cra_name = "echainiv(authenc(hmac(sha224)," 1714 "cbc(des3_ede)))", 1715 .cra_driver_name = "echainiv-authenc-" 1716 "hmac-sha224-" 1717 "cbc-des3_ede-caam-qi", 1718 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1719 }, 1720 .setkey = aead_setkey, 1721 .setauthsize = aead_setauthsize, 1722 .encrypt = aead_encrypt, 1723 .decrypt = aead_decrypt, 1724 .ivsize = DES3_EDE_BLOCK_SIZE, 1725 .maxauthsize = SHA224_DIGEST_SIZE, 1726 }, 1727 .caam = { 1728 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1729 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1730 OP_ALG_AAI_HMAC_PRECOMP, 1731 .geniv = true, 1732 } 1733 }, 1734 { 1735 .aead = { 1736 .base = { 1737 .cra_name = "authenc(hmac(sha256)," 1738 "cbc(des3_ede))", 1739 .cra_driver_name = "authenc-hmac-sha256-" 1740 "cbc-des3_ede-caam-qi", 1741 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1742 }, 1743 .setkey = aead_setkey, 1744 .setauthsize = aead_setauthsize, 1745 .encrypt = aead_encrypt, 1746 .decrypt = aead_decrypt, 1747 .ivsize = DES3_EDE_BLOCK_SIZE, 1748 .maxauthsize = SHA256_DIGEST_SIZE, 1749 }, 1750 .caam = { 1751 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1752 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1753 OP_ALG_AAI_HMAC_PRECOMP, 1754 }, 1755 }, 1756 { 1757 .aead = { 1758 .base = { 1759 .cra_name = "echainiv(authenc(hmac(sha256)," 1760 "cbc(des3_ede)))", 1761 .cra_driver_name = "echainiv-authenc-" 1762 "hmac-sha256-" 1763 "cbc-des3_ede-caam-qi", 1764 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1765 }, 1766 .setkey = aead_setkey, 1767 .setauthsize = aead_setauthsize, 1768 .encrypt = aead_encrypt, 1769 .decrypt = aead_decrypt, 1770 .ivsize = DES3_EDE_BLOCK_SIZE, 1771 .maxauthsize = SHA256_DIGEST_SIZE, 1772 }, 1773 .caam = { 1774 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1775 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1776 OP_ALG_AAI_HMAC_PRECOMP, 1777 .geniv = true, 1778 } 1779 }, 1780 { 1781 .aead = { 1782 .base = { 1783 .cra_name = "authenc(hmac(sha384)," 1784 "cbc(des3_ede))", 1785 .cra_driver_name = "authenc-hmac-sha384-" 1786 "cbc-des3_ede-caam-qi", 1787 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1788 }, 1789 .setkey = aead_setkey, 1790 .setauthsize = aead_setauthsize, 1791 .encrypt = aead_encrypt, 1792 .decrypt = aead_decrypt, 1793 .ivsize = DES3_EDE_BLOCK_SIZE, 1794 .maxauthsize = SHA384_DIGEST_SIZE, 1795 }, 1796 .caam = { 1797 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1798 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1799 OP_ALG_AAI_HMAC_PRECOMP, 1800 }, 1801 }, 1802 { 1803 .aead = { 1804 .base = { 1805 .cra_name = "echainiv(authenc(hmac(sha384)," 1806 "cbc(des3_ede)))", 1807 .cra_driver_name = "echainiv-authenc-" 1808 "hmac-sha384-" 1809 "cbc-des3_ede-caam-qi", 1810 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1811 }, 1812 .setkey = aead_setkey, 1813 .setauthsize = aead_setauthsize, 1814 .encrypt = aead_encrypt, 1815 .decrypt = aead_decrypt, 1816 .ivsize = DES3_EDE_BLOCK_SIZE, 1817 .maxauthsize = SHA384_DIGEST_SIZE, 1818 }, 1819 .caam = { 1820 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1821 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1822 OP_ALG_AAI_HMAC_PRECOMP, 1823 .geniv = true, 1824 } 1825 }, 1826 { 1827 .aead = { 1828 .base = { 1829 .cra_name = "authenc(hmac(sha512)," 1830 "cbc(des3_ede))", 1831 .cra_driver_name = "authenc-hmac-sha512-" 1832 "cbc-des3_ede-caam-qi", 1833 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1834 }, 1835 .setkey = aead_setkey, 1836 .setauthsize = aead_setauthsize, 1837 .encrypt = aead_encrypt, 1838 .decrypt = aead_decrypt, 1839 .ivsize = DES3_EDE_BLOCK_SIZE, 1840 .maxauthsize = SHA512_DIGEST_SIZE, 1841 }, 1842 .caam = { 1843 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1844 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1845 OP_ALG_AAI_HMAC_PRECOMP, 1846 }, 1847 }, 1848 { 1849 .aead = { 1850 .base = { 1851 .cra_name = "echainiv(authenc(hmac(sha512)," 1852 "cbc(des3_ede)))", 1853 .cra_driver_name = "echainiv-authenc-" 1854 "hmac-sha512-" 1855 "cbc-des3_ede-caam-qi", 1856 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1857 }, 1858 .setkey = aead_setkey, 1859 .setauthsize = aead_setauthsize, 1860 .encrypt = aead_encrypt, 1861 .decrypt = aead_decrypt, 1862 .ivsize = DES3_EDE_BLOCK_SIZE, 1863 .maxauthsize = SHA512_DIGEST_SIZE, 1864 }, 1865 .caam = { 1866 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1867 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1868 OP_ALG_AAI_HMAC_PRECOMP, 1869 .geniv = true, 1870 } 1871 }, 1872 { 1873 .aead = { 1874 .base = { 1875 .cra_name = "authenc(hmac(md5),cbc(des))", 1876 .cra_driver_name = "authenc-hmac-md5-" 1877 "cbc-des-caam-qi", 1878 .cra_blocksize = DES_BLOCK_SIZE, 1879 }, 1880 .setkey = aead_setkey, 1881 .setauthsize = aead_setauthsize, 1882 .encrypt = aead_encrypt, 1883 .decrypt = aead_decrypt, 1884 .ivsize = DES_BLOCK_SIZE, 1885 .maxauthsize = MD5_DIGEST_SIZE, 1886 }, 1887 .caam = { 1888 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1889 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1890 OP_ALG_AAI_HMAC_PRECOMP, 1891 }, 1892 }, 1893 { 1894 .aead = { 1895 .base = { 1896 .cra_name = "echainiv(authenc(hmac(md5)," 1897 "cbc(des)))", 1898 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1899 "cbc-des-caam-qi", 1900 .cra_blocksize = DES_BLOCK_SIZE, 1901 }, 1902 .setkey = aead_setkey, 1903 .setauthsize = aead_setauthsize, 1904 .encrypt = aead_encrypt, 1905 .decrypt = aead_decrypt, 1906 .ivsize = DES_BLOCK_SIZE, 1907 .maxauthsize = MD5_DIGEST_SIZE, 1908 }, 1909 .caam = { 1910 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1911 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1912 OP_ALG_AAI_HMAC_PRECOMP, 1913 .geniv = true, 1914 } 1915 }, 1916 { 1917 .aead = { 1918 .base = { 1919 .cra_name = "authenc(hmac(sha1),cbc(des))", 1920 .cra_driver_name = "authenc-hmac-sha1-" 1921 "cbc-des-caam-qi", 1922 .cra_blocksize = DES_BLOCK_SIZE, 1923 }, 1924 .setkey = aead_setkey, 1925 .setauthsize = aead_setauthsize, 1926 .encrypt = aead_encrypt, 1927 .decrypt = aead_decrypt, 1928 .ivsize = DES_BLOCK_SIZE, 1929 .maxauthsize = SHA1_DIGEST_SIZE, 1930 }, 1931 .caam = { 1932 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1933 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1934 OP_ALG_AAI_HMAC_PRECOMP, 1935 }, 1936 }, 1937 { 1938 .aead = { 1939 .base = { 1940 .cra_name = "echainiv(authenc(hmac(sha1)," 1941 "cbc(des)))", 1942 .cra_driver_name = "echainiv-authenc-" 1943 "hmac-sha1-cbc-des-caam-qi", 1944 .cra_blocksize = DES_BLOCK_SIZE, 1945 }, 1946 .setkey = aead_setkey, 1947 .setauthsize = aead_setauthsize, 1948 .encrypt = aead_encrypt, 1949 .decrypt = aead_decrypt, 1950 .ivsize = DES_BLOCK_SIZE, 1951 .maxauthsize = SHA1_DIGEST_SIZE, 1952 }, 1953 .caam = { 1954 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1955 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1956 OP_ALG_AAI_HMAC_PRECOMP, 1957 .geniv = true, 1958 } 1959 }, 1960 { 1961 .aead = { 1962 .base = { 1963 .cra_name = "authenc(hmac(sha224),cbc(des))", 1964 .cra_driver_name = "authenc-hmac-sha224-" 1965 "cbc-des-caam-qi", 1966 .cra_blocksize = DES_BLOCK_SIZE, 1967 }, 1968 .setkey = aead_setkey, 1969 .setauthsize = aead_setauthsize, 1970 .encrypt = aead_encrypt, 1971 .decrypt = aead_decrypt, 1972 .ivsize = DES_BLOCK_SIZE, 1973 .maxauthsize = SHA224_DIGEST_SIZE, 1974 }, 1975 .caam = { 1976 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1977 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1978 OP_ALG_AAI_HMAC_PRECOMP, 1979 }, 1980 }, 1981 { 1982 .aead = { 1983 .base = { 1984 .cra_name = "echainiv(authenc(hmac(sha224)," 1985 "cbc(des)))", 1986 .cra_driver_name = "echainiv-authenc-" 1987 "hmac-sha224-cbc-des-" 1988 "caam-qi", 1989 .cra_blocksize = DES_BLOCK_SIZE, 1990 }, 1991 .setkey = aead_setkey, 1992 .setauthsize = aead_setauthsize, 1993 .encrypt = aead_encrypt, 1994 .decrypt = aead_decrypt, 1995 .ivsize = DES_BLOCK_SIZE, 1996 .maxauthsize = SHA224_DIGEST_SIZE, 1997 }, 1998 .caam = { 1999 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2000 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2001 OP_ALG_AAI_HMAC_PRECOMP, 2002 .geniv = true, 2003 } 2004 }, 2005 { 2006 .aead = { 2007 .base = { 2008 .cra_name = "authenc(hmac(sha256),cbc(des))", 2009 .cra_driver_name = "authenc-hmac-sha256-" 2010 "cbc-des-caam-qi", 2011 .cra_blocksize = DES_BLOCK_SIZE, 2012 }, 2013 .setkey = aead_setkey, 2014 .setauthsize = aead_setauthsize, 2015 .encrypt = aead_encrypt, 2016 .decrypt = aead_decrypt, 2017 .ivsize = DES_BLOCK_SIZE, 2018 .maxauthsize = SHA256_DIGEST_SIZE, 2019 }, 2020 .caam = { 2021 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2022 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2023 OP_ALG_AAI_HMAC_PRECOMP, 2024 }, 2025 }, 2026 { 2027 .aead = { 2028 .base = { 2029 .cra_name = "echainiv(authenc(hmac(sha256)," 2030 "cbc(des)))", 2031 .cra_driver_name = "echainiv-authenc-" 2032 "hmac-sha256-cbc-des-" 2033 "caam-qi", 2034 .cra_blocksize = DES_BLOCK_SIZE, 2035 }, 2036 .setkey = aead_setkey, 2037 .setauthsize = aead_setauthsize, 2038 .encrypt = aead_encrypt, 2039 .decrypt = aead_decrypt, 2040 .ivsize = DES_BLOCK_SIZE, 2041 .maxauthsize = SHA256_DIGEST_SIZE, 2042 }, 2043 .caam = { 2044 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2045 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2046 OP_ALG_AAI_HMAC_PRECOMP, 2047 .geniv = true, 2048 }, 2049 }, 2050 { 2051 .aead = { 2052 .base = { 2053 .cra_name = "authenc(hmac(sha384),cbc(des))", 2054 .cra_driver_name = "authenc-hmac-sha384-" 2055 "cbc-des-caam-qi", 2056 .cra_blocksize = DES_BLOCK_SIZE, 2057 }, 2058 .setkey = aead_setkey, 2059 .setauthsize = aead_setauthsize, 2060 .encrypt = aead_encrypt, 2061 .decrypt = aead_decrypt, 2062 .ivsize = DES_BLOCK_SIZE, 2063 .maxauthsize = SHA384_DIGEST_SIZE, 2064 }, 2065 .caam = { 2066 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2067 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2068 OP_ALG_AAI_HMAC_PRECOMP, 2069 }, 2070 }, 2071 { 2072 .aead = { 2073 .base = { 2074 .cra_name = "echainiv(authenc(hmac(sha384)," 2075 "cbc(des)))", 2076 .cra_driver_name = "echainiv-authenc-" 2077 "hmac-sha384-cbc-des-" 2078 "caam-qi", 2079 .cra_blocksize = DES_BLOCK_SIZE, 2080 }, 2081 .setkey = aead_setkey, 2082 .setauthsize = aead_setauthsize, 2083 .encrypt = aead_encrypt, 2084 .decrypt = aead_decrypt, 2085 .ivsize = DES_BLOCK_SIZE, 2086 .maxauthsize = SHA384_DIGEST_SIZE, 2087 }, 2088 .caam = { 2089 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2090 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2091 OP_ALG_AAI_HMAC_PRECOMP, 2092 .geniv = true, 2093 } 2094 }, 2095 { 2096 .aead = { 2097 .base = { 2098 .cra_name = "authenc(hmac(sha512),cbc(des))", 2099 .cra_driver_name = "authenc-hmac-sha512-" 2100 "cbc-des-caam-qi", 2101 .cra_blocksize = DES_BLOCK_SIZE, 2102 }, 2103 .setkey = aead_setkey, 2104 .setauthsize = aead_setauthsize, 2105 .encrypt = aead_encrypt, 2106 .decrypt = aead_decrypt, 2107 .ivsize = DES_BLOCK_SIZE, 2108 .maxauthsize = SHA512_DIGEST_SIZE, 2109 }, 2110 .caam = { 2111 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2112 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2113 OP_ALG_AAI_HMAC_PRECOMP, 2114 } 2115 }, 2116 { 2117 .aead = { 2118 .base = { 2119 .cra_name = "echainiv(authenc(hmac(sha512)," 2120 "cbc(des)))", 2121 .cra_driver_name = "echainiv-authenc-" 2122 "hmac-sha512-cbc-des-" 2123 "caam-qi", 2124 .cra_blocksize = DES_BLOCK_SIZE, 2125 }, 2126 .setkey = aead_setkey, 2127 .setauthsize = aead_setauthsize, 2128 .encrypt = aead_encrypt, 2129 .decrypt = aead_decrypt, 2130 .ivsize = DES_BLOCK_SIZE, 2131 .maxauthsize = SHA512_DIGEST_SIZE, 2132 }, 2133 .caam = { 2134 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2135 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2136 OP_ALG_AAI_HMAC_PRECOMP, 2137 .geniv = true, 2138 } 2139 }, 2140 }; 2141 2142 struct caam_crypto_alg { 2143 struct list_head entry; 2144 struct crypto_alg crypto_alg; 2145 struct caam_alg_entry caam; 2146 }; 2147 2148 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2149 bool uses_dkp) 2150 { 2151 struct caam_drv_private *priv; 2152 2153 /* 2154 * distribute tfms across job rings to ensure in-order 2155 * crypto request processing per tfm 2156 */ 2157 ctx->jrdev = caam_jr_alloc(); 2158 if (IS_ERR(ctx->jrdev)) { 2159 pr_err("Job Ring Device allocation for transform failed\n"); 2160 return PTR_ERR(ctx->jrdev); 2161 } 2162 2163 priv = dev_get_drvdata(ctx->jrdev->parent); 2164 if (priv->era >= 6 && uses_dkp) 2165 ctx->dir = DMA_BIDIRECTIONAL; 2166 else 2167 ctx->dir = DMA_TO_DEVICE; 2168 2169 ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), 2170 ctx->dir); 2171 if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { 2172 dev_err(ctx->jrdev, "unable to map key\n"); 2173 caam_jr_free(ctx->jrdev); 2174 return -ENOMEM; 2175 } 2176 2177 /* copy descriptor header template value */ 2178 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2179 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2180 2181 ctx->qidev = priv->qidev; 2182 2183 spin_lock_init(&ctx->lock); 2184 ctx->drv_ctx[ENCRYPT] = NULL; 2185 ctx->drv_ctx[DECRYPT] = NULL; 2186 ctx->drv_ctx[GIVENCRYPT] = NULL; 2187 2188 return 0; 2189 } 2190 2191 static int caam_cra_init(struct crypto_tfm *tfm) 2192 { 2193 struct crypto_alg *alg = tfm->__crt_alg; 2194 struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2195 crypto_alg); 2196 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2197 2198 return caam_init_common(ctx, &caam_alg->caam, false); 2199 } 2200 2201 static int caam_aead_init(struct crypto_aead *tfm) 2202 { 2203 struct aead_alg *alg = crypto_aead_alg(tfm); 2204 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2205 aead); 2206 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 2207 2208 return caam_init_common(ctx, &caam_alg->caam, 2209 alg->setkey == aead_setkey); 2210 } 2211 2212 static void caam_exit_common(struct caam_ctx *ctx) 2213 { 2214 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2215 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2216 caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); 2217 2218 dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); 2219 2220 caam_jr_free(ctx->jrdev); 2221 } 2222 2223 static void caam_cra_exit(struct crypto_tfm *tfm) 2224 { 2225 caam_exit_common(crypto_tfm_ctx(tfm)); 2226 } 2227 2228 static void caam_aead_exit(struct crypto_aead *tfm) 2229 { 2230 caam_exit_common(crypto_aead_ctx(tfm)); 2231 } 2232 2233 static struct list_head alg_list; 2234 static void __exit caam_qi_algapi_exit(void) 2235 { 2236 struct caam_crypto_alg *t_alg, *n; 2237 int i; 2238 2239 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2240 struct caam_aead_alg *t_alg = driver_aeads + i; 2241 2242 if (t_alg->registered) 2243 crypto_unregister_aead(&t_alg->aead); 2244 } 2245 2246 if (!alg_list.next) 2247 return; 2248 2249 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 2250 crypto_unregister_alg(&t_alg->crypto_alg); 2251 list_del(&t_alg->entry); 2252 kfree(t_alg); 2253 } 2254 } 2255 2256 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 2257 *template) 2258 { 2259 struct caam_crypto_alg *t_alg; 2260 struct crypto_alg *alg; 2261 2262 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2263 if (!t_alg) 2264 return ERR_PTR(-ENOMEM); 2265 2266 alg = &t_alg->crypto_alg; 2267 2268 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 2269 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2270 template->driver_name); 2271 alg->cra_module = THIS_MODULE; 2272 alg->cra_init = caam_cra_init; 2273 alg->cra_exit = caam_cra_exit; 2274 alg->cra_priority = CAAM_CRA_PRIORITY; 2275 alg->cra_blocksize = template->blocksize; 2276 alg->cra_alignmask = 0; 2277 alg->cra_ctxsize = sizeof(struct caam_ctx); 2278 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 2279 template->type; 2280 switch (template->type) { 2281 case CRYPTO_ALG_TYPE_GIVCIPHER: 2282 alg->cra_type = &crypto_givcipher_type; 2283 alg->cra_ablkcipher = template->template_ablkcipher; 2284 break; 2285 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2286 alg->cra_type = &crypto_ablkcipher_type; 2287 alg->cra_ablkcipher = template->template_ablkcipher; 2288 break; 2289 } 2290 2291 t_alg->caam.class1_alg_type = template->class1_alg_type; 2292 t_alg->caam.class2_alg_type = template->class2_alg_type; 2293 2294 return t_alg; 2295 } 2296 2297 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2298 { 2299 struct aead_alg *alg = &t_alg->aead; 2300 2301 alg->base.cra_module = THIS_MODULE; 2302 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2303 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2304 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2305 2306 alg->init = caam_aead_init; 2307 alg->exit = caam_aead_exit; 2308 } 2309 2310 static int __init caam_qi_algapi_init(void) 2311 { 2312 struct device_node *dev_node; 2313 struct platform_device *pdev; 2314 struct device *ctrldev; 2315 struct caam_drv_private *priv; 2316 int i = 0, err = 0; 2317 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 2318 unsigned int md_limit = SHA512_DIGEST_SIZE; 2319 bool registered = false; 2320 2321 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2322 if (!dev_node) { 2323 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2324 if (!dev_node) 2325 return -ENODEV; 2326 } 2327 2328 pdev = of_find_device_by_node(dev_node); 2329 of_node_put(dev_node); 2330 if (!pdev) 2331 return -ENODEV; 2332 2333 ctrldev = &pdev->dev; 2334 priv = dev_get_drvdata(ctrldev); 2335 2336 /* 2337 * If priv is NULL, it's probably because the caam driver wasn't 2338 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 2339 */ 2340 if (!priv || !priv->qi_present) 2341 return -ENODEV; 2342 2343 if (caam_dpaa2) { 2344 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); 2345 return -ENODEV; 2346 } 2347 2348 INIT_LIST_HEAD(&alg_list); 2349 2350 /* 2351 * Register crypto algorithms the device supports. 2352 * First, detect presence and attributes of DES, AES, and MD blocks. 2353 */ 2354 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2355 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2356 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 2357 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 2358 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2359 2360 /* If MD is present, limit digest size based on LP256 */ 2361 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 2362 md_limit = SHA256_DIGEST_SIZE; 2363 2364 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2365 struct caam_crypto_alg *t_alg; 2366 struct caam_alg_template *alg = driver_algs + i; 2367 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 2368 2369 /* Skip DES algorithms if not supported by device */ 2370 if (!des_inst && 2371 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2372 (alg_sel == OP_ALG_ALGSEL_DES))) 2373 continue; 2374 2375 /* Skip AES algorithms if not supported by device */ 2376 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2377 continue; 2378 2379 t_alg = caam_alg_alloc(alg); 2380 if (IS_ERR(t_alg)) { 2381 err = PTR_ERR(t_alg); 2382 dev_warn(priv->qidev, "%s alg allocation failed\n", 2383 alg->driver_name); 2384 continue; 2385 } 2386 2387 err = crypto_register_alg(&t_alg->crypto_alg); 2388 if (err) { 2389 dev_warn(priv->qidev, "%s alg registration failed\n", 2390 t_alg->crypto_alg.cra_driver_name); 2391 kfree(t_alg); 2392 continue; 2393 } 2394 2395 list_add_tail(&t_alg->entry, &alg_list); 2396 registered = true; 2397 } 2398 2399 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2400 struct caam_aead_alg *t_alg = driver_aeads + i; 2401 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2402 OP_ALG_ALGSEL_MASK; 2403 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2404 OP_ALG_ALGSEL_MASK; 2405 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2406 2407 /* Skip DES algorithms if not supported by device */ 2408 if (!des_inst && 2409 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2410 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2411 continue; 2412 2413 /* Skip AES algorithms if not supported by device */ 2414 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2415 continue; 2416 2417 /* 2418 * Check support for AES algorithms not available 2419 * on LP devices. 2420 */ 2421 if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) && 2422 (alg_aai == OP_ALG_AAI_GCM)) 2423 continue; 2424 2425 /* 2426 * Skip algorithms requiring message digests 2427 * if MD or MD size is not supported by device. 2428 */ 2429 if (c2_alg_sel && 2430 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2431 continue; 2432 2433 caam_aead_alg_init(t_alg); 2434 2435 err = crypto_register_aead(&t_alg->aead); 2436 if (err) { 2437 pr_warn("%s alg registration failed\n", 2438 t_alg->aead.base.cra_driver_name); 2439 continue; 2440 } 2441 2442 t_alg->registered = true; 2443 registered = true; 2444 } 2445 2446 if (registered) 2447 dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); 2448 2449 return err; 2450 } 2451 2452 module_init(caam_qi_algapi_init); 2453 module_exit(caam_qi_algapi_exit); 2454 2455 MODULE_LICENSE("GPL"); 2456 MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); 2457 MODULE_AUTHOR("Freescale Semiconductor"); 2458