1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2018 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include <linux/fsl/mc.h> 19 #include <soc/fsl/dpaa2-io.h> 20 #include <soc/fsl/dpaa2-fd.h> 21 22 #define CAAM_CRA_PRIORITY 2000 23 24 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 25 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 26 SHA512_DIGEST_SIZE * 2) 27 28 #if !IS_ENABLED(CONFIG_CRYPTO_DEV_FSL_CAAM) 29 bool caam_little_end; 30 EXPORT_SYMBOL(caam_little_end); 31 bool caam_imx; 32 EXPORT_SYMBOL(caam_imx); 33 #endif 34 35 /* 36 * This is a a cache of buffers, from which the users of CAAM QI driver 37 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 38 * NOTE: A more elegant solution would be to have some headroom in the frames 39 * being processed. This can be added by the dpaa2-eth driver. This would 40 * pose a problem for userspace application processing which cannot 41 * know of this limitation. So for now, this will work. 42 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 43 */ 44 static struct kmem_cache *qi_cache; 45 46 struct caam_alg_entry { 47 struct device *dev; 48 int class1_alg_type; 49 int class2_alg_type; 50 bool rfc3686; 51 bool geniv; 52 }; 53 54 struct caam_aead_alg { 55 struct aead_alg aead; 56 struct caam_alg_entry caam; 57 bool registered; 58 }; 59 60 struct caam_skcipher_alg { 61 struct skcipher_alg skcipher; 62 struct caam_alg_entry caam; 63 bool registered; 64 }; 65 66 /** 67 * caam_ctx - per-session context 68 * @flc: Flow Contexts array 69 * @key: [authentication key], encryption key 70 * @flc_dma: I/O virtual addresses of the Flow Contexts 71 * @key_dma: I/O virtual address of the key 72 * @dir: DMA direction for mapping key and Flow Contexts 73 * @dev: dpseci device 74 * @adata: authentication algorithm details 75 * @cdata: encryption algorithm details 76 * @authsize: authentication tag (a.k.a. ICV / MAC) size 77 */ 78 struct caam_ctx { 79 struct caam_flc flc[NUM_OP]; 80 u8 key[CAAM_MAX_KEY_SIZE]; 81 dma_addr_t flc_dma[NUM_OP]; 82 dma_addr_t key_dma; 83 enum dma_data_direction dir; 84 struct device *dev; 85 struct alginfo adata; 86 struct alginfo cdata; 87 unsigned int authsize; 88 }; 89 90 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 91 dma_addr_t iova_addr) 92 { 93 phys_addr_t phys_addr; 94 95 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 96 iova_addr; 97 98 return phys_to_virt(phys_addr); 99 } 100 101 /* 102 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 103 * 104 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 105 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 106 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 107 * hosting 16 SG entries. 108 * 109 * @flags - flags that would be used for the equivalent kmalloc(..) call 110 * 111 * Returns a pointer to a retrieved buffer on success or NULL on failure. 112 */ 113 static inline void *qi_cache_zalloc(gfp_t flags) 114 { 115 return kmem_cache_zalloc(qi_cache, flags); 116 } 117 118 /* 119 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 120 * 121 * @obj - buffer previously allocated by qi_cache_zalloc 122 * 123 * No checking is being done, the call is a passthrough call to 124 * kmem_cache_free(...) 125 */ 126 static inline void qi_cache_free(void *obj) 127 { 128 kmem_cache_free(qi_cache, obj); 129 } 130 131 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 132 { 133 switch (crypto_tfm_alg_type(areq->tfm)) { 134 case CRYPTO_ALG_TYPE_SKCIPHER: 135 return skcipher_request_ctx(skcipher_request_cast(areq)); 136 case CRYPTO_ALG_TYPE_AEAD: 137 return aead_request_ctx(container_of(areq, struct aead_request, 138 base)); 139 case CRYPTO_ALG_TYPE_AHASH: 140 return ahash_request_ctx(ahash_request_cast(areq)); 141 default: 142 return ERR_PTR(-EINVAL); 143 } 144 } 145 146 static void caam_unmap(struct device *dev, struct scatterlist *src, 147 struct scatterlist *dst, int src_nents, 148 int dst_nents, dma_addr_t iv_dma, int ivsize, 149 dma_addr_t qm_sg_dma, int qm_sg_bytes) 150 { 151 if (dst != src) { 152 if (src_nents) 153 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 154 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 155 } else { 156 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 157 } 158 159 if (iv_dma) 160 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 161 162 if (qm_sg_bytes) 163 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 164 } 165 166 static int aead_set_sh_desc(struct crypto_aead *aead) 167 { 168 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 169 typeof(*alg), aead); 170 struct caam_ctx *ctx = crypto_aead_ctx(aead); 171 unsigned int ivsize = crypto_aead_ivsize(aead); 172 struct device *dev = ctx->dev; 173 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 174 struct caam_flc *flc; 175 u32 *desc; 176 u32 ctx1_iv_off = 0; 177 u32 *nonce = NULL; 178 unsigned int data_len[2]; 179 u32 inl_mask; 180 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 181 OP_ALG_AAI_CTR_MOD128); 182 const bool is_rfc3686 = alg->caam.rfc3686; 183 184 if (!ctx->cdata.keylen || !ctx->authsize) 185 return 0; 186 187 /* 188 * AES-CTR needs to load IV in CONTEXT1 reg 189 * at an offset of 128bits (16bytes) 190 * CONTEXT1[255:128] = IV 191 */ 192 if (ctr_mode) 193 ctx1_iv_off = 16; 194 195 /* 196 * RFC3686 specific: 197 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 198 */ 199 if (is_rfc3686) { 200 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 201 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 202 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 203 } 204 205 data_len[0] = ctx->adata.keylen_pad; 206 data_len[1] = ctx->cdata.keylen; 207 208 /* aead_encrypt shared descriptor */ 209 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 210 DESC_QI_AEAD_ENC_LEN) + 211 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 212 DESC_JOB_IO_LEN, data_len, &inl_mask, 213 ARRAY_SIZE(data_len)) < 0) 214 return -EINVAL; 215 216 if (inl_mask & 1) 217 ctx->adata.key_virt = ctx->key; 218 else 219 ctx->adata.key_dma = ctx->key_dma; 220 221 if (inl_mask & 2) 222 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 223 else 224 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 225 226 ctx->adata.key_inline = !!(inl_mask & 1); 227 ctx->cdata.key_inline = !!(inl_mask & 2); 228 229 flc = &ctx->flc[ENCRYPT]; 230 desc = flc->sh_desc; 231 232 if (alg->caam.geniv) 233 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 234 ivsize, ctx->authsize, is_rfc3686, 235 nonce, ctx1_iv_off, true, 236 priv->sec_attr.era); 237 else 238 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 239 ivsize, ctx->authsize, is_rfc3686, nonce, 240 ctx1_iv_off, true, priv->sec_attr.era); 241 242 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 243 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 244 sizeof(flc->flc) + desc_bytes(desc), 245 ctx->dir); 246 247 /* aead_decrypt shared descriptor */ 248 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 249 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 250 DESC_JOB_IO_LEN, data_len, &inl_mask, 251 ARRAY_SIZE(data_len)) < 0) 252 return -EINVAL; 253 254 if (inl_mask & 1) 255 ctx->adata.key_virt = ctx->key; 256 else 257 ctx->adata.key_dma = ctx->key_dma; 258 259 if (inl_mask & 2) 260 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 261 else 262 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 263 264 ctx->adata.key_inline = !!(inl_mask & 1); 265 ctx->cdata.key_inline = !!(inl_mask & 2); 266 267 flc = &ctx->flc[DECRYPT]; 268 desc = flc->sh_desc; 269 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 270 ivsize, ctx->authsize, alg->caam.geniv, 271 is_rfc3686, nonce, ctx1_iv_off, true, 272 priv->sec_attr.era); 273 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 274 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 275 sizeof(flc->flc) + desc_bytes(desc), 276 ctx->dir); 277 278 return 0; 279 } 280 281 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 282 { 283 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 284 285 ctx->authsize = authsize; 286 aead_set_sh_desc(authenc); 287 288 return 0; 289 } 290 291 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 292 unsigned int keylen) 293 { 294 struct caam_ctx *ctx = crypto_aead_ctx(aead); 295 struct device *dev = ctx->dev; 296 struct crypto_authenc_keys keys; 297 298 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 299 goto badkey; 300 301 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 302 keys.authkeylen + keys.enckeylen, keys.enckeylen, 303 keys.authkeylen); 304 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 305 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 306 307 ctx->adata.keylen = keys.authkeylen; 308 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 309 OP_ALG_ALGSEL_MASK); 310 311 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 312 goto badkey; 313 314 memcpy(ctx->key, keys.authkey, keys.authkeylen); 315 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 316 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 317 keys.enckeylen, ctx->dir); 318 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 319 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 320 ctx->adata.keylen_pad + keys.enckeylen, 1); 321 322 ctx->cdata.keylen = keys.enckeylen; 323 324 memzero_explicit(&keys, sizeof(keys)); 325 return aead_set_sh_desc(aead); 326 badkey: 327 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 328 memzero_explicit(&keys, sizeof(keys)); 329 return -EINVAL; 330 } 331 332 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 333 bool encrypt) 334 { 335 struct crypto_aead *aead = crypto_aead_reqtfm(req); 336 struct caam_request *req_ctx = aead_request_ctx(req); 337 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 338 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 339 struct caam_ctx *ctx = crypto_aead_ctx(aead); 340 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 341 typeof(*alg), aead); 342 struct device *dev = ctx->dev; 343 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 344 GFP_KERNEL : GFP_ATOMIC; 345 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 346 struct aead_edesc *edesc; 347 dma_addr_t qm_sg_dma, iv_dma = 0; 348 int ivsize = 0; 349 unsigned int authsize = ctx->authsize; 350 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 351 int in_len, out_len; 352 struct dpaa2_sg_entry *sg_table; 353 354 /* allocate space for base edesc, link tables and IV */ 355 edesc = qi_cache_zalloc(GFP_DMA | flags); 356 if (unlikely(!edesc)) { 357 dev_err(dev, "could not allocate extended descriptor\n"); 358 return ERR_PTR(-ENOMEM); 359 } 360 361 if (unlikely(req->dst != req->src)) { 362 src_nents = sg_nents_for_len(req->src, req->assoclen + 363 req->cryptlen); 364 if (unlikely(src_nents < 0)) { 365 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 366 req->assoclen + req->cryptlen); 367 qi_cache_free(edesc); 368 return ERR_PTR(src_nents); 369 } 370 371 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 372 req->cryptlen + 373 (encrypt ? authsize : 374 (-authsize))); 375 if (unlikely(dst_nents < 0)) { 376 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 377 req->assoclen + req->cryptlen + 378 (encrypt ? authsize : (-authsize))); 379 qi_cache_free(edesc); 380 return ERR_PTR(dst_nents); 381 } 382 383 if (src_nents) { 384 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 385 DMA_TO_DEVICE); 386 if (unlikely(!mapped_src_nents)) { 387 dev_err(dev, "unable to map source\n"); 388 qi_cache_free(edesc); 389 return ERR_PTR(-ENOMEM); 390 } 391 } else { 392 mapped_src_nents = 0; 393 } 394 395 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 396 DMA_FROM_DEVICE); 397 if (unlikely(!mapped_dst_nents)) { 398 dev_err(dev, "unable to map destination\n"); 399 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 400 qi_cache_free(edesc); 401 return ERR_PTR(-ENOMEM); 402 } 403 } else { 404 src_nents = sg_nents_for_len(req->src, req->assoclen + 405 req->cryptlen + 406 (encrypt ? authsize : 0)); 407 if (unlikely(src_nents < 0)) { 408 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 409 req->assoclen + req->cryptlen + 410 (encrypt ? authsize : 0)); 411 qi_cache_free(edesc); 412 return ERR_PTR(src_nents); 413 } 414 415 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 416 DMA_BIDIRECTIONAL); 417 if (unlikely(!mapped_src_nents)) { 418 dev_err(dev, "unable to map source\n"); 419 qi_cache_free(edesc); 420 return ERR_PTR(-ENOMEM); 421 } 422 } 423 424 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 425 ivsize = crypto_aead_ivsize(aead); 426 427 /* 428 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 429 * Input is not contiguous. 430 */ 431 qm_sg_nents = 1 + !!ivsize + mapped_src_nents + 432 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 433 sg_table = &edesc->sgt[0]; 434 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 435 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 436 CAAM_QI_MEMCACHE_SIZE)) { 437 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 438 qm_sg_nents, ivsize); 439 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 440 0, 0, 0); 441 qi_cache_free(edesc); 442 return ERR_PTR(-ENOMEM); 443 } 444 445 if (ivsize) { 446 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 447 448 /* Make sure IV is located in a DMAable area */ 449 memcpy(iv, req->iv, ivsize); 450 451 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 452 if (dma_mapping_error(dev, iv_dma)) { 453 dev_err(dev, "unable to map IV\n"); 454 caam_unmap(dev, req->src, req->dst, src_nents, 455 dst_nents, 0, 0, 0, 0); 456 qi_cache_free(edesc); 457 return ERR_PTR(-ENOMEM); 458 } 459 } 460 461 edesc->src_nents = src_nents; 462 edesc->dst_nents = dst_nents; 463 edesc->iv_dma = iv_dma; 464 465 edesc->assoclen = cpu_to_caam32(req->assoclen); 466 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 467 DMA_TO_DEVICE); 468 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 469 dev_err(dev, "unable to map assoclen\n"); 470 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 471 iv_dma, ivsize, 0, 0); 472 qi_cache_free(edesc); 473 return ERR_PTR(-ENOMEM); 474 } 475 476 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 477 qm_sg_index++; 478 if (ivsize) { 479 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 480 qm_sg_index++; 481 } 482 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); 483 qm_sg_index += mapped_src_nents; 484 485 if (mapped_dst_nents > 1) 486 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 487 qm_sg_index, 0); 488 489 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 490 if (dma_mapping_error(dev, qm_sg_dma)) { 491 dev_err(dev, "unable to map S/G table\n"); 492 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 493 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 494 iv_dma, ivsize, 0, 0); 495 qi_cache_free(edesc); 496 return ERR_PTR(-ENOMEM); 497 } 498 499 edesc->qm_sg_dma = qm_sg_dma; 500 edesc->qm_sg_bytes = qm_sg_bytes; 501 502 out_len = req->assoclen + req->cryptlen + 503 (encrypt ? ctx->authsize : (-ctx->authsize)); 504 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 505 506 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 507 dpaa2_fl_set_final(in_fle, true); 508 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 509 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 510 dpaa2_fl_set_len(in_fle, in_len); 511 512 if (req->dst == req->src) { 513 if (mapped_src_nents == 1) { 514 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 515 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 516 } else { 517 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 518 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 519 (1 + !!ivsize) * sizeof(*sg_table)); 520 } 521 } else if (mapped_dst_nents == 1) { 522 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 523 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 524 } else { 525 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 526 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 527 sizeof(*sg_table)); 528 } 529 530 dpaa2_fl_set_len(out_fle, out_len); 531 532 return edesc; 533 } 534 535 static int gcm_set_sh_desc(struct crypto_aead *aead) 536 { 537 struct caam_ctx *ctx = crypto_aead_ctx(aead); 538 struct device *dev = ctx->dev; 539 unsigned int ivsize = crypto_aead_ivsize(aead); 540 struct caam_flc *flc; 541 u32 *desc; 542 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 543 ctx->cdata.keylen; 544 545 if (!ctx->cdata.keylen || !ctx->authsize) 546 return 0; 547 548 /* 549 * AES GCM encrypt shared descriptor 550 * Job Descriptor and Shared Descriptor 551 * must fit into the 64-word Descriptor h/w Buffer 552 */ 553 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 554 ctx->cdata.key_inline = true; 555 ctx->cdata.key_virt = ctx->key; 556 } else { 557 ctx->cdata.key_inline = false; 558 ctx->cdata.key_dma = ctx->key_dma; 559 } 560 561 flc = &ctx->flc[ENCRYPT]; 562 desc = flc->sh_desc; 563 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 564 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 565 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 566 sizeof(flc->flc) + desc_bytes(desc), 567 ctx->dir); 568 569 /* 570 * Job Descriptor and Shared Descriptors 571 * must all fit into the 64-word Descriptor h/w Buffer 572 */ 573 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 574 ctx->cdata.key_inline = true; 575 ctx->cdata.key_virt = ctx->key; 576 } else { 577 ctx->cdata.key_inline = false; 578 ctx->cdata.key_dma = ctx->key_dma; 579 } 580 581 flc = &ctx->flc[DECRYPT]; 582 desc = flc->sh_desc; 583 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 584 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 585 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 586 sizeof(flc->flc) + desc_bytes(desc), 587 ctx->dir); 588 589 return 0; 590 } 591 592 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 593 { 594 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 595 596 ctx->authsize = authsize; 597 gcm_set_sh_desc(authenc); 598 599 return 0; 600 } 601 602 static int gcm_setkey(struct crypto_aead *aead, 603 const u8 *key, unsigned int keylen) 604 { 605 struct caam_ctx *ctx = crypto_aead_ctx(aead); 606 struct device *dev = ctx->dev; 607 608 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 609 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 610 611 memcpy(ctx->key, key, keylen); 612 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 613 ctx->cdata.keylen = keylen; 614 615 return gcm_set_sh_desc(aead); 616 } 617 618 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 619 { 620 struct caam_ctx *ctx = crypto_aead_ctx(aead); 621 struct device *dev = ctx->dev; 622 unsigned int ivsize = crypto_aead_ivsize(aead); 623 struct caam_flc *flc; 624 u32 *desc; 625 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 626 ctx->cdata.keylen; 627 628 if (!ctx->cdata.keylen || !ctx->authsize) 629 return 0; 630 631 ctx->cdata.key_virt = ctx->key; 632 633 /* 634 * RFC4106 encrypt shared descriptor 635 * Job Descriptor and Shared Descriptor 636 * must fit into the 64-word Descriptor h/w Buffer 637 */ 638 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 639 ctx->cdata.key_inline = true; 640 } else { 641 ctx->cdata.key_inline = false; 642 ctx->cdata.key_dma = ctx->key_dma; 643 } 644 645 flc = &ctx->flc[ENCRYPT]; 646 desc = flc->sh_desc; 647 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 648 true); 649 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 650 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 651 sizeof(flc->flc) + desc_bytes(desc), 652 ctx->dir); 653 654 /* 655 * Job Descriptor and Shared Descriptors 656 * must all fit into the 64-word Descriptor h/w Buffer 657 */ 658 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 659 ctx->cdata.key_inline = true; 660 } else { 661 ctx->cdata.key_inline = false; 662 ctx->cdata.key_dma = ctx->key_dma; 663 } 664 665 flc = &ctx->flc[DECRYPT]; 666 desc = flc->sh_desc; 667 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 668 true); 669 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 670 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 671 sizeof(flc->flc) + desc_bytes(desc), 672 ctx->dir); 673 674 return 0; 675 } 676 677 static int rfc4106_setauthsize(struct crypto_aead *authenc, 678 unsigned int authsize) 679 { 680 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 681 682 ctx->authsize = authsize; 683 rfc4106_set_sh_desc(authenc); 684 685 return 0; 686 } 687 688 static int rfc4106_setkey(struct crypto_aead *aead, 689 const u8 *key, unsigned int keylen) 690 { 691 struct caam_ctx *ctx = crypto_aead_ctx(aead); 692 struct device *dev = ctx->dev; 693 694 if (keylen < 4) 695 return -EINVAL; 696 697 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 698 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 699 700 memcpy(ctx->key, key, keylen); 701 /* 702 * The last four bytes of the key material are used as the salt value 703 * in the nonce. Update the AES key length. 704 */ 705 ctx->cdata.keylen = keylen - 4; 706 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 707 ctx->dir); 708 709 return rfc4106_set_sh_desc(aead); 710 } 711 712 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 713 { 714 struct caam_ctx *ctx = crypto_aead_ctx(aead); 715 struct device *dev = ctx->dev; 716 unsigned int ivsize = crypto_aead_ivsize(aead); 717 struct caam_flc *flc; 718 u32 *desc; 719 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 720 ctx->cdata.keylen; 721 722 if (!ctx->cdata.keylen || !ctx->authsize) 723 return 0; 724 725 ctx->cdata.key_virt = ctx->key; 726 727 /* 728 * RFC4543 encrypt shared descriptor 729 * Job Descriptor and Shared Descriptor 730 * must fit into the 64-word Descriptor h/w Buffer 731 */ 732 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 733 ctx->cdata.key_inline = true; 734 } else { 735 ctx->cdata.key_inline = false; 736 ctx->cdata.key_dma = ctx->key_dma; 737 } 738 739 flc = &ctx->flc[ENCRYPT]; 740 desc = flc->sh_desc; 741 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 742 true); 743 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 744 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 745 sizeof(flc->flc) + desc_bytes(desc), 746 ctx->dir); 747 748 /* 749 * Job Descriptor and Shared Descriptors 750 * must all fit into the 64-word Descriptor h/w Buffer 751 */ 752 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 753 ctx->cdata.key_inline = true; 754 } else { 755 ctx->cdata.key_inline = false; 756 ctx->cdata.key_dma = ctx->key_dma; 757 } 758 759 flc = &ctx->flc[DECRYPT]; 760 desc = flc->sh_desc; 761 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 762 true); 763 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 764 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 765 sizeof(flc->flc) + desc_bytes(desc), 766 ctx->dir); 767 768 return 0; 769 } 770 771 static int rfc4543_setauthsize(struct crypto_aead *authenc, 772 unsigned int authsize) 773 { 774 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 775 776 ctx->authsize = authsize; 777 rfc4543_set_sh_desc(authenc); 778 779 return 0; 780 } 781 782 static int rfc4543_setkey(struct crypto_aead *aead, 783 const u8 *key, unsigned int keylen) 784 { 785 struct caam_ctx *ctx = crypto_aead_ctx(aead); 786 struct device *dev = ctx->dev; 787 788 if (keylen < 4) 789 return -EINVAL; 790 791 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 792 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 793 794 memcpy(ctx->key, key, keylen); 795 /* 796 * The last four bytes of the key material are used as the salt value 797 * in the nonce. Update the AES key length. 798 */ 799 ctx->cdata.keylen = keylen - 4; 800 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 801 ctx->dir); 802 803 return rfc4543_set_sh_desc(aead); 804 } 805 806 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 807 unsigned int keylen) 808 { 809 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 810 struct caam_skcipher_alg *alg = 811 container_of(crypto_skcipher_alg(skcipher), 812 struct caam_skcipher_alg, skcipher); 813 struct device *dev = ctx->dev; 814 struct caam_flc *flc; 815 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 816 u32 *desc; 817 u32 ctx1_iv_off = 0; 818 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 819 OP_ALG_AAI_CTR_MOD128); 820 const bool is_rfc3686 = alg->caam.rfc3686; 821 822 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 823 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 824 825 /* 826 * AES-CTR needs to load IV in CONTEXT1 reg 827 * at an offset of 128bits (16bytes) 828 * CONTEXT1[255:128] = IV 829 */ 830 if (ctr_mode) 831 ctx1_iv_off = 16; 832 833 /* 834 * RFC3686 specific: 835 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 836 * | *key = {KEY, NONCE} 837 */ 838 if (is_rfc3686) { 839 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 840 keylen -= CTR_RFC3686_NONCE_SIZE; 841 } 842 843 ctx->cdata.keylen = keylen; 844 ctx->cdata.key_virt = key; 845 ctx->cdata.key_inline = true; 846 847 /* skcipher_encrypt shared descriptor */ 848 flc = &ctx->flc[ENCRYPT]; 849 desc = flc->sh_desc; 850 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 851 ctx1_iv_off); 852 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 853 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 854 sizeof(flc->flc) + desc_bytes(desc), 855 ctx->dir); 856 857 /* skcipher_decrypt shared descriptor */ 858 flc = &ctx->flc[DECRYPT]; 859 desc = flc->sh_desc; 860 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 861 ctx1_iv_off); 862 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 863 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 864 sizeof(flc->flc) + desc_bytes(desc), 865 ctx->dir); 866 867 return 0; 868 } 869 870 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 871 unsigned int keylen) 872 { 873 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 874 struct device *dev = ctx->dev; 875 struct caam_flc *flc; 876 u32 *desc; 877 878 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 879 dev_err(dev, "key size mismatch\n"); 880 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 881 return -EINVAL; 882 } 883 884 ctx->cdata.keylen = keylen; 885 ctx->cdata.key_virt = key; 886 ctx->cdata.key_inline = true; 887 888 /* xts_skcipher_encrypt shared descriptor */ 889 flc = &ctx->flc[ENCRYPT]; 890 desc = flc->sh_desc; 891 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 892 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 893 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 894 sizeof(flc->flc) + desc_bytes(desc), 895 ctx->dir); 896 897 /* xts_skcipher_decrypt shared descriptor */ 898 flc = &ctx->flc[DECRYPT]; 899 desc = flc->sh_desc; 900 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 901 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 902 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 903 sizeof(flc->flc) + desc_bytes(desc), 904 ctx->dir); 905 906 return 0; 907 } 908 909 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 910 { 911 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 912 struct caam_request *req_ctx = skcipher_request_ctx(req); 913 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 914 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 915 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 916 struct device *dev = ctx->dev; 917 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 918 GFP_KERNEL : GFP_ATOMIC; 919 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 920 struct skcipher_edesc *edesc; 921 dma_addr_t iv_dma; 922 u8 *iv; 923 int ivsize = crypto_skcipher_ivsize(skcipher); 924 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 925 struct dpaa2_sg_entry *sg_table; 926 927 src_nents = sg_nents_for_len(req->src, req->cryptlen); 928 if (unlikely(src_nents < 0)) { 929 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 930 req->cryptlen); 931 return ERR_PTR(src_nents); 932 } 933 934 if (unlikely(req->dst != req->src)) { 935 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 936 if (unlikely(dst_nents < 0)) { 937 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 938 req->cryptlen); 939 return ERR_PTR(dst_nents); 940 } 941 942 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 943 DMA_TO_DEVICE); 944 if (unlikely(!mapped_src_nents)) { 945 dev_err(dev, "unable to map source\n"); 946 return ERR_PTR(-ENOMEM); 947 } 948 949 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 950 DMA_FROM_DEVICE); 951 if (unlikely(!mapped_dst_nents)) { 952 dev_err(dev, "unable to map destination\n"); 953 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 954 return ERR_PTR(-ENOMEM); 955 } 956 } else { 957 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 958 DMA_BIDIRECTIONAL); 959 if (unlikely(!mapped_src_nents)) { 960 dev_err(dev, "unable to map source\n"); 961 return ERR_PTR(-ENOMEM); 962 } 963 } 964 965 qm_sg_ents = 1 + mapped_src_nents; 966 dst_sg_idx = qm_sg_ents; 967 968 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 969 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 970 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 971 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 972 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 973 qm_sg_ents, ivsize); 974 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 975 0, 0, 0); 976 return ERR_PTR(-ENOMEM); 977 } 978 979 /* allocate space for base edesc, link tables and IV */ 980 edesc = qi_cache_zalloc(GFP_DMA | flags); 981 if (unlikely(!edesc)) { 982 dev_err(dev, "could not allocate extended descriptor\n"); 983 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 984 0, 0, 0); 985 return ERR_PTR(-ENOMEM); 986 } 987 988 /* Make sure IV is located in a DMAable area */ 989 sg_table = &edesc->sgt[0]; 990 iv = (u8 *)(sg_table + qm_sg_ents); 991 memcpy(iv, req->iv, ivsize); 992 993 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 994 if (dma_mapping_error(dev, iv_dma)) { 995 dev_err(dev, "unable to map IV\n"); 996 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 997 0, 0, 0); 998 qi_cache_free(edesc); 999 return ERR_PTR(-ENOMEM); 1000 } 1001 1002 edesc->src_nents = src_nents; 1003 edesc->dst_nents = dst_nents; 1004 edesc->iv_dma = iv_dma; 1005 edesc->qm_sg_bytes = qm_sg_bytes; 1006 1007 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1008 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); 1009 1010 if (mapped_dst_nents > 1) 1011 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1012 dst_sg_idx, 0); 1013 1014 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1015 DMA_TO_DEVICE); 1016 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1017 dev_err(dev, "unable to map S/G table\n"); 1018 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1019 iv_dma, ivsize, 0, 0); 1020 qi_cache_free(edesc); 1021 return ERR_PTR(-ENOMEM); 1022 } 1023 1024 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1025 dpaa2_fl_set_final(in_fle, true); 1026 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1027 dpaa2_fl_set_len(out_fle, req->cryptlen); 1028 1029 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1030 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1031 1032 if (req->src == req->dst) { 1033 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1034 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1035 sizeof(*sg_table)); 1036 } else if (mapped_dst_nents > 1) { 1037 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1038 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1039 sizeof(*sg_table)); 1040 } else { 1041 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 1042 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 1043 } 1044 1045 return edesc; 1046 } 1047 1048 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1049 struct aead_request *req) 1050 { 1051 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1052 int ivsize = crypto_aead_ivsize(aead); 1053 1054 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1055 edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); 1056 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1057 } 1058 1059 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1060 struct skcipher_request *req) 1061 { 1062 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1063 int ivsize = crypto_skcipher_ivsize(skcipher); 1064 1065 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1066 edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); 1067 } 1068 1069 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1070 { 1071 struct crypto_async_request *areq = cbk_ctx; 1072 struct aead_request *req = container_of(areq, struct aead_request, 1073 base); 1074 struct caam_request *req_ctx = to_caam_req(areq); 1075 struct aead_edesc *edesc = req_ctx->edesc; 1076 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1077 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1078 int ecode = 0; 1079 1080 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1081 1082 if (unlikely(status)) { 1083 caam_qi2_strstatus(ctx->dev, status); 1084 ecode = -EIO; 1085 } 1086 1087 aead_unmap(ctx->dev, edesc, req); 1088 qi_cache_free(edesc); 1089 aead_request_complete(req, ecode); 1090 } 1091 1092 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1093 { 1094 struct crypto_async_request *areq = cbk_ctx; 1095 struct aead_request *req = container_of(areq, struct aead_request, 1096 base); 1097 struct caam_request *req_ctx = to_caam_req(areq); 1098 struct aead_edesc *edesc = req_ctx->edesc; 1099 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1100 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1101 int ecode = 0; 1102 1103 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1104 1105 if (unlikely(status)) { 1106 caam_qi2_strstatus(ctx->dev, status); 1107 /* 1108 * verify hw auth check passed else return -EBADMSG 1109 */ 1110 if ((status & JRSTA_CCBERR_ERRID_MASK) == 1111 JRSTA_CCBERR_ERRID_ICVCHK) 1112 ecode = -EBADMSG; 1113 else 1114 ecode = -EIO; 1115 } 1116 1117 aead_unmap(ctx->dev, edesc, req); 1118 qi_cache_free(edesc); 1119 aead_request_complete(req, ecode); 1120 } 1121 1122 static int aead_encrypt(struct aead_request *req) 1123 { 1124 struct aead_edesc *edesc; 1125 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1126 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1127 struct caam_request *caam_req = aead_request_ctx(req); 1128 int ret; 1129 1130 /* allocate extended descriptor */ 1131 edesc = aead_edesc_alloc(req, true); 1132 if (IS_ERR(edesc)) 1133 return PTR_ERR(edesc); 1134 1135 caam_req->flc = &ctx->flc[ENCRYPT]; 1136 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1137 caam_req->cbk = aead_encrypt_done; 1138 caam_req->ctx = &req->base; 1139 caam_req->edesc = edesc; 1140 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1141 if (ret != -EINPROGRESS && 1142 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1143 aead_unmap(ctx->dev, edesc, req); 1144 qi_cache_free(edesc); 1145 } 1146 1147 return ret; 1148 } 1149 1150 static int aead_decrypt(struct aead_request *req) 1151 { 1152 struct aead_edesc *edesc; 1153 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1154 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1155 struct caam_request *caam_req = aead_request_ctx(req); 1156 int ret; 1157 1158 /* allocate extended descriptor */ 1159 edesc = aead_edesc_alloc(req, false); 1160 if (IS_ERR(edesc)) 1161 return PTR_ERR(edesc); 1162 1163 caam_req->flc = &ctx->flc[DECRYPT]; 1164 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1165 caam_req->cbk = aead_decrypt_done; 1166 caam_req->ctx = &req->base; 1167 caam_req->edesc = edesc; 1168 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1169 if (ret != -EINPROGRESS && 1170 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1171 aead_unmap(ctx->dev, edesc, req); 1172 qi_cache_free(edesc); 1173 } 1174 1175 return ret; 1176 } 1177 1178 static int ipsec_gcm_encrypt(struct aead_request *req) 1179 { 1180 if (req->assoclen < 8) 1181 return -EINVAL; 1182 1183 return aead_encrypt(req); 1184 } 1185 1186 static int ipsec_gcm_decrypt(struct aead_request *req) 1187 { 1188 if (req->assoclen < 8) 1189 return -EINVAL; 1190 1191 return aead_decrypt(req); 1192 } 1193 1194 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1195 { 1196 struct crypto_async_request *areq = cbk_ctx; 1197 struct skcipher_request *req = skcipher_request_cast(areq); 1198 struct caam_request *req_ctx = to_caam_req(areq); 1199 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1200 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1201 struct skcipher_edesc *edesc = req_ctx->edesc; 1202 int ecode = 0; 1203 int ivsize = crypto_skcipher_ivsize(skcipher); 1204 1205 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1206 1207 if (unlikely(status)) { 1208 caam_qi2_strstatus(ctx->dev, status); 1209 ecode = -EIO; 1210 } 1211 1212 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1213 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1214 edesc->src_nents > 1 ? 100 : ivsize, 1); 1215 caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", 1216 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1217 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1218 1219 skcipher_unmap(ctx->dev, edesc, req); 1220 1221 /* 1222 * The crypto API expects us to set the IV (req->iv) to the last 1223 * ciphertext block. This is used e.g. by the CTS mode. 1224 */ 1225 scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, 1226 ivsize, 0); 1227 1228 qi_cache_free(edesc); 1229 skcipher_request_complete(req, ecode); 1230 } 1231 1232 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1233 { 1234 struct crypto_async_request *areq = cbk_ctx; 1235 struct skcipher_request *req = skcipher_request_cast(areq); 1236 struct caam_request *req_ctx = to_caam_req(areq); 1237 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1238 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1239 struct skcipher_edesc *edesc = req_ctx->edesc; 1240 int ecode = 0; 1241 int ivsize = crypto_skcipher_ivsize(skcipher); 1242 1243 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1244 1245 if (unlikely(status)) { 1246 caam_qi2_strstatus(ctx->dev, status); 1247 ecode = -EIO; 1248 } 1249 1250 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1251 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1252 edesc->src_nents > 1 ? 100 : ivsize, 1); 1253 caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", 1254 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1255 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1256 1257 skcipher_unmap(ctx->dev, edesc, req); 1258 qi_cache_free(edesc); 1259 skcipher_request_complete(req, ecode); 1260 } 1261 1262 static int skcipher_encrypt(struct skcipher_request *req) 1263 { 1264 struct skcipher_edesc *edesc; 1265 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1266 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1267 struct caam_request *caam_req = skcipher_request_ctx(req); 1268 int ret; 1269 1270 /* allocate extended descriptor */ 1271 edesc = skcipher_edesc_alloc(req); 1272 if (IS_ERR(edesc)) 1273 return PTR_ERR(edesc); 1274 1275 caam_req->flc = &ctx->flc[ENCRYPT]; 1276 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1277 caam_req->cbk = skcipher_encrypt_done; 1278 caam_req->ctx = &req->base; 1279 caam_req->edesc = edesc; 1280 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1281 if (ret != -EINPROGRESS && 1282 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1283 skcipher_unmap(ctx->dev, edesc, req); 1284 qi_cache_free(edesc); 1285 } 1286 1287 return ret; 1288 } 1289 1290 static int skcipher_decrypt(struct skcipher_request *req) 1291 { 1292 struct skcipher_edesc *edesc; 1293 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1294 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1295 struct caam_request *caam_req = skcipher_request_ctx(req); 1296 int ivsize = crypto_skcipher_ivsize(skcipher); 1297 int ret; 1298 1299 /* allocate extended descriptor */ 1300 edesc = skcipher_edesc_alloc(req); 1301 if (IS_ERR(edesc)) 1302 return PTR_ERR(edesc); 1303 1304 /* 1305 * The crypto API expects us to set the IV (req->iv) to the last 1306 * ciphertext block. 1307 */ 1308 scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize, 1309 ivsize, 0); 1310 1311 caam_req->flc = &ctx->flc[DECRYPT]; 1312 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1313 caam_req->cbk = skcipher_decrypt_done; 1314 caam_req->ctx = &req->base; 1315 caam_req->edesc = edesc; 1316 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1317 if (ret != -EINPROGRESS && 1318 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1319 skcipher_unmap(ctx->dev, edesc, req); 1320 qi_cache_free(edesc); 1321 } 1322 1323 return ret; 1324 } 1325 1326 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1327 bool uses_dkp) 1328 { 1329 dma_addr_t dma_addr; 1330 int i; 1331 1332 /* copy descriptor header template value */ 1333 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1334 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1335 1336 ctx->dev = caam->dev; 1337 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1338 1339 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1340 offsetof(struct caam_ctx, flc_dma), 1341 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1342 if (dma_mapping_error(ctx->dev, dma_addr)) { 1343 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1344 return -ENOMEM; 1345 } 1346 1347 for (i = 0; i < NUM_OP; i++) 1348 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1349 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1350 1351 return 0; 1352 } 1353 1354 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1355 { 1356 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1357 struct caam_skcipher_alg *caam_alg = 1358 container_of(alg, typeof(*caam_alg), skcipher); 1359 1360 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); 1361 return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false); 1362 } 1363 1364 static int caam_cra_init_aead(struct crypto_aead *tfm) 1365 { 1366 struct aead_alg *alg = crypto_aead_alg(tfm); 1367 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1368 aead); 1369 1370 crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); 1371 return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam, 1372 alg->setkey == aead_setkey); 1373 } 1374 1375 static void caam_exit_common(struct caam_ctx *ctx) 1376 { 1377 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1378 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1379 DMA_ATTR_SKIP_CPU_SYNC); 1380 } 1381 1382 static void caam_cra_exit(struct crypto_skcipher *tfm) 1383 { 1384 caam_exit_common(crypto_skcipher_ctx(tfm)); 1385 } 1386 1387 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1388 { 1389 caam_exit_common(crypto_aead_ctx(tfm)); 1390 } 1391 1392 static struct caam_skcipher_alg driver_algs[] = { 1393 { 1394 .skcipher = { 1395 .base = { 1396 .cra_name = "cbc(aes)", 1397 .cra_driver_name = "cbc-aes-caam-qi2", 1398 .cra_blocksize = AES_BLOCK_SIZE, 1399 }, 1400 .setkey = skcipher_setkey, 1401 .encrypt = skcipher_encrypt, 1402 .decrypt = skcipher_decrypt, 1403 .min_keysize = AES_MIN_KEY_SIZE, 1404 .max_keysize = AES_MAX_KEY_SIZE, 1405 .ivsize = AES_BLOCK_SIZE, 1406 }, 1407 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1408 }, 1409 { 1410 .skcipher = { 1411 .base = { 1412 .cra_name = "cbc(des3_ede)", 1413 .cra_driver_name = "cbc-3des-caam-qi2", 1414 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1415 }, 1416 .setkey = skcipher_setkey, 1417 .encrypt = skcipher_encrypt, 1418 .decrypt = skcipher_decrypt, 1419 .min_keysize = DES3_EDE_KEY_SIZE, 1420 .max_keysize = DES3_EDE_KEY_SIZE, 1421 .ivsize = DES3_EDE_BLOCK_SIZE, 1422 }, 1423 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1424 }, 1425 { 1426 .skcipher = { 1427 .base = { 1428 .cra_name = "cbc(des)", 1429 .cra_driver_name = "cbc-des-caam-qi2", 1430 .cra_blocksize = DES_BLOCK_SIZE, 1431 }, 1432 .setkey = skcipher_setkey, 1433 .encrypt = skcipher_encrypt, 1434 .decrypt = skcipher_decrypt, 1435 .min_keysize = DES_KEY_SIZE, 1436 .max_keysize = DES_KEY_SIZE, 1437 .ivsize = DES_BLOCK_SIZE, 1438 }, 1439 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1440 }, 1441 { 1442 .skcipher = { 1443 .base = { 1444 .cra_name = "ctr(aes)", 1445 .cra_driver_name = "ctr-aes-caam-qi2", 1446 .cra_blocksize = 1, 1447 }, 1448 .setkey = skcipher_setkey, 1449 .encrypt = skcipher_encrypt, 1450 .decrypt = skcipher_decrypt, 1451 .min_keysize = AES_MIN_KEY_SIZE, 1452 .max_keysize = AES_MAX_KEY_SIZE, 1453 .ivsize = AES_BLOCK_SIZE, 1454 .chunksize = AES_BLOCK_SIZE, 1455 }, 1456 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1457 OP_ALG_AAI_CTR_MOD128, 1458 }, 1459 { 1460 .skcipher = { 1461 .base = { 1462 .cra_name = "rfc3686(ctr(aes))", 1463 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1464 .cra_blocksize = 1, 1465 }, 1466 .setkey = skcipher_setkey, 1467 .encrypt = skcipher_encrypt, 1468 .decrypt = skcipher_decrypt, 1469 .min_keysize = AES_MIN_KEY_SIZE + 1470 CTR_RFC3686_NONCE_SIZE, 1471 .max_keysize = AES_MAX_KEY_SIZE + 1472 CTR_RFC3686_NONCE_SIZE, 1473 .ivsize = CTR_RFC3686_IV_SIZE, 1474 .chunksize = AES_BLOCK_SIZE, 1475 }, 1476 .caam = { 1477 .class1_alg_type = OP_ALG_ALGSEL_AES | 1478 OP_ALG_AAI_CTR_MOD128, 1479 .rfc3686 = true, 1480 }, 1481 }, 1482 { 1483 .skcipher = { 1484 .base = { 1485 .cra_name = "xts(aes)", 1486 .cra_driver_name = "xts-aes-caam-qi2", 1487 .cra_blocksize = AES_BLOCK_SIZE, 1488 }, 1489 .setkey = xts_skcipher_setkey, 1490 .encrypt = skcipher_encrypt, 1491 .decrypt = skcipher_decrypt, 1492 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1493 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1494 .ivsize = AES_BLOCK_SIZE, 1495 }, 1496 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1497 } 1498 }; 1499 1500 static struct caam_aead_alg driver_aeads[] = { 1501 { 1502 .aead = { 1503 .base = { 1504 .cra_name = "rfc4106(gcm(aes))", 1505 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1506 .cra_blocksize = 1, 1507 }, 1508 .setkey = rfc4106_setkey, 1509 .setauthsize = rfc4106_setauthsize, 1510 .encrypt = ipsec_gcm_encrypt, 1511 .decrypt = ipsec_gcm_decrypt, 1512 .ivsize = 8, 1513 .maxauthsize = AES_BLOCK_SIZE, 1514 }, 1515 .caam = { 1516 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1517 }, 1518 }, 1519 { 1520 .aead = { 1521 .base = { 1522 .cra_name = "rfc4543(gcm(aes))", 1523 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1524 .cra_blocksize = 1, 1525 }, 1526 .setkey = rfc4543_setkey, 1527 .setauthsize = rfc4543_setauthsize, 1528 .encrypt = ipsec_gcm_encrypt, 1529 .decrypt = ipsec_gcm_decrypt, 1530 .ivsize = 8, 1531 .maxauthsize = AES_BLOCK_SIZE, 1532 }, 1533 .caam = { 1534 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1535 }, 1536 }, 1537 /* Galois Counter Mode */ 1538 { 1539 .aead = { 1540 .base = { 1541 .cra_name = "gcm(aes)", 1542 .cra_driver_name = "gcm-aes-caam-qi2", 1543 .cra_blocksize = 1, 1544 }, 1545 .setkey = gcm_setkey, 1546 .setauthsize = gcm_setauthsize, 1547 .encrypt = aead_encrypt, 1548 .decrypt = aead_decrypt, 1549 .ivsize = 12, 1550 .maxauthsize = AES_BLOCK_SIZE, 1551 }, 1552 .caam = { 1553 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1554 } 1555 }, 1556 /* single-pass ipsec_esp descriptor */ 1557 { 1558 .aead = { 1559 .base = { 1560 .cra_name = "authenc(hmac(md5),cbc(aes))", 1561 .cra_driver_name = "authenc-hmac-md5-" 1562 "cbc-aes-caam-qi2", 1563 .cra_blocksize = AES_BLOCK_SIZE, 1564 }, 1565 .setkey = aead_setkey, 1566 .setauthsize = aead_setauthsize, 1567 .encrypt = aead_encrypt, 1568 .decrypt = aead_decrypt, 1569 .ivsize = AES_BLOCK_SIZE, 1570 .maxauthsize = MD5_DIGEST_SIZE, 1571 }, 1572 .caam = { 1573 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1574 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1575 OP_ALG_AAI_HMAC_PRECOMP, 1576 } 1577 }, 1578 { 1579 .aead = { 1580 .base = { 1581 .cra_name = "echainiv(authenc(hmac(md5)," 1582 "cbc(aes)))", 1583 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1584 "cbc-aes-caam-qi2", 1585 .cra_blocksize = AES_BLOCK_SIZE, 1586 }, 1587 .setkey = aead_setkey, 1588 .setauthsize = aead_setauthsize, 1589 .encrypt = aead_encrypt, 1590 .decrypt = aead_decrypt, 1591 .ivsize = AES_BLOCK_SIZE, 1592 .maxauthsize = MD5_DIGEST_SIZE, 1593 }, 1594 .caam = { 1595 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1596 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1597 OP_ALG_AAI_HMAC_PRECOMP, 1598 .geniv = true, 1599 } 1600 }, 1601 { 1602 .aead = { 1603 .base = { 1604 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1605 .cra_driver_name = "authenc-hmac-sha1-" 1606 "cbc-aes-caam-qi2", 1607 .cra_blocksize = AES_BLOCK_SIZE, 1608 }, 1609 .setkey = aead_setkey, 1610 .setauthsize = aead_setauthsize, 1611 .encrypt = aead_encrypt, 1612 .decrypt = aead_decrypt, 1613 .ivsize = AES_BLOCK_SIZE, 1614 .maxauthsize = SHA1_DIGEST_SIZE, 1615 }, 1616 .caam = { 1617 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1618 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1619 OP_ALG_AAI_HMAC_PRECOMP, 1620 } 1621 }, 1622 { 1623 .aead = { 1624 .base = { 1625 .cra_name = "echainiv(authenc(hmac(sha1)," 1626 "cbc(aes)))", 1627 .cra_driver_name = "echainiv-authenc-" 1628 "hmac-sha1-cbc-aes-caam-qi2", 1629 .cra_blocksize = AES_BLOCK_SIZE, 1630 }, 1631 .setkey = aead_setkey, 1632 .setauthsize = aead_setauthsize, 1633 .encrypt = aead_encrypt, 1634 .decrypt = aead_decrypt, 1635 .ivsize = AES_BLOCK_SIZE, 1636 .maxauthsize = SHA1_DIGEST_SIZE, 1637 }, 1638 .caam = { 1639 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1640 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1641 OP_ALG_AAI_HMAC_PRECOMP, 1642 .geniv = true, 1643 }, 1644 }, 1645 { 1646 .aead = { 1647 .base = { 1648 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1649 .cra_driver_name = "authenc-hmac-sha224-" 1650 "cbc-aes-caam-qi2", 1651 .cra_blocksize = AES_BLOCK_SIZE, 1652 }, 1653 .setkey = aead_setkey, 1654 .setauthsize = aead_setauthsize, 1655 .encrypt = aead_encrypt, 1656 .decrypt = aead_decrypt, 1657 .ivsize = AES_BLOCK_SIZE, 1658 .maxauthsize = SHA224_DIGEST_SIZE, 1659 }, 1660 .caam = { 1661 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1662 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1663 OP_ALG_AAI_HMAC_PRECOMP, 1664 } 1665 }, 1666 { 1667 .aead = { 1668 .base = { 1669 .cra_name = "echainiv(authenc(hmac(sha224)," 1670 "cbc(aes)))", 1671 .cra_driver_name = "echainiv-authenc-" 1672 "hmac-sha224-cbc-aes-caam-qi2", 1673 .cra_blocksize = AES_BLOCK_SIZE, 1674 }, 1675 .setkey = aead_setkey, 1676 .setauthsize = aead_setauthsize, 1677 .encrypt = aead_encrypt, 1678 .decrypt = aead_decrypt, 1679 .ivsize = AES_BLOCK_SIZE, 1680 .maxauthsize = SHA224_DIGEST_SIZE, 1681 }, 1682 .caam = { 1683 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1684 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1685 OP_ALG_AAI_HMAC_PRECOMP, 1686 .geniv = true, 1687 } 1688 }, 1689 { 1690 .aead = { 1691 .base = { 1692 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1693 .cra_driver_name = "authenc-hmac-sha256-" 1694 "cbc-aes-caam-qi2", 1695 .cra_blocksize = AES_BLOCK_SIZE, 1696 }, 1697 .setkey = aead_setkey, 1698 .setauthsize = aead_setauthsize, 1699 .encrypt = aead_encrypt, 1700 .decrypt = aead_decrypt, 1701 .ivsize = AES_BLOCK_SIZE, 1702 .maxauthsize = SHA256_DIGEST_SIZE, 1703 }, 1704 .caam = { 1705 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1706 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1707 OP_ALG_AAI_HMAC_PRECOMP, 1708 } 1709 }, 1710 { 1711 .aead = { 1712 .base = { 1713 .cra_name = "echainiv(authenc(hmac(sha256)," 1714 "cbc(aes)))", 1715 .cra_driver_name = "echainiv-authenc-" 1716 "hmac-sha256-cbc-aes-" 1717 "caam-qi2", 1718 .cra_blocksize = AES_BLOCK_SIZE, 1719 }, 1720 .setkey = aead_setkey, 1721 .setauthsize = aead_setauthsize, 1722 .encrypt = aead_encrypt, 1723 .decrypt = aead_decrypt, 1724 .ivsize = AES_BLOCK_SIZE, 1725 .maxauthsize = SHA256_DIGEST_SIZE, 1726 }, 1727 .caam = { 1728 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1729 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1730 OP_ALG_AAI_HMAC_PRECOMP, 1731 .geniv = true, 1732 } 1733 }, 1734 { 1735 .aead = { 1736 .base = { 1737 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1738 .cra_driver_name = "authenc-hmac-sha384-" 1739 "cbc-aes-caam-qi2", 1740 .cra_blocksize = AES_BLOCK_SIZE, 1741 }, 1742 .setkey = aead_setkey, 1743 .setauthsize = aead_setauthsize, 1744 .encrypt = aead_encrypt, 1745 .decrypt = aead_decrypt, 1746 .ivsize = AES_BLOCK_SIZE, 1747 .maxauthsize = SHA384_DIGEST_SIZE, 1748 }, 1749 .caam = { 1750 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1751 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1752 OP_ALG_AAI_HMAC_PRECOMP, 1753 } 1754 }, 1755 { 1756 .aead = { 1757 .base = { 1758 .cra_name = "echainiv(authenc(hmac(sha384)," 1759 "cbc(aes)))", 1760 .cra_driver_name = "echainiv-authenc-" 1761 "hmac-sha384-cbc-aes-" 1762 "caam-qi2", 1763 .cra_blocksize = AES_BLOCK_SIZE, 1764 }, 1765 .setkey = aead_setkey, 1766 .setauthsize = aead_setauthsize, 1767 .encrypt = aead_encrypt, 1768 .decrypt = aead_decrypt, 1769 .ivsize = AES_BLOCK_SIZE, 1770 .maxauthsize = SHA384_DIGEST_SIZE, 1771 }, 1772 .caam = { 1773 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1774 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1775 OP_ALG_AAI_HMAC_PRECOMP, 1776 .geniv = true, 1777 } 1778 }, 1779 { 1780 .aead = { 1781 .base = { 1782 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1783 .cra_driver_name = "authenc-hmac-sha512-" 1784 "cbc-aes-caam-qi2", 1785 .cra_blocksize = AES_BLOCK_SIZE, 1786 }, 1787 .setkey = aead_setkey, 1788 .setauthsize = aead_setauthsize, 1789 .encrypt = aead_encrypt, 1790 .decrypt = aead_decrypt, 1791 .ivsize = AES_BLOCK_SIZE, 1792 .maxauthsize = SHA512_DIGEST_SIZE, 1793 }, 1794 .caam = { 1795 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1796 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1797 OP_ALG_AAI_HMAC_PRECOMP, 1798 } 1799 }, 1800 { 1801 .aead = { 1802 .base = { 1803 .cra_name = "echainiv(authenc(hmac(sha512)," 1804 "cbc(aes)))", 1805 .cra_driver_name = "echainiv-authenc-" 1806 "hmac-sha512-cbc-aes-" 1807 "caam-qi2", 1808 .cra_blocksize = AES_BLOCK_SIZE, 1809 }, 1810 .setkey = aead_setkey, 1811 .setauthsize = aead_setauthsize, 1812 .encrypt = aead_encrypt, 1813 .decrypt = aead_decrypt, 1814 .ivsize = AES_BLOCK_SIZE, 1815 .maxauthsize = SHA512_DIGEST_SIZE, 1816 }, 1817 .caam = { 1818 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1819 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1820 OP_ALG_AAI_HMAC_PRECOMP, 1821 .geniv = true, 1822 } 1823 }, 1824 { 1825 .aead = { 1826 .base = { 1827 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1828 .cra_driver_name = "authenc-hmac-md5-" 1829 "cbc-des3_ede-caam-qi2", 1830 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1831 }, 1832 .setkey = aead_setkey, 1833 .setauthsize = aead_setauthsize, 1834 .encrypt = aead_encrypt, 1835 .decrypt = aead_decrypt, 1836 .ivsize = DES3_EDE_BLOCK_SIZE, 1837 .maxauthsize = MD5_DIGEST_SIZE, 1838 }, 1839 .caam = { 1840 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1841 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1842 OP_ALG_AAI_HMAC_PRECOMP, 1843 } 1844 }, 1845 { 1846 .aead = { 1847 .base = { 1848 .cra_name = "echainiv(authenc(hmac(md5)," 1849 "cbc(des3_ede)))", 1850 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1851 "cbc-des3_ede-caam-qi2", 1852 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1853 }, 1854 .setkey = aead_setkey, 1855 .setauthsize = aead_setauthsize, 1856 .encrypt = aead_encrypt, 1857 .decrypt = aead_decrypt, 1858 .ivsize = DES3_EDE_BLOCK_SIZE, 1859 .maxauthsize = MD5_DIGEST_SIZE, 1860 }, 1861 .caam = { 1862 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1863 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1864 OP_ALG_AAI_HMAC_PRECOMP, 1865 .geniv = true, 1866 } 1867 }, 1868 { 1869 .aead = { 1870 .base = { 1871 .cra_name = "authenc(hmac(sha1)," 1872 "cbc(des3_ede))", 1873 .cra_driver_name = "authenc-hmac-sha1-" 1874 "cbc-des3_ede-caam-qi2", 1875 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1876 }, 1877 .setkey = aead_setkey, 1878 .setauthsize = aead_setauthsize, 1879 .encrypt = aead_encrypt, 1880 .decrypt = aead_decrypt, 1881 .ivsize = DES3_EDE_BLOCK_SIZE, 1882 .maxauthsize = SHA1_DIGEST_SIZE, 1883 }, 1884 .caam = { 1885 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1886 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1887 OP_ALG_AAI_HMAC_PRECOMP, 1888 }, 1889 }, 1890 { 1891 .aead = { 1892 .base = { 1893 .cra_name = "echainiv(authenc(hmac(sha1)," 1894 "cbc(des3_ede)))", 1895 .cra_driver_name = "echainiv-authenc-" 1896 "hmac-sha1-" 1897 "cbc-des3_ede-caam-qi2", 1898 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1899 }, 1900 .setkey = aead_setkey, 1901 .setauthsize = aead_setauthsize, 1902 .encrypt = aead_encrypt, 1903 .decrypt = aead_decrypt, 1904 .ivsize = DES3_EDE_BLOCK_SIZE, 1905 .maxauthsize = SHA1_DIGEST_SIZE, 1906 }, 1907 .caam = { 1908 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1909 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1910 OP_ALG_AAI_HMAC_PRECOMP, 1911 .geniv = true, 1912 } 1913 }, 1914 { 1915 .aead = { 1916 .base = { 1917 .cra_name = "authenc(hmac(sha224)," 1918 "cbc(des3_ede))", 1919 .cra_driver_name = "authenc-hmac-sha224-" 1920 "cbc-des3_ede-caam-qi2", 1921 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1922 }, 1923 .setkey = aead_setkey, 1924 .setauthsize = aead_setauthsize, 1925 .encrypt = aead_encrypt, 1926 .decrypt = aead_decrypt, 1927 .ivsize = DES3_EDE_BLOCK_SIZE, 1928 .maxauthsize = SHA224_DIGEST_SIZE, 1929 }, 1930 .caam = { 1931 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1932 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1933 OP_ALG_AAI_HMAC_PRECOMP, 1934 }, 1935 }, 1936 { 1937 .aead = { 1938 .base = { 1939 .cra_name = "echainiv(authenc(hmac(sha224)," 1940 "cbc(des3_ede)))", 1941 .cra_driver_name = "echainiv-authenc-" 1942 "hmac-sha224-" 1943 "cbc-des3_ede-caam-qi2", 1944 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1945 }, 1946 .setkey = aead_setkey, 1947 .setauthsize = aead_setauthsize, 1948 .encrypt = aead_encrypt, 1949 .decrypt = aead_decrypt, 1950 .ivsize = DES3_EDE_BLOCK_SIZE, 1951 .maxauthsize = SHA224_DIGEST_SIZE, 1952 }, 1953 .caam = { 1954 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1955 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1956 OP_ALG_AAI_HMAC_PRECOMP, 1957 .geniv = true, 1958 } 1959 }, 1960 { 1961 .aead = { 1962 .base = { 1963 .cra_name = "authenc(hmac(sha256)," 1964 "cbc(des3_ede))", 1965 .cra_driver_name = "authenc-hmac-sha256-" 1966 "cbc-des3_ede-caam-qi2", 1967 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1968 }, 1969 .setkey = aead_setkey, 1970 .setauthsize = aead_setauthsize, 1971 .encrypt = aead_encrypt, 1972 .decrypt = aead_decrypt, 1973 .ivsize = DES3_EDE_BLOCK_SIZE, 1974 .maxauthsize = SHA256_DIGEST_SIZE, 1975 }, 1976 .caam = { 1977 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1978 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1979 OP_ALG_AAI_HMAC_PRECOMP, 1980 }, 1981 }, 1982 { 1983 .aead = { 1984 .base = { 1985 .cra_name = "echainiv(authenc(hmac(sha256)," 1986 "cbc(des3_ede)))", 1987 .cra_driver_name = "echainiv-authenc-" 1988 "hmac-sha256-" 1989 "cbc-des3_ede-caam-qi2", 1990 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1991 }, 1992 .setkey = aead_setkey, 1993 .setauthsize = aead_setauthsize, 1994 .encrypt = aead_encrypt, 1995 .decrypt = aead_decrypt, 1996 .ivsize = DES3_EDE_BLOCK_SIZE, 1997 .maxauthsize = SHA256_DIGEST_SIZE, 1998 }, 1999 .caam = { 2000 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2001 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2002 OP_ALG_AAI_HMAC_PRECOMP, 2003 .geniv = true, 2004 } 2005 }, 2006 { 2007 .aead = { 2008 .base = { 2009 .cra_name = "authenc(hmac(sha384)," 2010 "cbc(des3_ede))", 2011 .cra_driver_name = "authenc-hmac-sha384-" 2012 "cbc-des3_ede-caam-qi2", 2013 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2014 }, 2015 .setkey = aead_setkey, 2016 .setauthsize = aead_setauthsize, 2017 .encrypt = aead_encrypt, 2018 .decrypt = aead_decrypt, 2019 .ivsize = DES3_EDE_BLOCK_SIZE, 2020 .maxauthsize = SHA384_DIGEST_SIZE, 2021 }, 2022 .caam = { 2023 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2024 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2025 OP_ALG_AAI_HMAC_PRECOMP, 2026 }, 2027 }, 2028 { 2029 .aead = { 2030 .base = { 2031 .cra_name = "echainiv(authenc(hmac(sha384)," 2032 "cbc(des3_ede)))", 2033 .cra_driver_name = "echainiv-authenc-" 2034 "hmac-sha384-" 2035 "cbc-des3_ede-caam-qi2", 2036 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2037 }, 2038 .setkey = aead_setkey, 2039 .setauthsize = aead_setauthsize, 2040 .encrypt = aead_encrypt, 2041 .decrypt = aead_decrypt, 2042 .ivsize = DES3_EDE_BLOCK_SIZE, 2043 .maxauthsize = SHA384_DIGEST_SIZE, 2044 }, 2045 .caam = { 2046 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2047 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2048 OP_ALG_AAI_HMAC_PRECOMP, 2049 .geniv = true, 2050 } 2051 }, 2052 { 2053 .aead = { 2054 .base = { 2055 .cra_name = "authenc(hmac(sha512)," 2056 "cbc(des3_ede))", 2057 .cra_driver_name = "authenc-hmac-sha512-" 2058 "cbc-des3_ede-caam-qi2", 2059 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2060 }, 2061 .setkey = aead_setkey, 2062 .setauthsize = aead_setauthsize, 2063 .encrypt = aead_encrypt, 2064 .decrypt = aead_decrypt, 2065 .ivsize = DES3_EDE_BLOCK_SIZE, 2066 .maxauthsize = SHA512_DIGEST_SIZE, 2067 }, 2068 .caam = { 2069 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2070 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2071 OP_ALG_AAI_HMAC_PRECOMP, 2072 }, 2073 }, 2074 { 2075 .aead = { 2076 .base = { 2077 .cra_name = "echainiv(authenc(hmac(sha512)," 2078 "cbc(des3_ede)))", 2079 .cra_driver_name = "echainiv-authenc-" 2080 "hmac-sha512-" 2081 "cbc-des3_ede-caam-qi2", 2082 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2083 }, 2084 .setkey = aead_setkey, 2085 .setauthsize = aead_setauthsize, 2086 .encrypt = aead_encrypt, 2087 .decrypt = aead_decrypt, 2088 .ivsize = DES3_EDE_BLOCK_SIZE, 2089 .maxauthsize = SHA512_DIGEST_SIZE, 2090 }, 2091 .caam = { 2092 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2093 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2094 OP_ALG_AAI_HMAC_PRECOMP, 2095 .geniv = true, 2096 } 2097 }, 2098 { 2099 .aead = { 2100 .base = { 2101 .cra_name = "authenc(hmac(md5),cbc(des))", 2102 .cra_driver_name = "authenc-hmac-md5-" 2103 "cbc-des-caam-qi2", 2104 .cra_blocksize = DES_BLOCK_SIZE, 2105 }, 2106 .setkey = aead_setkey, 2107 .setauthsize = aead_setauthsize, 2108 .encrypt = aead_encrypt, 2109 .decrypt = aead_decrypt, 2110 .ivsize = DES_BLOCK_SIZE, 2111 .maxauthsize = MD5_DIGEST_SIZE, 2112 }, 2113 .caam = { 2114 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2115 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2116 OP_ALG_AAI_HMAC_PRECOMP, 2117 }, 2118 }, 2119 { 2120 .aead = { 2121 .base = { 2122 .cra_name = "echainiv(authenc(hmac(md5)," 2123 "cbc(des)))", 2124 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2125 "cbc-des-caam-qi2", 2126 .cra_blocksize = DES_BLOCK_SIZE, 2127 }, 2128 .setkey = aead_setkey, 2129 .setauthsize = aead_setauthsize, 2130 .encrypt = aead_encrypt, 2131 .decrypt = aead_decrypt, 2132 .ivsize = DES_BLOCK_SIZE, 2133 .maxauthsize = MD5_DIGEST_SIZE, 2134 }, 2135 .caam = { 2136 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2137 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2138 OP_ALG_AAI_HMAC_PRECOMP, 2139 .geniv = true, 2140 } 2141 }, 2142 { 2143 .aead = { 2144 .base = { 2145 .cra_name = "authenc(hmac(sha1),cbc(des))", 2146 .cra_driver_name = "authenc-hmac-sha1-" 2147 "cbc-des-caam-qi2", 2148 .cra_blocksize = DES_BLOCK_SIZE, 2149 }, 2150 .setkey = aead_setkey, 2151 .setauthsize = aead_setauthsize, 2152 .encrypt = aead_encrypt, 2153 .decrypt = aead_decrypt, 2154 .ivsize = DES_BLOCK_SIZE, 2155 .maxauthsize = SHA1_DIGEST_SIZE, 2156 }, 2157 .caam = { 2158 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2159 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2160 OP_ALG_AAI_HMAC_PRECOMP, 2161 }, 2162 }, 2163 { 2164 .aead = { 2165 .base = { 2166 .cra_name = "echainiv(authenc(hmac(sha1)," 2167 "cbc(des)))", 2168 .cra_driver_name = "echainiv-authenc-" 2169 "hmac-sha1-cbc-des-caam-qi2", 2170 .cra_blocksize = DES_BLOCK_SIZE, 2171 }, 2172 .setkey = aead_setkey, 2173 .setauthsize = aead_setauthsize, 2174 .encrypt = aead_encrypt, 2175 .decrypt = aead_decrypt, 2176 .ivsize = DES_BLOCK_SIZE, 2177 .maxauthsize = SHA1_DIGEST_SIZE, 2178 }, 2179 .caam = { 2180 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2181 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2182 OP_ALG_AAI_HMAC_PRECOMP, 2183 .geniv = true, 2184 } 2185 }, 2186 { 2187 .aead = { 2188 .base = { 2189 .cra_name = "authenc(hmac(sha224),cbc(des))", 2190 .cra_driver_name = "authenc-hmac-sha224-" 2191 "cbc-des-caam-qi2", 2192 .cra_blocksize = DES_BLOCK_SIZE, 2193 }, 2194 .setkey = aead_setkey, 2195 .setauthsize = aead_setauthsize, 2196 .encrypt = aead_encrypt, 2197 .decrypt = aead_decrypt, 2198 .ivsize = DES_BLOCK_SIZE, 2199 .maxauthsize = SHA224_DIGEST_SIZE, 2200 }, 2201 .caam = { 2202 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2203 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2204 OP_ALG_AAI_HMAC_PRECOMP, 2205 }, 2206 }, 2207 { 2208 .aead = { 2209 .base = { 2210 .cra_name = "echainiv(authenc(hmac(sha224)," 2211 "cbc(des)))", 2212 .cra_driver_name = "echainiv-authenc-" 2213 "hmac-sha224-cbc-des-" 2214 "caam-qi2", 2215 .cra_blocksize = DES_BLOCK_SIZE, 2216 }, 2217 .setkey = aead_setkey, 2218 .setauthsize = aead_setauthsize, 2219 .encrypt = aead_encrypt, 2220 .decrypt = aead_decrypt, 2221 .ivsize = DES_BLOCK_SIZE, 2222 .maxauthsize = SHA224_DIGEST_SIZE, 2223 }, 2224 .caam = { 2225 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2226 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2227 OP_ALG_AAI_HMAC_PRECOMP, 2228 .geniv = true, 2229 } 2230 }, 2231 { 2232 .aead = { 2233 .base = { 2234 .cra_name = "authenc(hmac(sha256),cbc(des))", 2235 .cra_driver_name = "authenc-hmac-sha256-" 2236 "cbc-des-caam-qi2", 2237 .cra_blocksize = DES_BLOCK_SIZE, 2238 }, 2239 .setkey = aead_setkey, 2240 .setauthsize = aead_setauthsize, 2241 .encrypt = aead_encrypt, 2242 .decrypt = aead_decrypt, 2243 .ivsize = DES_BLOCK_SIZE, 2244 .maxauthsize = SHA256_DIGEST_SIZE, 2245 }, 2246 .caam = { 2247 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2248 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2249 OP_ALG_AAI_HMAC_PRECOMP, 2250 }, 2251 }, 2252 { 2253 .aead = { 2254 .base = { 2255 .cra_name = "echainiv(authenc(hmac(sha256)," 2256 "cbc(des)))", 2257 .cra_driver_name = "echainiv-authenc-" 2258 "hmac-sha256-cbc-desi-" 2259 "caam-qi2", 2260 .cra_blocksize = DES_BLOCK_SIZE, 2261 }, 2262 .setkey = aead_setkey, 2263 .setauthsize = aead_setauthsize, 2264 .encrypt = aead_encrypt, 2265 .decrypt = aead_decrypt, 2266 .ivsize = DES_BLOCK_SIZE, 2267 .maxauthsize = SHA256_DIGEST_SIZE, 2268 }, 2269 .caam = { 2270 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2271 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2272 OP_ALG_AAI_HMAC_PRECOMP, 2273 .geniv = true, 2274 }, 2275 }, 2276 { 2277 .aead = { 2278 .base = { 2279 .cra_name = "authenc(hmac(sha384),cbc(des))", 2280 .cra_driver_name = "authenc-hmac-sha384-" 2281 "cbc-des-caam-qi2", 2282 .cra_blocksize = DES_BLOCK_SIZE, 2283 }, 2284 .setkey = aead_setkey, 2285 .setauthsize = aead_setauthsize, 2286 .encrypt = aead_encrypt, 2287 .decrypt = aead_decrypt, 2288 .ivsize = DES_BLOCK_SIZE, 2289 .maxauthsize = SHA384_DIGEST_SIZE, 2290 }, 2291 .caam = { 2292 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2293 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2294 OP_ALG_AAI_HMAC_PRECOMP, 2295 }, 2296 }, 2297 { 2298 .aead = { 2299 .base = { 2300 .cra_name = "echainiv(authenc(hmac(sha384)," 2301 "cbc(des)))", 2302 .cra_driver_name = "echainiv-authenc-" 2303 "hmac-sha384-cbc-des-" 2304 "caam-qi2", 2305 .cra_blocksize = DES_BLOCK_SIZE, 2306 }, 2307 .setkey = aead_setkey, 2308 .setauthsize = aead_setauthsize, 2309 .encrypt = aead_encrypt, 2310 .decrypt = aead_decrypt, 2311 .ivsize = DES_BLOCK_SIZE, 2312 .maxauthsize = SHA384_DIGEST_SIZE, 2313 }, 2314 .caam = { 2315 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2316 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2317 OP_ALG_AAI_HMAC_PRECOMP, 2318 .geniv = true, 2319 } 2320 }, 2321 { 2322 .aead = { 2323 .base = { 2324 .cra_name = "authenc(hmac(sha512),cbc(des))", 2325 .cra_driver_name = "authenc-hmac-sha512-" 2326 "cbc-des-caam-qi2", 2327 .cra_blocksize = DES_BLOCK_SIZE, 2328 }, 2329 .setkey = aead_setkey, 2330 .setauthsize = aead_setauthsize, 2331 .encrypt = aead_encrypt, 2332 .decrypt = aead_decrypt, 2333 .ivsize = DES_BLOCK_SIZE, 2334 .maxauthsize = SHA512_DIGEST_SIZE, 2335 }, 2336 .caam = { 2337 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2338 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2339 OP_ALG_AAI_HMAC_PRECOMP, 2340 } 2341 }, 2342 { 2343 .aead = { 2344 .base = { 2345 .cra_name = "echainiv(authenc(hmac(sha512)," 2346 "cbc(des)))", 2347 .cra_driver_name = "echainiv-authenc-" 2348 "hmac-sha512-cbc-des-" 2349 "caam-qi2", 2350 .cra_blocksize = DES_BLOCK_SIZE, 2351 }, 2352 .setkey = aead_setkey, 2353 .setauthsize = aead_setauthsize, 2354 .encrypt = aead_encrypt, 2355 .decrypt = aead_decrypt, 2356 .ivsize = DES_BLOCK_SIZE, 2357 .maxauthsize = SHA512_DIGEST_SIZE, 2358 }, 2359 .caam = { 2360 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2361 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2362 OP_ALG_AAI_HMAC_PRECOMP, 2363 .geniv = true, 2364 } 2365 }, 2366 { 2367 .aead = { 2368 .base = { 2369 .cra_name = "authenc(hmac(md5)," 2370 "rfc3686(ctr(aes)))", 2371 .cra_driver_name = "authenc-hmac-md5-" 2372 "rfc3686-ctr-aes-caam-qi2", 2373 .cra_blocksize = 1, 2374 }, 2375 .setkey = aead_setkey, 2376 .setauthsize = aead_setauthsize, 2377 .encrypt = aead_encrypt, 2378 .decrypt = aead_decrypt, 2379 .ivsize = CTR_RFC3686_IV_SIZE, 2380 .maxauthsize = MD5_DIGEST_SIZE, 2381 }, 2382 .caam = { 2383 .class1_alg_type = OP_ALG_ALGSEL_AES | 2384 OP_ALG_AAI_CTR_MOD128, 2385 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2386 OP_ALG_AAI_HMAC_PRECOMP, 2387 .rfc3686 = true, 2388 }, 2389 }, 2390 { 2391 .aead = { 2392 .base = { 2393 .cra_name = "seqiv(authenc(" 2394 "hmac(md5),rfc3686(ctr(aes))))", 2395 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2396 "rfc3686-ctr-aes-caam-qi2", 2397 .cra_blocksize = 1, 2398 }, 2399 .setkey = aead_setkey, 2400 .setauthsize = aead_setauthsize, 2401 .encrypt = aead_encrypt, 2402 .decrypt = aead_decrypt, 2403 .ivsize = CTR_RFC3686_IV_SIZE, 2404 .maxauthsize = MD5_DIGEST_SIZE, 2405 }, 2406 .caam = { 2407 .class1_alg_type = OP_ALG_ALGSEL_AES | 2408 OP_ALG_AAI_CTR_MOD128, 2409 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2410 OP_ALG_AAI_HMAC_PRECOMP, 2411 .rfc3686 = true, 2412 .geniv = true, 2413 }, 2414 }, 2415 { 2416 .aead = { 2417 .base = { 2418 .cra_name = "authenc(hmac(sha1)," 2419 "rfc3686(ctr(aes)))", 2420 .cra_driver_name = "authenc-hmac-sha1-" 2421 "rfc3686-ctr-aes-caam-qi2", 2422 .cra_blocksize = 1, 2423 }, 2424 .setkey = aead_setkey, 2425 .setauthsize = aead_setauthsize, 2426 .encrypt = aead_encrypt, 2427 .decrypt = aead_decrypt, 2428 .ivsize = CTR_RFC3686_IV_SIZE, 2429 .maxauthsize = SHA1_DIGEST_SIZE, 2430 }, 2431 .caam = { 2432 .class1_alg_type = OP_ALG_ALGSEL_AES | 2433 OP_ALG_AAI_CTR_MOD128, 2434 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2435 OP_ALG_AAI_HMAC_PRECOMP, 2436 .rfc3686 = true, 2437 }, 2438 }, 2439 { 2440 .aead = { 2441 .base = { 2442 .cra_name = "seqiv(authenc(" 2443 "hmac(sha1),rfc3686(ctr(aes))))", 2444 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2445 "rfc3686-ctr-aes-caam-qi2", 2446 .cra_blocksize = 1, 2447 }, 2448 .setkey = aead_setkey, 2449 .setauthsize = aead_setauthsize, 2450 .encrypt = aead_encrypt, 2451 .decrypt = aead_decrypt, 2452 .ivsize = CTR_RFC3686_IV_SIZE, 2453 .maxauthsize = SHA1_DIGEST_SIZE, 2454 }, 2455 .caam = { 2456 .class1_alg_type = OP_ALG_ALGSEL_AES | 2457 OP_ALG_AAI_CTR_MOD128, 2458 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2459 OP_ALG_AAI_HMAC_PRECOMP, 2460 .rfc3686 = true, 2461 .geniv = true, 2462 }, 2463 }, 2464 { 2465 .aead = { 2466 .base = { 2467 .cra_name = "authenc(hmac(sha224)," 2468 "rfc3686(ctr(aes)))", 2469 .cra_driver_name = "authenc-hmac-sha224-" 2470 "rfc3686-ctr-aes-caam-qi2", 2471 .cra_blocksize = 1, 2472 }, 2473 .setkey = aead_setkey, 2474 .setauthsize = aead_setauthsize, 2475 .encrypt = aead_encrypt, 2476 .decrypt = aead_decrypt, 2477 .ivsize = CTR_RFC3686_IV_SIZE, 2478 .maxauthsize = SHA224_DIGEST_SIZE, 2479 }, 2480 .caam = { 2481 .class1_alg_type = OP_ALG_ALGSEL_AES | 2482 OP_ALG_AAI_CTR_MOD128, 2483 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2484 OP_ALG_AAI_HMAC_PRECOMP, 2485 .rfc3686 = true, 2486 }, 2487 }, 2488 { 2489 .aead = { 2490 .base = { 2491 .cra_name = "seqiv(authenc(" 2492 "hmac(sha224),rfc3686(ctr(aes))))", 2493 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2494 "rfc3686-ctr-aes-caam-qi2", 2495 .cra_blocksize = 1, 2496 }, 2497 .setkey = aead_setkey, 2498 .setauthsize = aead_setauthsize, 2499 .encrypt = aead_encrypt, 2500 .decrypt = aead_decrypt, 2501 .ivsize = CTR_RFC3686_IV_SIZE, 2502 .maxauthsize = SHA224_DIGEST_SIZE, 2503 }, 2504 .caam = { 2505 .class1_alg_type = OP_ALG_ALGSEL_AES | 2506 OP_ALG_AAI_CTR_MOD128, 2507 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2508 OP_ALG_AAI_HMAC_PRECOMP, 2509 .rfc3686 = true, 2510 .geniv = true, 2511 }, 2512 }, 2513 { 2514 .aead = { 2515 .base = { 2516 .cra_name = "authenc(hmac(sha256)," 2517 "rfc3686(ctr(aes)))", 2518 .cra_driver_name = "authenc-hmac-sha256-" 2519 "rfc3686-ctr-aes-caam-qi2", 2520 .cra_blocksize = 1, 2521 }, 2522 .setkey = aead_setkey, 2523 .setauthsize = aead_setauthsize, 2524 .encrypt = aead_encrypt, 2525 .decrypt = aead_decrypt, 2526 .ivsize = CTR_RFC3686_IV_SIZE, 2527 .maxauthsize = SHA256_DIGEST_SIZE, 2528 }, 2529 .caam = { 2530 .class1_alg_type = OP_ALG_ALGSEL_AES | 2531 OP_ALG_AAI_CTR_MOD128, 2532 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2533 OP_ALG_AAI_HMAC_PRECOMP, 2534 .rfc3686 = true, 2535 }, 2536 }, 2537 { 2538 .aead = { 2539 .base = { 2540 .cra_name = "seqiv(authenc(hmac(sha256)," 2541 "rfc3686(ctr(aes))))", 2542 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2543 "rfc3686-ctr-aes-caam-qi2", 2544 .cra_blocksize = 1, 2545 }, 2546 .setkey = aead_setkey, 2547 .setauthsize = aead_setauthsize, 2548 .encrypt = aead_encrypt, 2549 .decrypt = aead_decrypt, 2550 .ivsize = CTR_RFC3686_IV_SIZE, 2551 .maxauthsize = SHA256_DIGEST_SIZE, 2552 }, 2553 .caam = { 2554 .class1_alg_type = OP_ALG_ALGSEL_AES | 2555 OP_ALG_AAI_CTR_MOD128, 2556 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2557 OP_ALG_AAI_HMAC_PRECOMP, 2558 .rfc3686 = true, 2559 .geniv = true, 2560 }, 2561 }, 2562 { 2563 .aead = { 2564 .base = { 2565 .cra_name = "authenc(hmac(sha384)," 2566 "rfc3686(ctr(aes)))", 2567 .cra_driver_name = "authenc-hmac-sha384-" 2568 "rfc3686-ctr-aes-caam-qi2", 2569 .cra_blocksize = 1, 2570 }, 2571 .setkey = aead_setkey, 2572 .setauthsize = aead_setauthsize, 2573 .encrypt = aead_encrypt, 2574 .decrypt = aead_decrypt, 2575 .ivsize = CTR_RFC3686_IV_SIZE, 2576 .maxauthsize = SHA384_DIGEST_SIZE, 2577 }, 2578 .caam = { 2579 .class1_alg_type = OP_ALG_ALGSEL_AES | 2580 OP_ALG_AAI_CTR_MOD128, 2581 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2582 OP_ALG_AAI_HMAC_PRECOMP, 2583 .rfc3686 = true, 2584 }, 2585 }, 2586 { 2587 .aead = { 2588 .base = { 2589 .cra_name = "seqiv(authenc(hmac(sha384)," 2590 "rfc3686(ctr(aes))))", 2591 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2592 "rfc3686-ctr-aes-caam-qi2", 2593 .cra_blocksize = 1, 2594 }, 2595 .setkey = aead_setkey, 2596 .setauthsize = aead_setauthsize, 2597 .encrypt = aead_encrypt, 2598 .decrypt = aead_decrypt, 2599 .ivsize = CTR_RFC3686_IV_SIZE, 2600 .maxauthsize = SHA384_DIGEST_SIZE, 2601 }, 2602 .caam = { 2603 .class1_alg_type = OP_ALG_ALGSEL_AES | 2604 OP_ALG_AAI_CTR_MOD128, 2605 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2606 OP_ALG_AAI_HMAC_PRECOMP, 2607 .rfc3686 = true, 2608 .geniv = true, 2609 }, 2610 }, 2611 { 2612 .aead = { 2613 .base = { 2614 .cra_name = "authenc(hmac(sha512)," 2615 "rfc3686(ctr(aes)))", 2616 .cra_driver_name = "authenc-hmac-sha512-" 2617 "rfc3686-ctr-aes-caam-qi2", 2618 .cra_blocksize = 1, 2619 }, 2620 .setkey = aead_setkey, 2621 .setauthsize = aead_setauthsize, 2622 .encrypt = aead_encrypt, 2623 .decrypt = aead_decrypt, 2624 .ivsize = CTR_RFC3686_IV_SIZE, 2625 .maxauthsize = SHA512_DIGEST_SIZE, 2626 }, 2627 .caam = { 2628 .class1_alg_type = OP_ALG_ALGSEL_AES | 2629 OP_ALG_AAI_CTR_MOD128, 2630 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2631 OP_ALG_AAI_HMAC_PRECOMP, 2632 .rfc3686 = true, 2633 }, 2634 }, 2635 { 2636 .aead = { 2637 .base = { 2638 .cra_name = "seqiv(authenc(hmac(sha512)," 2639 "rfc3686(ctr(aes))))", 2640 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2641 "rfc3686-ctr-aes-caam-qi2", 2642 .cra_blocksize = 1, 2643 }, 2644 .setkey = aead_setkey, 2645 .setauthsize = aead_setauthsize, 2646 .encrypt = aead_encrypt, 2647 .decrypt = aead_decrypt, 2648 .ivsize = CTR_RFC3686_IV_SIZE, 2649 .maxauthsize = SHA512_DIGEST_SIZE, 2650 }, 2651 .caam = { 2652 .class1_alg_type = OP_ALG_ALGSEL_AES | 2653 OP_ALG_AAI_CTR_MOD128, 2654 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2655 OP_ALG_AAI_HMAC_PRECOMP, 2656 .rfc3686 = true, 2657 .geniv = true, 2658 }, 2659 }, 2660 }; 2661 2662 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2663 { 2664 struct skcipher_alg *alg = &t_alg->skcipher; 2665 2666 alg->base.cra_module = THIS_MODULE; 2667 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2668 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2669 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2670 2671 alg->init = caam_cra_init_skcipher; 2672 alg->exit = caam_cra_exit; 2673 } 2674 2675 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2676 { 2677 struct aead_alg *alg = &t_alg->aead; 2678 2679 alg->base.cra_module = THIS_MODULE; 2680 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2681 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2682 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2683 2684 alg->init = caam_cra_init_aead; 2685 alg->exit = caam_cra_exit_aead; 2686 } 2687 2688 /* max hash key is max split key size */ 2689 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 2690 2691 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 2692 2693 /* caam context sizes for hashes: running digest + 8 */ 2694 #define HASH_MSG_LEN 8 2695 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 2696 2697 enum hash_optype { 2698 UPDATE = 0, 2699 UPDATE_FIRST, 2700 FINALIZE, 2701 DIGEST, 2702 HASH_NUM_OP 2703 }; 2704 2705 /** 2706 * caam_hash_ctx - ahash per-session context 2707 * @flc: Flow Contexts array 2708 * @flc_dma: I/O virtual addresses of the Flow Contexts 2709 * @dev: dpseci device 2710 * @ctx_len: size of Context Register 2711 * @adata: hashing algorithm details 2712 */ 2713 struct caam_hash_ctx { 2714 struct caam_flc flc[HASH_NUM_OP]; 2715 dma_addr_t flc_dma[HASH_NUM_OP]; 2716 struct device *dev; 2717 int ctx_len; 2718 struct alginfo adata; 2719 }; 2720 2721 /* ahash state */ 2722 struct caam_hash_state { 2723 struct caam_request caam_req; 2724 dma_addr_t buf_dma; 2725 dma_addr_t ctx_dma; 2726 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2727 int buflen_0; 2728 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2729 int buflen_1; 2730 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 2731 int (*update)(struct ahash_request *req); 2732 int (*final)(struct ahash_request *req); 2733 int (*finup)(struct ahash_request *req); 2734 int current_buf; 2735 }; 2736 2737 struct caam_export_state { 2738 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 2739 u8 caam_ctx[MAX_CTX_LEN]; 2740 int buflen; 2741 int (*update)(struct ahash_request *req); 2742 int (*final)(struct ahash_request *req); 2743 int (*finup)(struct ahash_request *req); 2744 }; 2745 2746 static inline void switch_buf(struct caam_hash_state *state) 2747 { 2748 state->current_buf ^= 1; 2749 } 2750 2751 static inline u8 *current_buf(struct caam_hash_state *state) 2752 { 2753 return state->current_buf ? state->buf_1 : state->buf_0; 2754 } 2755 2756 static inline u8 *alt_buf(struct caam_hash_state *state) 2757 { 2758 return state->current_buf ? state->buf_0 : state->buf_1; 2759 } 2760 2761 static inline int *current_buflen(struct caam_hash_state *state) 2762 { 2763 return state->current_buf ? &state->buflen_1 : &state->buflen_0; 2764 } 2765 2766 static inline int *alt_buflen(struct caam_hash_state *state) 2767 { 2768 return state->current_buf ? &state->buflen_0 : &state->buflen_1; 2769 } 2770 2771 /* Map current buffer in state (if length > 0) and put it in link table */ 2772 static inline int buf_map_to_qm_sg(struct device *dev, 2773 struct dpaa2_sg_entry *qm_sg, 2774 struct caam_hash_state *state) 2775 { 2776 int buflen = *current_buflen(state); 2777 2778 if (!buflen) 2779 return 0; 2780 2781 state->buf_dma = dma_map_single(dev, current_buf(state), buflen, 2782 DMA_TO_DEVICE); 2783 if (dma_mapping_error(dev, state->buf_dma)) { 2784 dev_err(dev, "unable to map buf\n"); 2785 state->buf_dma = 0; 2786 return -ENOMEM; 2787 } 2788 2789 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 2790 2791 return 0; 2792 } 2793 2794 /* Map state->caam_ctx, and add it to link table */ 2795 static inline int ctx_map_to_qm_sg(struct device *dev, 2796 struct caam_hash_state *state, int ctx_len, 2797 struct dpaa2_sg_entry *qm_sg, u32 flag) 2798 { 2799 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 2800 if (dma_mapping_error(dev, state->ctx_dma)) { 2801 dev_err(dev, "unable to map ctx\n"); 2802 state->ctx_dma = 0; 2803 return -ENOMEM; 2804 } 2805 2806 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 2807 2808 return 0; 2809 } 2810 2811 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 2812 { 2813 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 2814 int digestsize = crypto_ahash_digestsize(ahash); 2815 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 2816 struct caam_flc *flc; 2817 u32 *desc; 2818 2819 /* ahash_update shared descriptor */ 2820 flc = &ctx->flc[UPDATE]; 2821 desc = flc->sh_desc; 2822 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 2823 ctx->ctx_len, true, priv->sec_attr.era); 2824 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 2825 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 2826 desc_bytes(desc), DMA_BIDIRECTIONAL); 2827 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 2828 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 2829 1); 2830 2831 /* ahash_update_first shared descriptor */ 2832 flc = &ctx->flc[UPDATE_FIRST]; 2833 desc = flc->sh_desc; 2834 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 2835 ctx->ctx_len, false, priv->sec_attr.era); 2836 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 2837 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 2838 desc_bytes(desc), DMA_BIDIRECTIONAL); 2839 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 2840 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 2841 1); 2842 2843 /* ahash_final shared descriptor */ 2844 flc = &ctx->flc[FINALIZE]; 2845 desc = flc->sh_desc; 2846 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 2847 ctx->ctx_len, true, priv->sec_attr.era); 2848 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 2849 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 2850 desc_bytes(desc), DMA_BIDIRECTIONAL); 2851 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 2852 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 2853 1); 2854 2855 /* ahash_digest shared descriptor */ 2856 flc = &ctx->flc[DIGEST]; 2857 desc = flc->sh_desc; 2858 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 2859 ctx->ctx_len, false, priv->sec_attr.era); 2860 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 2861 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 2862 desc_bytes(desc), DMA_BIDIRECTIONAL); 2863 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 2864 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 2865 1); 2866 2867 return 0; 2868 } 2869 2870 struct split_key_sh_result { 2871 struct completion completion; 2872 int err; 2873 struct device *dev; 2874 }; 2875 2876 static void split_key_sh_done(void *cbk_ctx, u32 err) 2877 { 2878 struct split_key_sh_result *res = cbk_ctx; 2879 2880 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 2881 2882 if (err) 2883 caam_qi2_strstatus(res->dev, err); 2884 2885 res->err = err; 2886 complete(&res->completion); 2887 } 2888 2889 /* Digest hash size if it is too large */ 2890 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, 2891 u32 *keylen, u8 *key_out, u32 digestsize) 2892 { 2893 struct caam_request *req_ctx; 2894 u32 *desc; 2895 struct split_key_sh_result result; 2896 dma_addr_t src_dma, dst_dma; 2897 struct caam_flc *flc; 2898 dma_addr_t flc_dma; 2899 int ret = -ENOMEM; 2900 struct dpaa2_fl_entry *in_fle, *out_fle; 2901 2902 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); 2903 if (!req_ctx) 2904 return -ENOMEM; 2905 2906 in_fle = &req_ctx->fd_flt[1]; 2907 out_fle = &req_ctx->fd_flt[0]; 2908 2909 flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); 2910 if (!flc) 2911 goto err_flc; 2912 2913 src_dma = dma_map_single(ctx->dev, (void *)key_in, *keylen, 2914 DMA_TO_DEVICE); 2915 if (dma_mapping_error(ctx->dev, src_dma)) { 2916 dev_err(ctx->dev, "unable to map key input memory\n"); 2917 goto err_src_dma; 2918 } 2919 dst_dma = dma_map_single(ctx->dev, (void *)key_out, digestsize, 2920 DMA_FROM_DEVICE); 2921 if (dma_mapping_error(ctx->dev, dst_dma)) { 2922 dev_err(ctx->dev, "unable to map key output memory\n"); 2923 goto err_dst_dma; 2924 } 2925 2926 desc = flc->sh_desc; 2927 2928 init_sh_desc(desc, 0); 2929 2930 /* descriptor to perform unkeyed hash on key_in */ 2931 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 2932 OP_ALG_AS_INITFINAL); 2933 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 2934 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 2935 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 2936 LDST_SRCDST_BYTE_CONTEXT); 2937 2938 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 2939 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 2940 desc_bytes(desc), DMA_TO_DEVICE); 2941 if (dma_mapping_error(ctx->dev, flc_dma)) { 2942 dev_err(ctx->dev, "unable to map shared descriptor\n"); 2943 goto err_flc_dma; 2944 } 2945 2946 dpaa2_fl_set_final(in_fle, true); 2947 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 2948 dpaa2_fl_set_addr(in_fle, src_dma); 2949 dpaa2_fl_set_len(in_fle, *keylen); 2950 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 2951 dpaa2_fl_set_addr(out_fle, dst_dma); 2952 dpaa2_fl_set_len(out_fle, digestsize); 2953 2954 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 2955 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); 2956 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 2957 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 2958 1); 2959 2960 result.err = 0; 2961 init_completion(&result.completion); 2962 result.dev = ctx->dev; 2963 2964 req_ctx->flc = flc; 2965 req_ctx->flc_dma = flc_dma; 2966 req_ctx->cbk = split_key_sh_done; 2967 req_ctx->ctx = &result; 2968 2969 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 2970 if (ret == -EINPROGRESS) { 2971 /* in progress */ 2972 wait_for_completion(&result.completion); 2973 ret = result.err; 2974 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 2975 DUMP_PREFIX_ADDRESS, 16, 4, key_in, 2976 digestsize, 1); 2977 } 2978 2979 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 2980 DMA_TO_DEVICE); 2981 err_flc_dma: 2982 dma_unmap_single(ctx->dev, dst_dma, digestsize, DMA_FROM_DEVICE); 2983 err_dst_dma: 2984 dma_unmap_single(ctx->dev, src_dma, *keylen, DMA_TO_DEVICE); 2985 err_src_dma: 2986 kfree(flc); 2987 err_flc: 2988 kfree(req_ctx); 2989 2990 *keylen = digestsize; 2991 2992 return ret; 2993 } 2994 2995 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 2996 unsigned int keylen) 2997 { 2998 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 2999 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3000 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3001 int ret; 3002 u8 *hashed_key = NULL; 3003 3004 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3005 3006 if (keylen > blocksize) { 3007 hashed_key = kmalloc_array(digestsize, sizeof(*hashed_key), 3008 GFP_KERNEL | GFP_DMA); 3009 if (!hashed_key) 3010 return -ENOMEM; 3011 ret = hash_digest_key(ctx, key, &keylen, hashed_key, 3012 digestsize); 3013 if (ret) 3014 goto bad_free_key; 3015 key = hashed_key; 3016 } 3017 3018 ctx->adata.keylen = keylen; 3019 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3020 OP_ALG_ALGSEL_MASK); 3021 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3022 goto bad_free_key; 3023 3024 ctx->adata.key_virt = key; 3025 ctx->adata.key_inline = true; 3026 3027 ret = ahash_set_sh_desc(ahash); 3028 kfree(hashed_key); 3029 return ret; 3030 bad_free_key: 3031 kfree(hashed_key); 3032 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 3033 return -EINVAL; 3034 } 3035 3036 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3037 struct ahash_request *req, int dst_len) 3038 { 3039 struct caam_hash_state *state = ahash_request_ctx(req); 3040 3041 if (edesc->src_nents) 3042 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3043 if (edesc->dst_dma) 3044 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); 3045 3046 if (edesc->qm_sg_bytes) 3047 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3048 DMA_TO_DEVICE); 3049 3050 if (state->buf_dma) { 3051 dma_unmap_single(dev, state->buf_dma, *current_buflen(state), 3052 DMA_TO_DEVICE); 3053 state->buf_dma = 0; 3054 } 3055 } 3056 3057 static inline void ahash_unmap_ctx(struct device *dev, 3058 struct ahash_edesc *edesc, 3059 struct ahash_request *req, int dst_len, 3060 u32 flag) 3061 { 3062 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3063 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3064 struct caam_hash_state *state = ahash_request_ctx(req); 3065 3066 if (state->ctx_dma) { 3067 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); 3068 state->ctx_dma = 0; 3069 } 3070 ahash_unmap(dev, edesc, req, dst_len); 3071 } 3072 3073 static void ahash_done(void *cbk_ctx, u32 status) 3074 { 3075 struct crypto_async_request *areq = cbk_ctx; 3076 struct ahash_request *req = ahash_request_cast(areq); 3077 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3078 struct caam_hash_state *state = ahash_request_ctx(req); 3079 struct ahash_edesc *edesc = state->caam_req.edesc; 3080 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3081 int digestsize = crypto_ahash_digestsize(ahash); 3082 int ecode = 0; 3083 3084 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3085 3086 if (unlikely(status)) { 3087 caam_qi2_strstatus(ctx->dev, status); 3088 ecode = -EIO; 3089 } 3090 3091 ahash_unmap(ctx->dev, edesc, req, digestsize); 3092 qi_cache_free(edesc); 3093 3094 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3095 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3096 ctx->ctx_len, 1); 3097 if (req->result) 3098 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3099 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3100 digestsize, 1); 3101 3102 req->base.complete(&req->base, ecode); 3103 } 3104 3105 static void ahash_done_bi(void *cbk_ctx, u32 status) 3106 { 3107 struct crypto_async_request *areq = cbk_ctx; 3108 struct ahash_request *req = ahash_request_cast(areq); 3109 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3110 struct caam_hash_state *state = ahash_request_ctx(req); 3111 struct ahash_edesc *edesc = state->caam_req.edesc; 3112 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3113 int ecode = 0; 3114 3115 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3116 3117 if (unlikely(status)) { 3118 caam_qi2_strstatus(ctx->dev, status); 3119 ecode = -EIO; 3120 } 3121 3122 ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 3123 switch_buf(state); 3124 qi_cache_free(edesc); 3125 3126 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3127 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3128 ctx->ctx_len, 1); 3129 if (req->result) 3130 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3131 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3132 crypto_ahash_digestsize(ahash), 1); 3133 3134 req->base.complete(&req->base, ecode); 3135 } 3136 3137 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3138 { 3139 struct crypto_async_request *areq = cbk_ctx; 3140 struct ahash_request *req = ahash_request_cast(areq); 3141 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3142 struct caam_hash_state *state = ahash_request_ctx(req); 3143 struct ahash_edesc *edesc = state->caam_req.edesc; 3144 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3145 int digestsize = crypto_ahash_digestsize(ahash); 3146 int ecode = 0; 3147 3148 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3149 3150 if (unlikely(status)) { 3151 caam_qi2_strstatus(ctx->dev, status); 3152 ecode = -EIO; 3153 } 3154 3155 ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_TO_DEVICE); 3156 qi_cache_free(edesc); 3157 3158 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3159 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3160 ctx->ctx_len, 1); 3161 if (req->result) 3162 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3163 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3164 digestsize, 1); 3165 3166 req->base.complete(&req->base, ecode); 3167 } 3168 3169 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3170 { 3171 struct crypto_async_request *areq = cbk_ctx; 3172 struct ahash_request *req = ahash_request_cast(areq); 3173 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3174 struct caam_hash_state *state = ahash_request_ctx(req); 3175 struct ahash_edesc *edesc = state->caam_req.edesc; 3176 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3177 int ecode = 0; 3178 3179 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3180 3181 if (unlikely(status)) { 3182 caam_qi2_strstatus(ctx->dev, status); 3183 ecode = -EIO; 3184 } 3185 3186 ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 3187 switch_buf(state); 3188 qi_cache_free(edesc); 3189 3190 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3191 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3192 ctx->ctx_len, 1); 3193 if (req->result) 3194 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3195 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3196 crypto_ahash_digestsize(ahash), 1); 3197 3198 req->base.complete(&req->base, ecode); 3199 } 3200 3201 static int ahash_update_ctx(struct ahash_request *req) 3202 { 3203 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3204 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3205 struct caam_hash_state *state = ahash_request_ctx(req); 3206 struct caam_request *req_ctx = &state->caam_req; 3207 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3208 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3209 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3210 GFP_KERNEL : GFP_ATOMIC; 3211 u8 *buf = current_buf(state); 3212 int *buflen = current_buflen(state); 3213 u8 *next_buf = alt_buf(state); 3214 int *next_buflen = alt_buflen(state), last_buflen; 3215 int in_len = *buflen + req->nbytes, to_hash; 3216 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3217 struct ahash_edesc *edesc; 3218 int ret = 0; 3219 3220 last_buflen = *next_buflen; 3221 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3222 to_hash = in_len - *next_buflen; 3223 3224 if (to_hash) { 3225 struct dpaa2_sg_entry *sg_table; 3226 3227 src_nents = sg_nents_for_len(req->src, 3228 req->nbytes - (*next_buflen)); 3229 if (src_nents < 0) { 3230 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3231 return src_nents; 3232 } 3233 3234 if (src_nents) { 3235 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3236 DMA_TO_DEVICE); 3237 if (!mapped_nents) { 3238 dev_err(ctx->dev, "unable to DMA map source\n"); 3239 return -ENOMEM; 3240 } 3241 } else { 3242 mapped_nents = 0; 3243 } 3244 3245 /* allocate space for base edesc and link tables */ 3246 edesc = qi_cache_zalloc(GFP_DMA | flags); 3247 if (!edesc) { 3248 dma_unmap_sg(ctx->dev, req->src, src_nents, 3249 DMA_TO_DEVICE); 3250 return -ENOMEM; 3251 } 3252 3253 edesc->src_nents = src_nents; 3254 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3255 qm_sg_bytes = (qm_sg_src_index + mapped_nents) * 3256 sizeof(*sg_table); 3257 sg_table = &edesc->sgt[0]; 3258 3259 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3260 DMA_BIDIRECTIONAL); 3261 if (ret) 3262 goto unmap_ctx; 3263 3264 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3265 if (ret) 3266 goto unmap_ctx; 3267 3268 if (mapped_nents) { 3269 sg_to_qm_sg_last(req->src, mapped_nents, 3270 sg_table + qm_sg_src_index, 0); 3271 if (*next_buflen) 3272 scatterwalk_map_and_copy(next_buf, req->src, 3273 to_hash - *buflen, 3274 *next_buflen, 0); 3275 } else { 3276 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3277 true); 3278 } 3279 3280 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3281 qm_sg_bytes, DMA_TO_DEVICE); 3282 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3283 dev_err(ctx->dev, "unable to map S/G table\n"); 3284 ret = -ENOMEM; 3285 goto unmap_ctx; 3286 } 3287 edesc->qm_sg_bytes = qm_sg_bytes; 3288 3289 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3290 dpaa2_fl_set_final(in_fle, true); 3291 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3292 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3293 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3294 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3295 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3296 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3297 3298 req_ctx->flc = &ctx->flc[UPDATE]; 3299 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3300 req_ctx->cbk = ahash_done_bi; 3301 req_ctx->ctx = &req->base; 3302 req_ctx->edesc = edesc; 3303 3304 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3305 if (ret != -EINPROGRESS && 3306 !(ret == -EBUSY && 3307 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3308 goto unmap_ctx; 3309 } else if (*next_buflen) { 3310 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3311 req->nbytes, 0); 3312 *buflen = *next_buflen; 3313 *next_buflen = last_buflen; 3314 } 3315 3316 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3317 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 3318 print_hex_dump_debug("next buf@" __stringify(__LINE__)": ", 3319 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 3320 1); 3321 3322 return ret; 3323 unmap_ctx: 3324 ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 3325 qi_cache_free(edesc); 3326 return ret; 3327 } 3328 3329 static int ahash_final_ctx(struct ahash_request *req) 3330 { 3331 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3332 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3333 struct caam_hash_state *state = ahash_request_ctx(req); 3334 struct caam_request *req_ctx = &state->caam_req; 3335 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3336 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3337 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3338 GFP_KERNEL : GFP_ATOMIC; 3339 int buflen = *current_buflen(state); 3340 int qm_sg_bytes, qm_sg_src_index; 3341 int digestsize = crypto_ahash_digestsize(ahash); 3342 struct ahash_edesc *edesc; 3343 struct dpaa2_sg_entry *sg_table; 3344 int ret; 3345 3346 /* allocate space for base edesc and link tables */ 3347 edesc = qi_cache_zalloc(GFP_DMA | flags); 3348 if (!edesc) 3349 return -ENOMEM; 3350 3351 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3352 qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); 3353 sg_table = &edesc->sgt[0]; 3354 3355 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3356 DMA_TO_DEVICE); 3357 if (ret) 3358 goto unmap_ctx; 3359 3360 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3361 if (ret) 3362 goto unmap_ctx; 3363 3364 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); 3365 3366 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3367 DMA_TO_DEVICE); 3368 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3369 dev_err(ctx->dev, "unable to map S/G table\n"); 3370 ret = -ENOMEM; 3371 goto unmap_ctx; 3372 } 3373 edesc->qm_sg_bytes = qm_sg_bytes; 3374 3375 edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, 3376 DMA_FROM_DEVICE); 3377 if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { 3378 dev_err(ctx->dev, "unable to map dst\n"); 3379 edesc->dst_dma = 0; 3380 ret = -ENOMEM; 3381 goto unmap_ctx; 3382 } 3383 3384 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3385 dpaa2_fl_set_final(in_fle, true); 3386 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3387 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3388 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3389 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3390 dpaa2_fl_set_addr(out_fle, edesc->dst_dma); 3391 dpaa2_fl_set_len(out_fle, digestsize); 3392 3393 req_ctx->flc = &ctx->flc[FINALIZE]; 3394 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3395 req_ctx->cbk = ahash_done_ctx_src; 3396 req_ctx->ctx = &req->base; 3397 req_ctx->edesc = edesc; 3398 3399 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3400 if (ret == -EINPROGRESS || 3401 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3402 return ret; 3403 3404 unmap_ctx: 3405 ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); 3406 qi_cache_free(edesc); 3407 return ret; 3408 } 3409 3410 static int ahash_finup_ctx(struct ahash_request *req) 3411 { 3412 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3413 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3414 struct caam_hash_state *state = ahash_request_ctx(req); 3415 struct caam_request *req_ctx = &state->caam_req; 3416 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3417 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3418 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3419 GFP_KERNEL : GFP_ATOMIC; 3420 int buflen = *current_buflen(state); 3421 int qm_sg_bytes, qm_sg_src_index; 3422 int src_nents, mapped_nents; 3423 int digestsize = crypto_ahash_digestsize(ahash); 3424 struct ahash_edesc *edesc; 3425 struct dpaa2_sg_entry *sg_table; 3426 int ret; 3427 3428 src_nents = sg_nents_for_len(req->src, req->nbytes); 3429 if (src_nents < 0) { 3430 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3431 return src_nents; 3432 } 3433 3434 if (src_nents) { 3435 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3436 DMA_TO_DEVICE); 3437 if (!mapped_nents) { 3438 dev_err(ctx->dev, "unable to DMA map source\n"); 3439 return -ENOMEM; 3440 } 3441 } else { 3442 mapped_nents = 0; 3443 } 3444 3445 /* allocate space for base edesc and link tables */ 3446 edesc = qi_cache_zalloc(GFP_DMA | flags); 3447 if (!edesc) { 3448 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3449 return -ENOMEM; 3450 } 3451 3452 edesc->src_nents = src_nents; 3453 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3454 qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); 3455 sg_table = &edesc->sgt[0]; 3456 3457 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3458 DMA_TO_DEVICE); 3459 if (ret) 3460 goto unmap_ctx; 3461 3462 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3463 if (ret) 3464 goto unmap_ctx; 3465 3466 sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); 3467 3468 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3469 DMA_TO_DEVICE); 3470 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3471 dev_err(ctx->dev, "unable to map S/G table\n"); 3472 ret = -ENOMEM; 3473 goto unmap_ctx; 3474 } 3475 edesc->qm_sg_bytes = qm_sg_bytes; 3476 3477 edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, 3478 DMA_FROM_DEVICE); 3479 if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { 3480 dev_err(ctx->dev, "unable to map dst\n"); 3481 edesc->dst_dma = 0; 3482 ret = -ENOMEM; 3483 goto unmap_ctx; 3484 } 3485 3486 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3487 dpaa2_fl_set_final(in_fle, true); 3488 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3489 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3490 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3491 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3492 dpaa2_fl_set_addr(out_fle, edesc->dst_dma); 3493 dpaa2_fl_set_len(out_fle, digestsize); 3494 3495 req_ctx->flc = &ctx->flc[FINALIZE]; 3496 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3497 req_ctx->cbk = ahash_done_ctx_src; 3498 req_ctx->ctx = &req->base; 3499 req_ctx->edesc = edesc; 3500 3501 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3502 if (ret == -EINPROGRESS || 3503 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3504 return ret; 3505 3506 unmap_ctx: 3507 ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); 3508 qi_cache_free(edesc); 3509 return ret; 3510 } 3511 3512 static int ahash_digest(struct ahash_request *req) 3513 { 3514 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3515 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3516 struct caam_hash_state *state = ahash_request_ctx(req); 3517 struct caam_request *req_ctx = &state->caam_req; 3518 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3519 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3520 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3521 GFP_KERNEL : GFP_ATOMIC; 3522 int digestsize = crypto_ahash_digestsize(ahash); 3523 int src_nents, mapped_nents; 3524 struct ahash_edesc *edesc; 3525 int ret = -ENOMEM; 3526 3527 state->buf_dma = 0; 3528 3529 src_nents = sg_nents_for_len(req->src, req->nbytes); 3530 if (src_nents < 0) { 3531 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3532 return src_nents; 3533 } 3534 3535 if (src_nents) { 3536 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3537 DMA_TO_DEVICE); 3538 if (!mapped_nents) { 3539 dev_err(ctx->dev, "unable to map source for DMA\n"); 3540 return ret; 3541 } 3542 } else { 3543 mapped_nents = 0; 3544 } 3545 3546 /* allocate space for base edesc and link tables */ 3547 edesc = qi_cache_zalloc(GFP_DMA | flags); 3548 if (!edesc) { 3549 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3550 return ret; 3551 } 3552 3553 edesc->src_nents = src_nents; 3554 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3555 3556 if (mapped_nents > 1) { 3557 int qm_sg_bytes; 3558 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3559 3560 qm_sg_bytes = mapped_nents * sizeof(*sg_table); 3561 sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); 3562 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3563 qm_sg_bytes, DMA_TO_DEVICE); 3564 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3565 dev_err(ctx->dev, "unable to map S/G table\n"); 3566 goto unmap; 3567 } 3568 edesc->qm_sg_bytes = qm_sg_bytes; 3569 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3570 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3571 } else { 3572 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3573 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3574 } 3575 3576 edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, 3577 DMA_FROM_DEVICE); 3578 if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { 3579 dev_err(ctx->dev, "unable to map dst\n"); 3580 edesc->dst_dma = 0; 3581 goto unmap; 3582 } 3583 3584 dpaa2_fl_set_final(in_fle, true); 3585 dpaa2_fl_set_len(in_fle, req->nbytes); 3586 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3587 dpaa2_fl_set_addr(out_fle, edesc->dst_dma); 3588 dpaa2_fl_set_len(out_fle, digestsize); 3589 3590 req_ctx->flc = &ctx->flc[DIGEST]; 3591 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3592 req_ctx->cbk = ahash_done; 3593 req_ctx->ctx = &req->base; 3594 req_ctx->edesc = edesc; 3595 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3596 if (ret == -EINPROGRESS || 3597 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3598 return ret; 3599 3600 unmap: 3601 ahash_unmap(ctx->dev, edesc, req, digestsize); 3602 qi_cache_free(edesc); 3603 return ret; 3604 } 3605 3606 static int ahash_final_no_ctx(struct ahash_request *req) 3607 { 3608 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3609 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3610 struct caam_hash_state *state = ahash_request_ctx(req); 3611 struct caam_request *req_ctx = &state->caam_req; 3612 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3613 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3614 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3615 GFP_KERNEL : GFP_ATOMIC; 3616 u8 *buf = current_buf(state); 3617 int buflen = *current_buflen(state); 3618 int digestsize = crypto_ahash_digestsize(ahash); 3619 struct ahash_edesc *edesc; 3620 int ret = -ENOMEM; 3621 3622 /* allocate space for base edesc and link tables */ 3623 edesc = qi_cache_zalloc(GFP_DMA | flags); 3624 if (!edesc) 3625 return ret; 3626 3627 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, DMA_TO_DEVICE); 3628 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3629 dev_err(ctx->dev, "unable to map src\n"); 3630 goto unmap; 3631 } 3632 3633 edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, 3634 DMA_FROM_DEVICE); 3635 if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { 3636 dev_err(ctx->dev, "unable to map dst\n"); 3637 edesc->dst_dma = 0; 3638 goto unmap; 3639 } 3640 3641 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3642 dpaa2_fl_set_final(in_fle, true); 3643 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3644 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3645 dpaa2_fl_set_len(in_fle, buflen); 3646 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3647 dpaa2_fl_set_addr(out_fle, edesc->dst_dma); 3648 dpaa2_fl_set_len(out_fle, digestsize); 3649 3650 req_ctx->flc = &ctx->flc[DIGEST]; 3651 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3652 req_ctx->cbk = ahash_done; 3653 req_ctx->ctx = &req->base; 3654 req_ctx->edesc = edesc; 3655 3656 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3657 if (ret == -EINPROGRESS || 3658 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3659 return ret; 3660 3661 unmap: 3662 ahash_unmap(ctx->dev, edesc, req, digestsize); 3663 qi_cache_free(edesc); 3664 return ret; 3665 } 3666 3667 static int ahash_update_no_ctx(struct ahash_request *req) 3668 { 3669 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3670 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3671 struct caam_hash_state *state = ahash_request_ctx(req); 3672 struct caam_request *req_ctx = &state->caam_req; 3673 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3674 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3675 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3676 GFP_KERNEL : GFP_ATOMIC; 3677 u8 *buf = current_buf(state); 3678 int *buflen = current_buflen(state); 3679 u8 *next_buf = alt_buf(state); 3680 int *next_buflen = alt_buflen(state); 3681 int in_len = *buflen + req->nbytes, to_hash; 3682 int qm_sg_bytes, src_nents, mapped_nents; 3683 struct ahash_edesc *edesc; 3684 int ret = 0; 3685 3686 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3687 to_hash = in_len - *next_buflen; 3688 3689 if (to_hash) { 3690 struct dpaa2_sg_entry *sg_table; 3691 3692 src_nents = sg_nents_for_len(req->src, 3693 req->nbytes - *next_buflen); 3694 if (src_nents < 0) { 3695 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3696 return src_nents; 3697 } 3698 3699 if (src_nents) { 3700 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3701 DMA_TO_DEVICE); 3702 if (!mapped_nents) { 3703 dev_err(ctx->dev, "unable to DMA map source\n"); 3704 return -ENOMEM; 3705 } 3706 } else { 3707 mapped_nents = 0; 3708 } 3709 3710 /* allocate space for base edesc and link tables */ 3711 edesc = qi_cache_zalloc(GFP_DMA | flags); 3712 if (!edesc) { 3713 dma_unmap_sg(ctx->dev, req->src, src_nents, 3714 DMA_TO_DEVICE); 3715 return -ENOMEM; 3716 } 3717 3718 edesc->src_nents = src_nents; 3719 qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); 3720 sg_table = &edesc->sgt[0]; 3721 3722 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 3723 if (ret) 3724 goto unmap_ctx; 3725 3726 sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); 3727 3728 if (*next_buflen) 3729 scatterwalk_map_and_copy(next_buf, req->src, 3730 to_hash - *buflen, 3731 *next_buflen, 0); 3732 3733 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3734 qm_sg_bytes, DMA_TO_DEVICE); 3735 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3736 dev_err(ctx->dev, "unable to map S/G table\n"); 3737 ret = -ENOMEM; 3738 goto unmap_ctx; 3739 } 3740 edesc->qm_sg_bytes = qm_sg_bytes; 3741 3742 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 3743 ctx->ctx_len, DMA_FROM_DEVICE); 3744 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3745 dev_err(ctx->dev, "unable to map ctx\n"); 3746 state->ctx_dma = 0; 3747 ret = -ENOMEM; 3748 goto unmap_ctx; 3749 } 3750 3751 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3752 dpaa2_fl_set_final(in_fle, true); 3753 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3754 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3755 dpaa2_fl_set_len(in_fle, to_hash); 3756 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3757 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3758 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3759 3760 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 3761 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 3762 req_ctx->cbk = ahash_done_ctx_dst; 3763 req_ctx->ctx = &req->base; 3764 req_ctx->edesc = edesc; 3765 3766 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3767 if (ret != -EINPROGRESS && 3768 !(ret == -EBUSY && 3769 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3770 goto unmap_ctx; 3771 3772 state->update = ahash_update_ctx; 3773 state->finup = ahash_finup_ctx; 3774 state->final = ahash_final_ctx; 3775 } else if (*next_buflen) { 3776 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3777 req->nbytes, 0); 3778 *buflen = *next_buflen; 3779 *next_buflen = 0; 3780 } 3781 3782 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3783 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 3784 print_hex_dump_debug("next buf@" __stringify(__LINE__)": ", 3785 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 3786 1); 3787 3788 return ret; 3789 unmap_ctx: 3790 ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 3791 qi_cache_free(edesc); 3792 return ret; 3793 } 3794 3795 static int ahash_finup_no_ctx(struct ahash_request *req) 3796 { 3797 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3798 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3799 struct caam_hash_state *state = ahash_request_ctx(req); 3800 struct caam_request *req_ctx = &state->caam_req; 3801 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3802 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3803 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3804 GFP_KERNEL : GFP_ATOMIC; 3805 int buflen = *current_buflen(state); 3806 int qm_sg_bytes, src_nents, mapped_nents; 3807 int digestsize = crypto_ahash_digestsize(ahash); 3808 struct ahash_edesc *edesc; 3809 struct dpaa2_sg_entry *sg_table; 3810 int ret; 3811 3812 src_nents = sg_nents_for_len(req->src, req->nbytes); 3813 if (src_nents < 0) { 3814 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3815 return src_nents; 3816 } 3817 3818 if (src_nents) { 3819 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3820 DMA_TO_DEVICE); 3821 if (!mapped_nents) { 3822 dev_err(ctx->dev, "unable to DMA map source\n"); 3823 return -ENOMEM; 3824 } 3825 } else { 3826 mapped_nents = 0; 3827 } 3828 3829 /* allocate space for base edesc and link tables */ 3830 edesc = qi_cache_zalloc(GFP_DMA | flags); 3831 if (!edesc) { 3832 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3833 return -ENOMEM; 3834 } 3835 3836 edesc->src_nents = src_nents; 3837 qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); 3838 sg_table = &edesc->sgt[0]; 3839 3840 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 3841 if (ret) 3842 goto unmap; 3843 3844 sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); 3845 3846 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3847 DMA_TO_DEVICE); 3848 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3849 dev_err(ctx->dev, "unable to map S/G table\n"); 3850 ret = -ENOMEM; 3851 goto unmap; 3852 } 3853 edesc->qm_sg_bytes = qm_sg_bytes; 3854 3855 edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, 3856 DMA_FROM_DEVICE); 3857 if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { 3858 dev_err(ctx->dev, "unable to map dst\n"); 3859 edesc->dst_dma = 0; 3860 ret = -ENOMEM; 3861 goto unmap; 3862 } 3863 3864 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3865 dpaa2_fl_set_final(in_fle, true); 3866 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3867 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3868 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 3869 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3870 dpaa2_fl_set_addr(out_fle, edesc->dst_dma); 3871 dpaa2_fl_set_len(out_fle, digestsize); 3872 3873 req_ctx->flc = &ctx->flc[DIGEST]; 3874 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3875 req_ctx->cbk = ahash_done; 3876 req_ctx->ctx = &req->base; 3877 req_ctx->edesc = edesc; 3878 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3879 if (ret != -EINPROGRESS && 3880 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3881 goto unmap; 3882 3883 return ret; 3884 unmap: 3885 ahash_unmap(ctx->dev, edesc, req, digestsize); 3886 qi_cache_free(edesc); 3887 return -ENOMEM; 3888 } 3889 3890 static int ahash_update_first(struct ahash_request *req) 3891 { 3892 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3893 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3894 struct caam_hash_state *state = ahash_request_ctx(req); 3895 struct caam_request *req_ctx = &state->caam_req; 3896 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3897 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3898 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3899 GFP_KERNEL : GFP_ATOMIC; 3900 u8 *next_buf = alt_buf(state); 3901 int *next_buflen = alt_buflen(state); 3902 int to_hash; 3903 int src_nents, mapped_nents; 3904 struct ahash_edesc *edesc; 3905 int ret = 0; 3906 3907 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 3908 1); 3909 to_hash = req->nbytes - *next_buflen; 3910 3911 if (to_hash) { 3912 struct dpaa2_sg_entry *sg_table; 3913 3914 src_nents = sg_nents_for_len(req->src, 3915 req->nbytes - (*next_buflen)); 3916 if (src_nents < 0) { 3917 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3918 return src_nents; 3919 } 3920 3921 if (src_nents) { 3922 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3923 DMA_TO_DEVICE); 3924 if (!mapped_nents) { 3925 dev_err(ctx->dev, "unable to map source for DMA\n"); 3926 return -ENOMEM; 3927 } 3928 } else { 3929 mapped_nents = 0; 3930 } 3931 3932 /* allocate space for base edesc and link tables */ 3933 edesc = qi_cache_zalloc(GFP_DMA | flags); 3934 if (!edesc) { 3935 dma_unmap_sg(ctx->dev, req->src, src_nents, 3936 DMA_TO_DEVICE); 3937 return -ENOMEM; 3938 } 3939 3940 edesc->src_nents = src_nents; 3941 sg_table = &edesc->sgt[0]; 3942 3943 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3944 dpaa2_fl_set_final(in_fle, true); 3945 dpaa2_fl_set_len(in_fle, to_hash); 3946 3947 if (mapped_nents > 1) { 3948 int qm_sg_bytes; 3949 3950 sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); 3951 qm_sg_bytes = mapped_nents * sizeof(*sg_table); 3952 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3953 qm_sg_bytes, 3954 DMA_TO_DEVICE); 3955 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3956 dev_err(ctx->dev, "unable to map S/G table\n"); 3957 ret = -ENOMEM; 3958 goto unmap_ctx; 3959 } 3960 edesc->qm_sg_bytes = qm_sg_bytes; 3961 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3962 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3963 } else { 3964 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3965 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3966 } 3967 3968 if (*next_buflen) 3969 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 3970 *next_buflen, 0); 3971 3972 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 3973 ctx->ctx_len, DMA_FROM_DEVICE); 3974 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3975 dev_err(ctx->dev, "unable to map ctx\n"); 3976 state->ctx_dma = 0; 3977 ret = -ENOMEM; 3978 goto unmap_ctx; 3979 } 3980 3981 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3982 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3983 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3984 3985 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 3986 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 3987 req_ctx->cbk = ahash_done_ctx_dst; 3988 req_ctx->ctx = &req->base; 3989 req_ctx->edesc = edesc; 3990 3991 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3992 if (ret != -EINPROGRESS && 3993 !(ret == -EBUSY && req->base.flags & 3994 CRYPTO_TFM_REQ_MAY_BACKLOG)) 3995 goto unmap_ctx; 3996 3997 state->update = ahash_update_ctx; 3998 state->finup = ahash_finup_ctx; 3999 state->final = ahash_final_ctx; 4000 } else if (*next_buflen) { 4001 state->update = ahash_update_no_ctx; 4002 state->finup = ahash_finup_no_ctx; 4003 state->final = ahash_final_no_ctx; 4004 scatterwalk_map_and_copy(next_buf, req->src, 0, 4005 req->nbytes, 0); 4006 switch_buf(state); 4007 } 4008 4009 print_hex_dump_debug("next buf@" __stringify(__LINE__)": ", 4010 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 4011 1); 4012 4013 return ret; 4014 unmap_ctx: 4015 ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 4016 qi_cache_free(edesc); 4017 return ret; 4018 } 4019 4020 static int ahash_finup_first(struct ahash_request *req) 4021 { 4022 return ahash_digest(req); 4023 } 4024 4025 static int ahash_init(struct ahash_request *req) 4026 { 4027 struct caam_hash_state *state = ahash_request_ctx(req); 4028 4029 state->update = ahash_update_first; 4030 state->finup = ahash_finup_first; 4031 state->final = ahash_final_no_ctx; 4032 4033 state->ctx_dma = 0; 4034 state->current_buf = 0; 4035 state->buf_dma = 0; 4036 state->buflen_0 = 0; 4037 state->buflen_1 = 0; 4038 4039 return 0; 4040 } 4041 4042 static int ahash_update(struct ahash_request *req) 4043 { 4044 struct caam_hash_state *state = ahash_request_ctx(req); 4045 4046 return state->update(req); 4047 } 4048 4049 static int ahash_finup(struct ahash_request *req) 4050 { 4051 struct caam_hash_state *state = ahash_request_ctx(req); 4052 4053 return state->finup(req); 4054 } 4055 4056 static int ahash_final(struct ahash_request *req) 4057 { 4058 struct caam_hash_state *state = ahash_request_ctx(req); 4059 4060 return state->final(req); 4061 } 4062 4063 static int ahash_export(struct ahash_request *req, void *out) 4064 { 4065 struct caam_hash_state *state = ahash_request_ctx(req); 4066 struct caam_export_state *export = out; 4067 int len; 4068 u8 *buf; 4069 4070 if (state->current_buf) { 4071 buf = state->buf_1; 4072 len = state->buflen_1; 4073 } else { 4074 buf = state->buf_0; 4075 len = state->buflen_0; 4076 } 4077 4078 memcpy(export->buf, buf, len); 4079 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4080 export->buflen = len; 4081 export->update = state->update; 4082 export->final = state->final; 4083 export->finup = state->finup; 4084 4085 return 0; 4086 } 4087 4088 static int ahash_import(struct ahash_request *req, const void *in) 4089 { 4090 struct caam_hash_state *state = ahash_request_ctx(req); 4091 const struct caam_export_state *export = in; 4092 4093 memset(state, 0, sizeof(*state)); 4094 memcpy(state->buf_0, export->buf, export->buflen); 4095 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4096 state->buflen_0 = export->buflen; 4097 state->update = export->update; 4098 state->final = export->final; 4099 state->finup = export->finup; 4100 4101 return 0; 4102 } 4103 4104 struct caam_hash_template { 4105 char name[CRYPTO_MAX_ALG_NAME]; 4106 char driver_name[CRYPTO_MAX_ALG_NAME]; 4107 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4108 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4109 unsigned int blocksize; 4110 struct ahash_alg template_ahash; 4111 u32 alg_type; 4112 }; 4113 4114 /* ahash descriptors */ 4115 static struct caam_hash_template driver_hash[] = { 4116 { 4117 .name = "sha1", 4118 .driver_name = "sha1-caam-qi2", 4119 .hmac_name = "hmac(sha1)", 4120 .hmac_driver_name = "hmac-sha1-caam-qi2", 4121 .blocksize = SHA1_BLOCK_SIZE, 4122 .template_ahash = { 4123 .init = ahash_init, 4124 .update = ahash_update, 4125 .final = ahash_final, 4126 .finup = ahash_finup, 4127 .digest = ahash_digest, 4128 .export = ahash_export, 4129 .import = ahash_import, 4130 .setkey = ahash_setkey, 4131 .halg = { 4132 .digestsize = SHA1_DIGEST_SIZE, 4133 .statesize = sizeof(struct caam_export_state), 4134 }, 4135 }, 4136 .alg_type = OP_ALG_ALGSEL_SHA1, 4137 }, { 4138 .name = "sha224", 4139 .driver_name = "sha224-caam-qi2", 4140 .hmac_name = "hmac(sha224)", 4141 .hmac_driver_name = "hmac-sha224-caam-qi2", 4142 .blocksize = SHA224_BLOCK_SIZE, 4143 .template_ahash = { 4144 .init = ahash_init, 4145 .update = ahash_update, 4146 .final = ahash_final, 4147 .finup = ahash_finup, 4148 .digest = ahash_digest, 4149 .export = ahash_export, 4150 .import = ahash_import, 4151 .setkey = ahash_setkey, 4152 .halg = { 4153 .digestsize = SHA224_DIGEST_SIZE, 4154 .statesize = sizeof(struct caam_export_state), 4155 }, 4156 }, 4157 .alg_type = OP_ALG_ALGSEL_SHA224, 4158 }, { 4159 .name = "sha256", 4160 .driver_name = "sha256-caam-qi2", 4161 .hmac_name = "hmac(sha256)", 4162 .hmac_driver_name = "hmac-sha256-caam-qi2", 4163 .blocksize = SHA256_BLOCK_SIZE, 4164 .template_ahash = { 4165 .init = ahash_init, 4166 .update = ahash_update, 4167 .final = ahash_final, 4168 .finup = ahash_finup, 4169 .digest = ahash_digest, 4170 .export = ahash_export, 4171 .import = ahash_import, 4172 .setkey = ahash_setkey, 4173 .halg = { 4174 .digestsize = SHA256_DIGEST_SIZE, 4175 .statesize = sizeof(struct caam_export_state), 4176 }, 4177 }, 4178 .alg_type = OP_ALG_ALGSEL_SHA256, 4179 }, { 4180 .name = "sha384", 4181 .driver_name = "sha384-caam-qi2", 4182 .hmac_name = "hmac(sha384)", 4183 .hmac_driver_name = "hmac-sha384-caam-qi2", 4184 .blocksize = SHA384_BLOCK_SIZE, 4185 .template_ahash = { 4186 .init = ahash_init, 4187 .update = ahash_update, 4188 .final = ahash_final, 4189 .finup = ahash_finup, 4190 .digest = ahash_digest, 4191 .export = ahash_export, 4192 .import = ahash_import, 4193 .setkey = ahash_setkey, 4194 .halg = { 4195 .digestsize = SHA384_DIGEST_SIZE, 4196 .statesize = sizeof(struct caam_export_state), 4197 }, 4198 }, 4199 .alg_type = OP_ALG_ALGSEL_SHA384, 4200 }, { 4201 .name = "sha512", 4202 .driver_name = "sha512-caam-qi2", 4203 .hmac_name = "hmac(sha512)", 4204 .hmac_driver_name = "hmac-sha512-caam-qi2", 4205 .blocksize = SHA512_BLOCK_SIZE, 4206 .template_ahash = { 4207 .init = ahash_init, 4208 .update = ahash_update, 4209 .final = ahash_final, 4210 .finup = ahash_finup, 4211 .digest = ahash_digest, 4212 .export = ahash_export, 4213 .import = ahash_import, 4214 .setkey = ahash_setkey, 4215 .halg = { 4216 .digestsize = SHA512_DIGEST_SIZE, 4217 .statesize = sizeof(struct caam_export_state), 4218 }, 4219 }, 4220 .alg_type = OP_ALG_ALGSEL_SHA512, 4221 }, { 4222 .name = "md5", 4223 .driver_name = "md5-caam-qi2", 4224 .hmac_name = "hmac(md5)", 4225 .hmac_driver_name = "hmac-md5-caam-qi2", 4226 .blocksize = MD5_BLOCK_WORDS * 4, 4227 .template_ahash = { 4228 .init = ahash_init, 4229 .update = ahash_update, 4230 .final = ahash_final, 4231 .finup = ahash_finup, 4232 .digest = ahash_digest, 4233 .export = ahash_export, 4234 .import = ahash_import, 4235 .setkey = ahash_setkey, 4236 .halg = { 4237 .digestsize = MD5_DIGEST_SIZE, 4238 .statesize = sizeof(struct caam_export_state), 4239 }, 4240 }, 4241 .alg_type = OP_ALG_ALGSEL_MD5, 4242 } 4243 }; 4244 4245 struct caam_hash_alg { 4246 struct list_head entry; 4247 struct device *dev; 4248 int alg_type; 4249 struct ahash_alg ahash_alg; 4250 }; 4251 4252 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4253 { 4254 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4255 struct crypto_alg *base = tfm->__crt_alg; 4256 struct hash_alg_common *halg = 4257 container_of(base, struct hash_alg_common, base); 4258 struct ahash_alg *alg = 4259 container_of(halg, struct ahash_alg, halg); 4260 struct caam_hash_alg *caam_hash = 4261 container_of(alg, struct caam_hash_alg, ahash_alg); 4262 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4263 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4264 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4265 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4266 HASH_MSG_LEN + 32, 4267 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4268 HASH_MSG_LEN + 64, 4269 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4270 dma_addr_t dma_addr; 4271 int i; 4272 4273 ctx->dev = caam_hash->dev; 4274 4275 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4276 DMA_BIDIRECTIONAL, 4277 DMA_ATTR_SKIP_CPU_SYNC); 4278 if (dma_mapping_error(ctx->dev, dma_addr)) { 4279 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4280 return -ENOMEM; 4281 } 4282 4283 for (i = 0; i < HASH_NUM_OP; i++) 4284 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4285 4286 /* copy descriptor header template value */ 4287 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4288 4289 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4290 OP_ALG_ALGSEL_SUBMASK) >> 4291 OP_ALG_ALGSEL_SHIFT]; 4292 4293 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 4294 sizeof(struct caam_hash_state)); 4295 4296 return ahash_set_sh_desc(ahash); 4297 } 4298 4299 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4300 { 4301 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4302 4303 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4304 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4305 } 4306 4307 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4308 struct caam_hash_template *template, bool keyed) 4309 { 4310 struct caam_hash_alg *t_alg; 4311 struct ahash_alg *halg; 4312 struct crypto_alg *alg; 4313 4314 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4315 if (!t_alg) 4316 return ERR_PTR(-ENOMEM); 4317 4318 t_alg->ahash_alg = template->template_ahash; 4319 halg = &t_alg->ahash_alg; 4320 alg = &halg->halg.base; 4321 4322 if (keyed) { 4323 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4324 template->hmac_name); 4325 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4326 template->hmac_driver_name); 4327 } else { 4328 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4329 template->name); 4330 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4331 template->driver_name); 4332 t_alg->ahash_alg.setkey = NULL; 4333 } 4334 alg->cra_module = THIS_MODULE; 4335 alg->cra_init = caam_hash_cra_init; 4336 alg->cra_exit = caam_hash_cra_exit; 4337 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 4338 alg->cra_priority = CAAM_CRA_PRIORITY; 4339 alg->cra_blocksize = template->blocksize; 4340 alg->cra_alignmask = 0; 4341 alg->cra_flags = CRYPTO_ALG_ASYNC; 4342 4343 t_alg->alg_type = template->alg_type; 4344 t_alg->dev = dev; 4345 4346 return t_alg; 4347 } 4348 4349 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4350 { 4351 struct dpaa2_caam_priv_per_cpu *ppriv; 4352 4353 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4354 napi_schedule_irqoff(&ppriv->napi); 4355 } 4356 4357 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4358 { 4359 struct device *dev = priv->dev; 4360 struct dpaa2_io_notification_ctx *nctx; 4361 struct dpaa2_caam_priv_per_cpu *ppriv; 4362 int err, i = 0, cpu; 4363 4364 for_each_online_cpu(cpu) { 4365 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4366 ppriv->priv = priv; 4367 nctx = &ppriv->nctx; 4368 nctx->is_cdan = 0; 4369 nctx->id = ppriv->rsp_fqid; 4370 nctx->desired_cpu = cpu; 4371 nctx->cb = dpaa2_caam_fqdan_cb; 4372 4373 /* Register notification callbacks */ 4374 err = dpaa2_io_service_register(NULL, nctx); 4375 if (unlikely(err)) { 4376 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4377 nctx->cb = NULL; 4378 /* 4379 * If no affine DPIO for this core, there's probably 4380 * none available for next cores either. Signal we want 4381 * to retry later, in case the DPIO devices weren't 4382 * probed yet. 4383 */ 4384 err = -EPROBE_DEFER; 4385 goto err; 4386 } 4387 4388 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4389 dev); 4390 if (unlikely(!ppriv->store)) { 4391 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4392 err = -ENOMEM; 4393 goto err; 4394 } 4395 4396 if (++i == priv->num_pairs) 4397 break; 4398 } 4399 4400 return 0; 4401 4402 err: 4403 for_each_online_cpu(cpu) { 4404 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4405 if (!ppriv->nctx.cb) 4406 break; 4407 dpaa2_io_service_deregister(NULL, &ppriv->nctx); 4408 } 4409 4410 for_each_online_cpu(cpu) { 4411 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4412 if (!ppriv->store) 4413 break; 4414 dpaa2_io_store_destroy(ppriv->store); 4415 } 4416 4417 return err; 4418 } 4419 4420 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4421 { 4422 struct dpaa2_caam_priv_per_cpu *ppriv; 4423 int i = 0, cpu; 4424 4425 for_each_online_cpu(cpu) { 4426 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4427 dpaa2_io_service_deregister(NULL, &ppriv->nctx); 4428 dpaa2_io_store_destroy(ppriv->store); 4429 4430 if (++i == priv->num_pairs) 4431 return; 4432 } 4433 } 4434 4435 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4436 { 4437 struct dpseci_rx_queue_cfg rx_queue_cfg; 4438 struct device *dev = priv->dev; 4439 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4440 struct dpaa2_caam_priv_per_cpu *ppriv; 4441 int err = 0, i = 0, cpu; 4442 4443 /* Configure Rx queues */ 4444 for_each_online_cpu(cpu) { 4445 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4446 4447 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4448 DPSECI_QUEUE_OPT_USER_CTX; 4449 rx_queue_cfg.order_preservation_en = 0; 4450 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4451 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4452 /* 4453 * Rx priority (WQ) doesn't really matter, since we use 4454 * pull mode, i.e. volatile dequeues from specific FQs 4455 */ 4456 rx_queue_cfg.dest_cfg.priority = 0; 4457 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4458 4459 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4460 &rx_queue_cfg); 4461 if (err) { 4462 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4463 err); 4464 return err; 4465 } 4466 4467 if (++i == priv->num_pairs) 4468 break; 4469 } 4470 4471 return err; 4472 } 4473 4474 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4475 { 4476 struct device *dev = priv->dev; 4477 4478 if (!priv->cscn_mem) 4479 return; 4480 4481 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4482 kfree(priv->cscn_mem); 4483 } 4484 4485 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4486 { 4487 struct device *dev = priv->dev; 4488 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4489 4490 dpaa2_dpseci_congestion_free(priv); 4491 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4492 } 4493 4494 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4495 const struct dpaa2_fd *fd) 4496 { 4497 struct caam_request *req; 4498 u32 fd_err; 4499 4500 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4501 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4502 return; 4503 } 4504 4505 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4506 if (unlikely(fd_err)) 4507 dev_err(priv->dev, "FD error: %08x\n", fd_err); 4508 4509 /* 4510 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4511 * in FD[ERR] or FD[FRC]. 4512 */ 4513 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4514 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4515 DMA_BIDIRECTIONAL); 4516 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4517 } 4518 4519 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4520 { 4521 int err; 4522 4523 /* Retry while portal is busy */ 4524 do { 4525 err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, 4526 ppriv->store); 4527 } while (err == -EBUSY); 4528 4529 if (unlikely(err)) 4530 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4531 4532 return err; 4533 } 4534 4535 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4536 { 4537 struct dpaa2_dq *dq; 4538 int cleaned = 0, is_last; 4539 4540 do { 4541 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4542 if (unlikely(!dq)) { 4543 if (unlikely(!is_last)) { 4544 dev_dbg(ppriv->priv->dev, 4545 "FQ %d returned no valid frames\n", 4546 ppriv->rsp_fqid); 4547 /* 4548 * MUST retry until we get some sort of 4549 * valid response token (be it "empty dequeue" 4550 * or a valid frame). 4551 */ 4552 continue; 4553 } 4554 break; 4555 } 4556 4557 /* Process FD */ 4558 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4559 cleaned++; 4560 } while (!is_last); 4561 4562 return cleaned; 4563 } 4564 4565 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4566 { 4567 struct dpaa2_caam_priv_per_cpu *ppriv; 4568 struct dpaa2_caam_priv *priv; 4569 int err, cleaned = 0, store_cleaned; 4570 4571 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4572 priv = ppriv->priv; 4573 4574 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4575 return 0; 4576 4577 do { 4578 store_cleaned = dpaa2_caam_store_consume(ppriv); 4579 cleaned += store_cleaned; 4580 4581 if (store_cleaned == 0 || 4582 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4583 break; 4584 4585 /* Try to dequeue some more */ 4586 err = dpaa2_caam_pull_fq(ppriv); 4587 if (unlikely(err)) 4588 break; 4589 } while (1); 4590 4591 if (cleaned < budget) { 4592 napi_complete_done(napi, cleaned); 4593 err = dpaa2_io_service_rearm(NULL, &ppriv->nctx); 4594 if (unlikely(err)) 4595 dev_err(priv->dev, "Notification rearm failed: %d\n", 4596 err); 4597 } 4598 4599 return cleaned; 4600 } 4601 4602 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4603 u16 token) 4604 { 4605 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4606 struct device *dev = priv->dev; 4607 int err; 4608 4609 /* 4610 * Congestion group feature supported starting with DPSECI API v5.1 4611 * and only when object has been created with this capability. 4612 */ 4613 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4614 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4615 return 0; 4616 4617 priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, 4618 GFP_KERNEL | GFP_DMA); 4619 if (!priv->cscn_mem) 4620 return -ENOMEM; 4621 4622 priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); 4623 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, 4624 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4625 if (dma_mapping_error(dev, priv->cscn_dma)) { 4626 dev_err(dev, "Error mapping CSCN memory area\n"); 4627 err = -ENOMEM; 4628 goto err_dma_map; 4629 } 4630 4631 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4632 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4633 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4634 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4635 cong_notif_cfg.message_iova = priv->cscn_dma; 4636 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4637 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4638 DPSECI_CGN_MODE_COHERENT_WRITE; 4639 4640 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4641 &cong_notif_cfg); 4642 if (err) { 4643 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4644 goto err_set_cong; 4645 } 4646 4647 return 0; 4648 4649 err_set_cong: 4650 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4651 err_dma_map: 4652 kfree(priv->cscn_mem); 4653 4654 return err; 4655 } 4656 4657 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4658 { 4659 struct device *dev = &ls_dev->dev; 4660 struct dpaa2_caam_priv *priv; 4661 struct dpaa2_caam_priv_per_cpu *ppriv; 4662 int err, cpu; 4663 u8 i; 4664 4665 priv = dev_get_drvdata(dev); 4666 4667 priv->dev = dev; 4668 priv->dpsec_id = ls_dev->obj_desc.id; 4669 4670 /* Get a handle for the DPSECI this interface is associate with */ 4671 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 4672 if (err) { 4673 dev_err(dev, "dpseci_open() failed: %d\n", err); 4674 goto err_open; 4675 } 4676 4677 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 4678 &priv->minor_ver); 4679 if (err) { 4680 dev_err(dev, "dpseci_get_api_version() failed\n"); 4681 goto err_get_vers; 4682 } 4683 4684 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 4685 4686 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 4687 &priv->dpseci_attr); 4688 if (err) { 4689 dev_err(dev, "dpseci_get_attributes() failed\n"); 4690 goto err_get_vers; 4691 } 4692 4693 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 4694 &priv->sec_attr); 4695 if (err) { 4696 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 4697 goto err_get_vers; 4698 } 4699 4700 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 4701 if (err) { 4702 dev_err(dev, "setup_congestion() failed\n"); 4703 goto err_get_vers; 4704 } 4705 4706 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 4707 priv->dpseci_attr.num_tx_queues); 4708 if (priv->num_pairs > num_online_cpus()) { 4709 dev_warn(dev, "%d queues won't be used\n", 4710 priv->num_pairs - num_online_cpus()); 4711 priv->num_pairs = num_online_cpus(); 4712 } 4713 4714 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 4715 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4716 &priv->rx_queue_attr[i]); 4717 if (err) { 4718 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 4719 goto err_get_rx_queue; 4720 } 4721 } 4722 4723 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 4724 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4725 &priv->tx_queue_attr[i]); 4726 if (err) { 4727 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 4728 goto err_get_rx_queue; 4729 } 4730 } 4731 4732 i = 0; 4733 for_each_online_cpu(cpu) { 4734 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", i, 4735 priv->rx_queue_attr[i].fqid, 4736 priv->tx_queue_attr[i].fqid); 4737 4738 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4739 ppriv->req_fqid = priv->tx_queue_attr[i].fqid; 4740 ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; 4741 ppriv->prio = i; 4742 4743 ppriv->net_dev.dev = *dev; 4744 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 4745 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, 4746 DPAA2_CAAM_NAPI_WEIGHT); 4747 if (++i == priv->num_pairs) 4748 break; 4749 } 4750 4751 return 0; 4752 4753 err_get_rx_queue: 4754 dpaa2_dpseci_congestion_free(priv); 4755 err_get_vers: 4756 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4757 err_open: 4758 return err; 4759 } 4760 4761 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 4762 { 4763 struct device *dev = priv->dev; 4764 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4765 struct dpaa2_caam_priv_per_cpu *ppriv; 4766 int i; 4767 4768 for (i = 0; i < priv->num_pairs; i++) { 4769 ppriv = per_cpu_ptr(priv->ppriv, i); 4770 napi_enable(&ppriv->napi); 4771 } 4772 4773 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 4774 } 4775 4776 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 4777 { 4778 struct device *dev = priv->dev; 4779 struct dpaa2_caam_priv_per_cpu *ppriv; 4780 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4781 int i, err = 0, enabled; 4782 4783 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 4784 if (err) { 4785 dev_err(dev, "dpseci_disable() failed\n"); 4786 return err; 4787 } 4788 4789 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 4790 if (err) { 4791 dev_err(dev, "dpseci_is_enabled() failed\n"); 4792 return err; 4793 } 4794 4795 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 4796 4797 for (i = 0; i < priv->num_pairs; i++) { 4798 ppriv = per_cpu_ptr(priv->ppriv, i); 4799 napi_disable(&ppriv->napi); 4800 netif_napi_del(&ppriv->napi); 4801 } 4802 4803 return 0; 4804 } 4805 4806 static struct list_head hash_list; 4807 4808 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 4809 { 4810 struct device *dev; 4811 struct dpaa2_caam_priv *priv; 4812 int i, err = 0; 4813 bool registered = false; 4814 4815 /* 4816 * There is no way to get CAAM endianness - there is no direct register 4817 * space access and MC f/w does not provide this attribute. 4818 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 4819 * property. 4820 */ 4821 caam_little_end = true; 4822 4823 caam_imx = false; 4824 4825 dev = &dpseci_dev->dev; 4826 4827 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 4828 if (!priv) 4829 return -ENOMEM; 4830 4831 dev_set_drvdata(dev, priv); 4832 4833 priv->domain = iommu_get_domain_for_dev(dev); 4834 4835 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 4836 0, SLAB_CACHE_DMA, NULL); 4837 if (!qi_cache) { 4838 dev_err(dev, "Can't allocate SEC cache\n"); 4839 return -ENOMEM; 4840 } 4841 4842 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 4843 if (err) { 4844 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 4845 goto err_dma_mask; 4846 } 4847 4848 /* Obtain a MC portal */ 4849 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 4850 if (err) { 4851 if (err == -ENXIO) 4852 err = -EPROBE_DEFER; 4853 else 4854 dev_err(dev, "MC portal allocation failed\n"); 4855 4856 goto err_dma_mask; 4857 } 4858 4859 priv->ppriv = alloc_percpu(*priv->ppriv); 4860 if (!priv->ppriv) { 4861 dev_err(dev, "alloc_percpu() failed\n"); 4862 err = -ENOMEM; 4863 goto err_alloc_ppriv; 4864 } 4865 4866 /* DPSECI initialization */ 4867 err = dpaa2_dpseci_setup(dpseci_dev); 4868 if (err) { 4869 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 4870 goto err_dpseci_setup; 4871 } 4872 4873 /* DPIO */ 4874 err = dpaa2_dpseci_dpio_setup(priv); 4875 if (err) { 4876 if (err != -EPROBE_DEFER) 4877 dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); 4878 goto err_dpio_setup; 4879 } 4880 4881 /* DPSECI binding to DPIO */ 4882 err = dpaa2_dpseci_bind(priv); 4883 if (err) { 4884 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 4885 goto err_bind; 4886 } 4887 4888 /* DPSECI enable */ 4889 err = dpaa2_dpseci_enable(priv); 4890 if (err) { 4891 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 4892 goto err_bind; 4893 } 4894 4895 /* register crypto algorithms the device supports */ 4896 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4897 struct caam_skcipher_alg *t_alg = driver_algs + i; 4898 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 4899 4900 /* Skip DES algorithms if not supported by device */ 4901 if (!priv->sec_attr.des_acc_num && 4902 (alg_sel == OP_ALG_ALGSEL_3DES || 4903 alg_sel == OP_ALG_ALGSEL_DES)) 4904 continue; 4905 4906 /* Skip AES algorithms if not supported by device */ 4907 if (!priv->sec_attr.aes_acc_num && 4908 alg_sel == OP_ALG_ALGSEL_AES) 4909 continue; 4910 4911 t_alg->caam.dev = dev; 4912 caam_skcipher_alg_init(t_alg); 4913 4914 err = crypto_register_skcipher(&t_alg->skcipher); 4915 if (err) { 4916 dev_warn(dev, "%s alg registration failed: %d\n", 4917 t_alg->skcipher.base.cra_driver_name, err); 4918 continue; 4919 } 4920 4921 t_alg->registered = true; 4922 registered = true; 4923 } 4924 4925 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4926 struct caam_aead_alg *t_alg = driver_aeads + i; 4927 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 4928 OP_ALG_ALGSEL_MASK; 4929 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 4930 OP_ALG_ALGSEL_MASK; 4931 4932 /* Skip DES algorithms if not supported by device */ 4933 if (!priv->sec_attr.des_acc_num && 4934 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 4935 c1_alg_sel == OP_ALG_ALGSEL_DES)) 4936 continue; 4937 4938 /* Skip AES algorithms if not supported by device */ 4939 if (!priv->sec_attr.aes_acc_num && 4940 c1_alg_sel == OP_ALG_ALGSEL_AES) 4941 continue; 4942 4943 /* 4944 * Skip algorithms requiring message digests 4945 * if MD not supported by device. 4946 */ 4947 if (!priv->sec_attr.md_acc_num && c2_alg_sel) 4948 continue; 4949 4950 t_alg->caam.dev = dev; 4951 caam_aead_alg_init(t_alg); 4952 4953 err = crypto_register_aead(&t_alg->aead); 4954 if (err) { 4955 dev_warn(dev, "%s alg registration failed: %d\n", 4956 t_alg->aead.base.cra_driver_name, err); 4957 continue; 4958 } 4959 4960 t_alg->registered = true; 4961 registered = true; 4962 } 4963 if (registered) 4964 dev_info(dev, "algorithms registered in /proc/crypto\n"); 4965 4966 /* register hash algorithms the device supports */ 4967 INIT_LIST_HEAD(&hash_list); 4968 4969 /* 4970 * Skip registration of any hashing algorithms if MD block 4971 * is not present. 4972 */ 4973 if (!priv->sec_attr.md_acc_num) 4974 return 0; 4975 4976 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 4977 struct caam_hash_alg *t_alg; 4978 struct caam_hash_template *alg = driver_hash + i; 4979 4980 /* register hmac version */ 4981 t_alg = caam_hash_alloc(dev, alg, true); 4982 if (IS_ERR(t_alg)) { 4983 err = PTR_ERR(t_alg); 4984 dev_warn(dev, "%s hash alg allocation failed: %d\n", 4985 alg->driver_name, err); 4986 continue; 4987 } 4988 4989 err = crypto_register_ahash(&t_alg->ahash_alg); 4990 if (err) { 4991 dev_warn(dev, "%s alg registration failed: %d\n", 4992 t_alg->ahash_alg.halg.base.cra_driver_name, 4993 err); 4994 kfree(t_alg); 4995 } else { 4996 list_add_tail(&t_alg->entry, &hash_list); 4997 } 4998 4999 /* register unkeyed version */ 5000 t_alg = caam_hash_alloc(dev, alg, false); 5001 if (IS_ERR(t_alg)) { 5002 err = PTR_ERR(t_alg); 5003 dev_warn(dev, "%s alg allocation failed: %d\n", 5004 alg->driver_name, err); 5005 continue; 5006 } 5007 5008 err = crypto_register_ahash(&t_alg->ahash_alg); 5009 if (err) { 5010 dev_warn(dev, "%s alg registration failed: %d\n", 5011 t_alg->ahash_alg.halg.base.cra_driver_name, 5012 err); 5013 kfree(t_alg); 5014 } else { 5015 list_add_tail(&t_alg->entry, &hash_list); 5016 } 5017 } 5018 if (!list_empty(&hash_list)) 5019 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5020 5021 return err; 5022 5023 err_bind: 5024 dpaa2_dpseci_dpio_free(priv); 5025 err_dpio_setup: 5026 dpaa2_dpseci_free(priv); 5027 err_dpseci_setup: 5028 free_percpu(priv->ppriv); 5029 err_alloc_ppriv: 5030 fsl_mc_portal_free(priv->mc_io); 5031 err_dma_mask: 5032 kmem_cache_destroy(qi_cache); 5033 5034 return err; 5035 } 5036 5037 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5038 { 5039 struct device *dev; 5040 struct dpaa2_caam_priv *priv; 5041 int i; 5042 5043 dev = &ls_dev->dev; 5044 priv = dev_get_drvdata(dev); 5045 5046 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5047 struct caam_aead_alg *t_alg = driver_aeads + i; 5048 5049 if (t_alg->registered) 5050 crypto_unregister_aead(&t_alg->aead); 5051 } 5052 5053 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5054 struct caam_skcipher_alg *t_alg = driver_algs + i; 5055 5056 if (t_alg->registered) 5057 crypto_unregister_skcipher(&t_alg->skcipher); 5058 } 5059 5060 if (hash_list.next) { 5061 struct caam_hash_alg *t_hash_alg, *p; 5062 5063 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5064 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5065 list_del(&t_hash_alg->entry); 5066 kfree(t_hash_alg); 5067 } 5068 } 5069 5070 dpaa2_dpseci_disable(priv); 5071 dpaa2_dpseci_dpio_free(priv); 5072 dpaa2_dpseci_free(priv); 5073 free_percpu(priv->ppriv); 5074 fsl_mc_portal_free(priv->mc_io); 5075 kmem_cache_destroy(qi_cache); 5076 5077 return 0; 5078 } 5079 5080 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5081 { 5082 struct dpaa2_fd fd; 5083 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5084 int err = 0, i, id; 5085 5086 if (IS_ERR(req)) 5087 return PTR_ERR(req); 5088 5089 if (priv->cscn_mem) { 5090 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5091 DPAA2_CSCN_SIZE, 5092 DMA_FROM_DEVICE); 5093 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { 5094 dev_dbg_ratelimited(dev, "Dropping request\n"); 5095 return -EBUSY; 5096 } 5097 } 5098 5099 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5100 5101 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5102 DMA_BIDIRECTIONAL); 5103 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5104 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5105 goto err_out; 5106 } 5107 5108 memset(&fd, 0, sizeof(fd)); 5109 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5110 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5111 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5112 dpaa2_fd_set_flc(&fd, req->flc_dma); 5113 5114 /* 5115 * There is no guarantee that preemption is disabled here, 5116 * thus take action. 5117 */ 5118 preempt_disable(); 5119 id = smp_processor_id() % priv->dpseci_attr.num_tx_queues; 5120 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5121 err = dpaa2_io_service_enqueue_fq(NULL, 5122 priv->tx_queue_attr[id].fqid, 5123 &fd); 5124 if (err != -EBUSY) 5125 break; 5126 } 5127 preempt_enable(); 5128 5129 if (unlikely(err)) { 5130 dev_err(dev, "Error enqueuing frame: %d\n", err); 5131 goto err_out; 5132 } 5133 5134 return -EINPROGRESS; 5135 5136 err_out: 5137 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5138 DMA_BIDIRECTIONAL); 5139 return -EIO; 5140 } 5141 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5142 5143 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5144 { 5145 .vendor = FSL_MC_VENDOR_FREESCALE, 5146 .obj_type = "dpseci", 5147 }, 5148 { .vendor = 0x0 } 5149 }; 5150 5151 static struct fsl_mc_driver dpaa2_caam_driver = { 5152 .driver = { 5153 .name = KBUILD_MODNAME, 5154 .owner = THIS_MODULE, 5155 }, 5156 .probe = dpaa2_caam_probe, 5157 .remove = dpaa2_caam_remove, 5158 .match_id_table = dpaa2_caam_match_id_table 5159 }; 5160 5161 MODULE_LICENSE("Dual BSD/GPL"); 5162 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5163 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5164 5165 module_fsl_mc_driver(dpaa2_caam_driver); 5166