1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Marvell. */ 3 4 #include <crypto/aes.h> 5 #include <crypto/authenc.h> 6 #include <crypto/cryptd.h> 7 #include <crypto/des.h> 8 #include <crypto/internal/aead.h> 9 #include <crypto/sha1.h> 10 #include <crypto/sha2.h> 11 #include <crypto/xts.h> 12 #include <crypto/gcm.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/sort.h> 15 #include <linux/module.h> 16 #include "otx2_cptvf.h" 17 #include "otx2_cptvf_algs.h" 18 #include "otx2_cpt_reqmgr.h" 19 20 /* Size of salt in AES GCM mode */ 21 #define AES_GCM_SALT_SIZE 4 22 /* Size of IV in AES GCM mode */ 23 #define AES_GCM_IV_SIZE 8 24 /* Size of ICV (Integrity Check Value) in AES GCM mode */ 25 #define AES_GCM_ICV_SIZE 16 26 /* Offset of IV in AES GCM mode */ 27 #define AES_GCM_IV_OFFSET 8 28 #define CONTROL_WORD_LEN 8 29 #define KEY2_OFFSET 48 30 #define DMA_MODE_FLAG(dma_mode) \ 31 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0) 32 33 /* Truncated SHA digest size */ 34 #define SHA1_TRUNC_DIGEST_SIZE 12 35 #define SHA256_TRUNC_DIGEST_SIZE 16 36 #define SHA384_TRUNC_DIGEST_SIZE 24 37 #define SHA512_TRUNC_DIGEST_SIZE 32 38 39 static DEFINE_MUTEX(mutex); 40 static int is_crypto_registered; 41 42 struct cpt_device_desc { 43 struct pci_dev *dev; 44 int num_queues; 45 }; 46 47 struct cpt_device_table { 48 atomic_t count; 49 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM]; 50 }; 51 52 static struct cpt_device_table se_devices = { 53 .count = ATOMIC_INIT(0) 54 }; 55 56 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg); 57 58 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) 59 { 60 int count; 61 62 count = atomic_read(&se_devices.count); 63 if (count < 1) 64 return -ENODEV; 65 66 *cpu_num = get_cpu(); 67 /* 68 * On OcteonTX2 platform CPT instruction queue is bound to each 69 * local function LF, in turn LFs can be attached to PF 70 * or VF therefore we always use first device. We get maximum 71 * performance if one CPT queue is available for each cpu 72 * otherwise CPT queues need to be shared between cpus. 73 */ 74 if (*cpu_num >= se_devices.desc[0].num_queues) 75 *cpu_num %= se_devices.desc[0].num_queues; 76 *pdev = se_devices.desc[0].dev; 77 78 put_cpu(); 79 80 return 0; 81 } 82 83 static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) 84 { 85 struct otx2_cpt_req_ctx *rctx; 86 struct aead_request *req; 87 struct crypto_aead *tfm; 88 89 req = container_of(cpt_req->areq, struct aead_request, base); 90 tfm = crypto_aead_reqtfm(req); 91 rctx = aead_request_ctx_dma(req); 92 if (memcmp(rctx->fctx.hmac.s.hmac_calc, 93 rctx->fctx.hmac.s.hmac_recv, 94 crypto_aead_authsize(tfm)) != 0) 95 return -EBADMSG; 96 97 return 0; 98 } 99 100 static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2) 101 { 102 struct otx2_cpt_inst_info *inst_info = arg2; 103 struct crypto_async_request *areq = arg1; 104 struct otx2_cpt_req_info *cpt_req; 105 struct pci_dev *pdev; 106 107 if (inst_info) { 108 cpt_req = inst_info->req; 109 if (!status) { 110 /* 111 * When selected cipher is NULL we need to manually 112 * verify whether calculated hmac value matches 113 * received hmac value 114 */ 115 if (cpt_req->req_type == 116 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ && 117 !cpt_req->is_enc) 118 status = validate_hmac_cipher_null(cpt_req); 119 } 120 pdev = inst_info->pdev; 121 otx2_cpt_info_destroy(pdev, inst_info); 122 } 123 if (areq) 124 crypto_request_complete(areq, status); 125 } 126 127 static void output_iv_copyback(struct crypto_async_request *areq) 128 { 129 struct otx2_cpt_req_info *req_info; 130 struct otx2_cpt_req_ctx *rctx; 131 struct skcipher_request *sreq; 132 struct crypto_skcipher *stfm; 133 struct otx2_cpt_enc_ctx *ctx; 134 u32 start, ivsize; 135 136 sreq = container_of(areq, struct skcipher_request, base); 137 stfm = crypto_skcipher_reqtfm(sreq); 138 ctx = crypto_skcipher_ctx(stfm); 139 if (ctx->cipher_type == OTX2_CPT_AES_CBC || 140 ctx->cipher_type == OTX2_CPT_DES3_CBC) { 141 rctx = skcipher_request_ctx_dma(sreq); 142 req_info = &rctx->cpt_req; 143 ivsize = crypto_skcipher_ivsize(stfm); 144 start = sreq->cryptlen - ivsize; 145 146 if (req_info->is_enc) { 147 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, 148 ivsize, 0); 149 } else { 150 if (sreq->src != sreq->dst) { 151 scatterwalk_map_and_copy(sreq->iv, sreq->src, 152 start, ivsize, 0); 153 } else { 154 memcpy(sreq->iv, req_info->iv_out, ivsize); 155 kfree(req_info->iv_out); 156 } 157 } 158 } 159 } 160 161 static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2) 162 { 163 struct otx2_cpt_inst_info *inst_info = arg2; 164 struct crypto_async_request *areq = arg1; 165 struct pci_dev *pdev; 166 167 if (areq) { 168 if (!status) 169 output_iv_copyback(areq); 170 if (inst_info) { 171 pdev = inst_info->pdev; 172 otx2_cpt_info_destroy(pdev, inst_info); 173 } 174 crypto_request_complete(areq, status); 175 } 176 } 177 178 static inline void update_input_data(struct otx2_cpt_req_info *req_info, 179 struct scatterlist *inp_sg, 180 u32 nbytes, u32 *argcnt) 181 { 182 req_info->req.dlen += nbytes; 183 184 while (nbytes) { 185 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length; 186 u8 *ptr = sg_virt(inp_sg); 187 188 req_info->in[*argcnt].vptr = (void *)ptr; 189 req_info->in[*argcnt].size = len; 190 nbytes -= len; 191 ++(*argcnt); 192 inp_sg = sg_next(inp_sg); 193 } 194 } 195 196 static inline void update_output_data(struct otx2_cpt_req_info *req_info, 197 struct scatterlist *outp_sg, 198 u32 offset, u32 nbytes, u32 *argcnt) 199 { 200 u32 len, sg_len; 201 u8 *ptr; 202 203 req_info->rlen += nbytes; 204 205 while (nbytes) { 206 sg_len = outp_sg->length - offset; 207 len = (nbytes < sg_len) ? nbytes : sg_len; 208 ptr = sg_virt(outp_sg); 209 210 req_info->out[*argcnt].vptr = (void *) (ptr + offset); 211 req_info->out[*argcnt].size = len; 212 nbytes -= len; 213 ++(*argcnt); 214 offset = 0; 215 outp_sg = sg_next(outp_sg); 216 } 217 } 218 219 static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, 220 u32 *argcnt) 221 { 222 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 223 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 224 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 225 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 226 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; 227 int ivsize = crypto_skcipher_ivsize(stfm); 228 u32 start = req->cryptlen - ivsize; 229 gfp_t flags; 230 231 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 232 GFP_KERNEL : GFP_ATOMIC; 233 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 234 req_info->ctrl.s.se_req = 1; 235 236 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | 237 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 238 if (enc) { 239 req_info->req.opcode.s.minor = 2; 240 } else { 241 req_info->req.opcode.s.minor = 3; 242 if ((ctx->cipher_type == OTX2_CPT_AES_CBC || 243 ctx->cipher_type == OTX2_CPT_DES3_CBC) && 244 req->src == req->dst) { 245 req_info->iv_out = kmalloc(ivsize, flags); 246 if (!req_info->iv_out) 247 return -ENOMEM; 248 249 scatterwalk_map_and_copy(req_info->iv_out, req->src, 250 start, ivsize, 0); 251 } 252 } 253 /* Encryption data length */ 254 req_info->req.param1 = req->cryptlen; 255 /* Authentication data length */ 256 req_info->req.param2 = 0; 257 258 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 259 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 260 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; 261 262 if (ctx->cipher_type == OTX2_CPT_AES_XTS) 263 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 264 else 265 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 266 267 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); 268 269 cpu_to_be64s(&fctx->enc.enc_ctrl.u); 270 271 /* 272 * Storing Packet Data Information in offset 273 * Control Word First 8 bytes 274 */ 275 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 276 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 277 req_info->req.dlen += CONTROL_WORD_LEN; 278 ++(*argcnt); 279 280 req_info->in[*argcnt].vptr = (u8 *)fctx; 281 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); 282 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); 283 284 ++(*argcnt); 285 286 return 0; 287 } 288 289 static inline int create_input_list(struct skcipher_request *req, u32 enc, 290 u32 enc_iv_len) 291 { 292 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 293 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 294 u32 argcnt = 0; 295 int ret; 296 297 ret = create_ctx_hdr(req, enc, &argcnt); 298 if (ret) 299 return ret; 300 301 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 302 req_info->in_cnt = argcnt; 303 304 return 0; 305 } 306 307 static inline void create_output_list(struct skcipher_request *req, 308 u32 enc_iv_len) 309 { 310 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 311 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 312 u32 argcnt = 0; 313 314 /* 315 * OUTPUT Buffer Processing 316 * AES encryption/decryption output would be 317 * received in the following format 318 * 319 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 320 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 321 */ 322 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); 323 req_info->out_cnt = argcnt; 324 } 325 326 static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc) 327 { 328 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 329 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 330 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 331 int ret; 332 333 if (ctx->fbk_cipher) { 334 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher); 335 skcipher_request_set_callback(&rctx->sk_fbk_req, 336 req->base.flags, 337 req->base.complete, 338 req->base.data); 339 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src, 340 req->dst, req->cryptlen, req->iv); 341 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) : 342 crypto_skcipher_decrypt(&rctx->sk_fbk_req); 343 } else { 344 ret = -EINVAL; 345 } 346 return ret; 347 } 348 349 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) 350 { 351 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 352 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 353 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 354 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 355 u32 enc_iv_len = crypto_skcipher_ivsize(stfm); 356 struct pci_dev *pdev; 357 int status, cpu_num; 358 359 if (req->cryptlen == 0) 360 return 0; 361 362 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len)) 363 return -EINVAL; 364 365 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE) 366 return skcipher_do_fallback(req, enc); 367 368 /* Clear control words */ 369 rctx->ctrl_word.flags = 0; 370 rctx->fctx.enc.enc_ctrl.u = 0; 371 372 status = create_input_list(req, enc, enc_iv_len); 373 if (status) 374 return status; 375 create_output_list(req, enc_iv_len); 376 377 status = get_se_device(&pdev, &cpu_num); 378 if (status) 379 return status; 380 381 req_info->callback = otx2_cpt_skcipher_callback; 382 req_info->areq = &req->base; 383 req_info->req_type = OTX2_CPT_ENC_DEC_REQ; 384 req_info->is_enc = enc; 385 req_info->is_trunc_hmac = false; 386 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); 387 388 /* 389 * We perform an asynchronous send and once 390 * the request is completed the driver would 391 * intimate through registered call back functions 392 */ 393 status = otx2_cpt_do_request(pdev, req_info, cpu_num); 394 395 return status; 396 } 397 398 static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req) 399 { 400 return cpt_enc_dec(req, true); 401 } 402 403 static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req) 404 { 405 return cpt_enc_dec(req, false); 406 } 407 408 static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, 409 const u8 *key, u32 keylen) 410 { 411 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 412 const u8 *key2 = key + (keylen / 2); 413 const u8 *key1 = key; 414 int ret; 415 416 ret = xts_verify_key(tfm, key, keylen); 417 if (ret) 418 return ret; 419 ctx->key_len = keylen; 420 ctx->enc_align_len = 1; 421 memcpy(ctx->enc_key, key1, keylen / 2); 422 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 423 ctx->cipher_type = OTX2_CPT_AES_XTS; 424 switch (ctx->key_len) { 425 case 2 * AES_KEYSIZE_128: 426 ctx->key_type = OTX2_CPT_AES_128_BIT; 427 break; 428 case 2 * AES_KEYSIZE_192: 429 ctx->key_type = OTX2_CPT_AES_192_BIT; 430 break; 431 case 2 * AES_KEYSIZE_256: 432 ctx->key_type = OTX2_CPT_AES_256_BIT; 433 break; 434 default: 435 return -EINVAL; 436 } 437 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 438 } 439 440 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 441 u32 keylen, u8 cipher_type) 442 { 443 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 444 445 if (keylen != DES3_EDE_KEY_SIZE) 446 return -EINVAL; 447 448 ctx->key_len = keylen; 449 ctx->cipher_type = cipher_type; 450 ctx->enc_align_len = 8; 451 452 memcpy(ctx->enc_key, key, keylen); 453 454 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 455 } 456 457 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 458 u32 keylen, u8 cipher_type) 459 { 460 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 461 462 switch (keylen) { 463 case AES_KEYSIZE_128: 464 ctx->key_type = OTX2_CPT_AES_128_BIT; 465 break; 466 case AES_KEYSIZE_192: 467 ctx->key_type = OTX2_CPT_AES_192_BIT; 468 break; 469 case AES_KEYSIZE_256: 470 ctx->key_type = OTX2_CPT_AES_256_BIT; 471 break; 472 default: 473 return -EINVAL; 474 } 475 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB) 476 ctx->enc_align_len = 16; 477 else 478 ctx->enc_align_len = 1; 479 480 ctx->key_len = keylen; 481 ctx->cipher_type = cipher_type; 482 483 memcpy(ctx->enc_key, key, keylen); 484 485 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 486 } 487 488 static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, 489 const u8 *key, u32 keylen) 490 { 491 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC); 492 } 493 494 static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, 495 const u8 *key, u32 keylen) 496 { 497 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB); 498 } 499 500 static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, 501 const u8 *key, u32 keylen) 502 { 503 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC); 504 } 505 506 static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, 507 const u8 *key, u32 keylen) 508 { 509 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB); 510 } 511 512 static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx, 513 struct crypto_alg *alg) 514 { 515 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 516 ctx->fbk_cipher = 517 crypto_alloc_skcipher(alg->cra_name, 0, 518 CRYPTO_ALG_ASYNC | 519 CRYPTO_ALG_NEED_FALLBACK); 520 if (IS_ERR(ctx->fbk_cipher)) { 521 pr_err("%s() failed to allocate fallback for %s\n", 522 __func__, alg->cra_name); 523 return PTR_ERR(ctx->fbk_cipher); 524 } 525 } 526 return 0; 527 } 528 529 static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) 530 { 531 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 532 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); 533 struct crypto_alg *alg = tfm->__crt_alg; 534 535 memset(ctx, 0, sizeof(*ctx)); 536 /* 537 * Additional memory for skcipher_request is 538 * allocated since the cryptd daemon uses 539 * this memory for request_ctx information 540 */ 541 crypto_skcipher_set_reqsize_dma( 542 stfm, sizeof(struct otx2_cpt_req_ctx) + 543 sizeof(struct skcipher_request)); 544 545 return cpt_skcipher_fallback_init(ctx, alg); 546 } 547 548 static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) 549 { 550 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 551 552 if (ctx->fbk_cipher) { 553 crypto_free_skcipher(ctx->fbk_cipher); 554 ctx->fbk_cipher = NULL; 555 } 556 } 557 558 static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, 559 struct crypto_alg *alg) 560 { 561 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 562 ctx->fbk_cipher = 563 crypto_alloc_aead(alg->cra_name, 0, 564 CRYPTO_ALG_ASYNC | 565 CRYPTO_ALG_NEED_FALLBACK); 566 if (IS_ERR(ctx->fbk_cipher)) { 567 pr_err("%s() failed to allocate fallback for %s\n", 568 __func__, alg->cra_name); 569 return PTR_ERR(ctx->fbk_cipher); 570 } 571 } 572 return 0; 573 } 574 575 static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) 576 { 577 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm); 578 struct crypto_tfm *tfm = crypto_aead_tfm(atfm); 579 struct crypto_alg *alg = tfm->__crt_alg; 580 581 ctx->cipher_type = cipher_type; 582 ctx->mac_type = mac_type; 583 584 switch (ctx->mac_type) { 585 case OTX2_CPT_SHA1: 586 ctx->hashalg = crypto_alloc_shash("sha1", 0, 0); 587 break; 588 589 case OTX2_CPT_SHA256: 590 ctx->hashalg = crypto_alloc_shash("sha256", 0, 0); 591 break; 592 593 case OTX2_CPT_SHA384: 594 ctx->hashalg = crypto_alloc_shash("sha384", 0, 0); 595 break; 596 597 case OTX2_CPT_SHA512: 598 ctx->hashalg = crypto_alloc_shash("sha512", 0, 0); 599 break; 600 } 601 602 if (IS_ERR(ctx->hashalg)) 603 return PTR_ERR(ctx->hashalg); 604 605 if (ctx->hashalg) { 606 ctx->sdesc = alloc_sdesc(ctx->hashalg); 607 if (!ctx->sdesc) { 608 crypto_free_shash(ctx->hashalg); 609 return -ENOMEM; 610 } 611 } 612 613 /* 614 * When selected cipher is NULL we use HMAC opcode instead of 615 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms 616 * for calculating ipad and opad 617 */ 618 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL && ctx->hashalg) { 619 int ss = crypto_shash_statesize(ctx->hashalg); 620 621 ctx->ipad = kzalloc(ss, GFP_KERNEL); 622 if (!ctx->ipad) { 623 kfree(ctx->sdesc); 624 crypto_free_shash(ctx->hashalg); 625 return -ENOMEM; 626 } 627 628 ctx->opad = kzalloc(ss, GFP_KERNEL); 629 if (!ctx->opad) { 630 kfree(ctx->ipad); 631 kfree(ctx->sdesc); 632 crypto_free_shash(ctx->hashalg); 633 return -ENOMEM; 634 } 635 } 636 switch (ctx->cipher_type) { 637 case OTX2_CPT_AES_CBC: 638 case OTX2_CPT_AES_ECB: 639 ctx->enc_align_len = 16; 640 break; 641 case OTX2_CPT_DES3_CBC: 642 case OTX2_CPT_DES3_ECB: 643 ctx->enc_align_len = 8; 644 break; 645 case OTX2_CPT_AES_GCM: 646 case OTX2_CPT_CIPHER_NULL: 647 ctx->enc_align_len = 1; 648 break; 649 } 650 crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx)); 651 652 return cpt_aead_fallback_init(ctx, alg); 653 } 654 655 static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) 656 { 657 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1); 658 } 659 660 static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) 661 { 662 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256); 663 } 664 665 static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) 666 { 667 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384); 668 } 669 670 static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) 671 { 672 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512); 673 } 674 675 static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) 676 { 677 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1); 678 } 679 680 static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) 681 { 682 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256); 683 } 684 685 static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) 686 { 687 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384); 688 } 689 690 static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) 691 { 692 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512); 693 } 694 695 static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) 696 { 697 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL); 698 } 699 700 static void otx2_cpt_aead_exit(struct crypto_aead *tfm) 701 { 702 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 703 704 kfree(ctx->ipad); 705 kfree(ctx->opad); 706 crypto_free_shash(ctx->hashalg); 707 kfree(ctx->sdesc); 708 709 if (ctx->fbk_cipher) { 710 crypto_free_aead(ctx->fbk_cipher); 711 ctx->fbk_cipher = NULL; 712 } 713 } 714 715 static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, 716 unsigned int authsize) 717 { 718 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 719 720 if (crypto_rfc4106_check_authsize(authsize)) 721 return -EINVAL; 722 723 tfm->authsize = authsize; 724 /* Set authsize for fallback case */ 725 if (ctx->fbk_cipher) 726 ctx->fbk_cipher->authsize = authsize; 727 728 return 0; 729 } 730 731 static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm, 732 unsigned int authsize) 733 { 734 tfm->authsize = authsize; 735 736 return 0; 737 } 738 739 static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm, 740 unsigned int authsize) 741 { 742 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 743 744 ctx->is_trunc_hmac = true; 745 tfm->authsize = authsize; 746 747 return 0; 748 } 749 750 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) 751 { 752 struct otx2_cpt_sdesc *sdesc; 753 int size; 754 755 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); 756 sdesc = kmalloc(size, GFP_KERNEL); 757 if (!sdesc) 758 return NULL; 759 760 sdesc->shash.tfm = alg; 761 762 return sdesc; 763 } 764 765 static inline void swap_data32(void *buf, u32 len) 766 { 767 cpu_to_be32_array(buf, buf, len / 4); 768 } 769 770 static inline void swap_data64(void *buf, u32 len) 771 { 772 u64 *src = buf; 773 int i = 0; 774 775 for (i = 0 ; i < len / 8; i++, src++) 776 cpu_to_be64s(src); 777 } 778 779 static int swap_pad(u8 mac_type, u8 *pad) 780 { 781 struct sha512_state *sha512; 782 struct sha256_state *sha256; 783 struct sha1_state *sha1; 784 785 switch (mac_type) { 786 case OTX2_CPT_SHA1: 787 sha1 = (struct sha1_state *)pad; 788 swap_data32(sha1->state, SHA1_DIGEST_SIZE); 789 break; 790 791 case OTX2_CPT_SHA256: 792 sha256 = (struct sha256_state *)pad; 793 swap_data32(sha256->state, SHA256_DIGEST_SIZE); 794 break; 795 796 case OTX2_CPT_SHA384: 797 case OTX2_CPT_SHA512: 798 sha512 = (struct sha512_state *)pad; 799 swap_data64(sha512->state, SHA512_DIGEST_SIZE); 800 break; 801 802 default: 803 return -EINVAL; 804 } 805 806 return 0; 807 } 808 809 static int aead_hmac_init(struct crypto_aead *cipher, 810 struct crypto_authenc_keys *keys) 811 { 812 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 813 int ds = crypto_shash_digestsize(ctx->hashalg); 814 int bs = crypto_shash_blocksize(ctx->hashalg); 815 int authkeylen = keys->authkeylen; 816 u8 *ipad = NULL, *opad = NULL; 817 int icount = 0; 818 int ret; 819 820 if (authkeylen > bs) { 821 ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey, 822 authkeylen, ctx->key); 823 if (ret) 824 goto calc_fail; 825 826 authkeylen = ds; 827 } else 828 memcpy(ctx->key, keys->authkey, authkeylen); 829 830 ctx->enc_key_len = keys->enckeylen; 831 ctx->auth_key_len = authkeylen; 832 833 if (ctx->cipher_type == OTX2_CPT_CIPHER_NULL) 834 return keys->enckeylen ? -EINVAL : 0; 835 836 switch (keys->enckeylen) { 837 case AES_KEYSIZE_128: 838 ctx->key_type = OTX2_CPT_AES_128_BIT; 839 break; 840 case AES_KEYSIZE_192: 841 ctx->key_type = OTX2_CPT_AES_192_BIT; 842 break; 843 case AES_KEYSIZE_256: 844 ctx->key_type = OTX2_CPT_AES_256_BIT; 845 break; 846 default: 847 /* Invalid key length */ 848 return -EINVAL; 849 } 850 851 memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen); 852 853 ipad = ctx->ipad; 854 opad = ctx->opad; 855 856 memcpy(ipad, ctx->key, authkeylen); 857 memset(ipad + authkeylen, 0, bs - authkeylen); 858 memcpy(opad, ipad, bs); 859 860 for (icount = 0; icount < bs; icount++) { 861 ipad[icount] ^= 0x36; 862 opad[icount] ^= 0x5c; 863 } 864 865 /* 866 * Partial Hash calculated from the software 867 * algorithm is retrieved for IPAD & OPAD 868 */ 869 870 /* IPAD Calculation */ 871 crypto_shash_init(&ctx->sdesc->shash); 872 crypto_shash_update(&ctx->sdesc->shash, ipad, bs); 873 crypto_shash_export(&ctx->sdesc->shash, ipad); 874 ret = swap_pad(ctx->mac_type, ipad); 875 if (ret) 876 goto calc_fail; 877 878 /* OPAD Calculation */ 879 crypto_shash_init(&ctx->sdesc->shash); 880 crypto_shash_update(&ctx->sdesc->shash, opad, bs); 881 crypto_shash_export(&ctx->sdesc->shash, opad); 882 ret = swap_pad(ctx->mac_type, opad); 883 884 calc_fail: 885 return ret; 886 } 887 888 static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, 889 const unsigned char *key, 890 unsigned int keylen) 891 { 892 struct crypto_authenc_keys authenc_keys; 893 894 return crypto_authenc_extractkeys(&authenc_keys, key, keylen) ?: 895 aead_hmac_init(cipher, &authenc_keys); 896 } 897 898 static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, 899 const unsigned char *key, 900 unsigned int keylen) 901 { 902 return otx2_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen); 903 } 904 905 static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, 906 const unsigned char *key, 907 unsigned int keylen) 908 { 909 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 910 911 /* 912 * For aes gcm we expect to get encryption key (16, 24, 32 bytes) 913 * and salt (4 bytes) 914 */ 915 switch (keylen) { 916 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: 917 ctx->key_type = OTX2_CPT_AES_128_BIT; 918 ctx->enc_key_len = AES_KEYSIZE_128; 919 break; 920 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: 921 ctx->key_type = OTX2_CPT_AES_192_BIT; 922 ctx->enc_key_len = AES_KEYSIZE_192; 923 break; 924 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: 925 ctx->key_type = OTX2_CPT_AES_256_BIT; 926 ctx->enc_key_len = AES_KEYSIZE_256; 927 break; 928 default: 929 /* Invalid key and salt length */ 930 return -EINVAL; 931 } 932 933 /* Store encryption key and salt */ 934 memcpy(ctx->key, key, keylen); 935 936 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen); 937 } 938 939 static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc, 940 u32 *argcnt) 941 { 942 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 943 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 944 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 945 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 946 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; 947 int mac_len = crypto_aead_authsize(tfm); 948 int ds; 949 950 rctx->ctrl_word.e.enc_data_offset = req->assoclen; 951 952 switch (ctx->cipher_type) { 953 case OTX2_CPT_AES_CBC: 954 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8)) 955 return -EINVAL; 956 957 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; 958 /* Copy encryption key to context */ 959 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, 960 ctx->enc_key_len); 961 /* Copy IV to context */ 962 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); 963 964 ds = crypto_shash_digestsize(ctx->hashalg); 965 if (ctx->mac_type == OTX2_CPT_SHA384) 966 ds = SHA512_DIGEST_SIZE; 967 if (ctx->ipad) 968 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); 969 if (ctx->opad) 970 memcpy(fctx->hmac.e.opad, ctx->opad, ds); 971 break; 972 973 case OTX2_CPT_AES_GCM: 974 if (crypto_ipsec_check_assoclen(req->assoclen)) 975 return -EINVAL; 976 977 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR; 978 /* Copy encryption key to context */ 979 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); 980 /* Copy salt to context */ 981 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, 982 AES_GCM_SALT_SIZE); 983 984 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; 985 break; 986 987 default: 988 /* Unknown cipher type */ 989 return -EINVAL; 990 } 991 cpu_to_be64s(&rctx->ctrl_word.flags); 992 993 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 994 req_info->ctrl.s.se_req = 1; 995 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | 996 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 997 if (enc) { 998 req_info->req.opcode.s.minor = 2; 999 req_info->req.param1 = req->cryptlen; 1000 req_info->req.param2 = req->cryptlen + req->assoclen; 1001 } else { 1002 req_info->req.opcode.s.minor = 3; 1003 req_info->req.param1 = req->cryptlen - mac_len; 1004 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; 1005 } 1006 1007 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 1008 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 1009 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; 1010 fctx->enc.enc_ctrl.e.mac_len = mac_len; 1011 cpu_to_be64s(&fctx->enc.enc_ctrl.u); 1012 1013 /* 1014 * Storing Packet Data Information in offset 1015 * Control Word First 8 bytes 1016 */ 1017 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 1018 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 1019 req_info->req.dlen += CONTROL_WORD_LEN; 1020 ++(*argcnt); 1021 1022 req_info->in[*argcnt].vptr = (u8 *)fctx; 1023 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); 1024 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); 1025 ++(*argcnt); 1026 1027 return 0; 1028 } 1029 1030 static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, 1031 u32 enc) 1032 { 1033 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1034 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1035 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 1036 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1037 1038 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 1039 req_info->ctrl.s.se_req = 1; 1040 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC | 1041 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 1042 req_info->is_trunc_hmac = ctx->is_trunc_hmac; 1043 1044 req_info->req.opcode.s.minor = 0; 1045 req_info->req.param1 = ctx->auth_key_len; 1046 req_info->req.param2 = ctx->mac_type << 8; 1047 1048 /* Add authentication key */ 1049 req_info->in[*argcnt].vptr = ctx->key; 1050 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); 1051 req_info->req.dlen += round_up(ctx->auth_key_len, 8); 1052 ++(*argcnt); 1053 } 1054 1055 static inline int create_aead_input_list(struct aead_request *req, u32 enc) 1056 { 1057 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1058 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1059 u32 inputlen = req->cryptlen + req->assoclen; 1060 u32 status, argcnt = 0; 1061 1062 status = create_aead_ctx_hdr(req, enc, &argcnt); 1063 if (status) 1064 return status; 1065 update_input_data(req_info, req->src, inputlen, &argcnt); 1066 req_info->in_cnt = argcnt; 1067 1068 return 0; 1069 } 1070 1071 static inline void create_aead_output_list(struct aead_request *req, u32 enc, 1072 u32 mac_len) 1073 { 1074 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1075 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1076 u32 argcnt = 0, outputlen = 0; 1077 1078 if (enc) 1079 outputlen = req->cryptlen + req->assoclen + mac_len; 1080 else 1081 outputlen = req->cryptlen + req->assoclen - mac_len; 1082 1083 update_output_data(req_info, req->dst, 0, outputlen, &argcnt); 1084 req_info->out_cnt = argcnt; 1085 } 1086 1087 static inline void create_aead_null_input_list(struct aead_request *req, 1088 u32 enc, u32 mac_len) 1089 { 1090 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1091 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1092 u32 inputlen, argcnt = 0; 1093 1094 if (enc) 1095 inputlen = req->cryptlen + req->assoclen; 1096 else 1097 inputlen = req->cryptlen + req->assoclen - mac_len; 1098 1099 create_hmac_ctx_hdr(req, &argcnt, enc); 1100 update_input_data(req_info, req->src, inputlen, &argcnt); 1101 req_info->in_cnt = argcnt; 1102 } 1103 1104 static inline int create_aead_null_output_list(struct aead_request *req, 1105 u32 enc, u32 mac_len) 1106 { 1107 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1108 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1109 struct scatterlist *dst; 1110 u8 *ptr = NULL; 1111 int argcnt = 0, status, offset; 1112 u32 inputlen; 1113 1114 if (enc) 1115 inputlen = req->cryptlen + req->assoclen; 1116 else 1117 inputlen = req->cryptlen + req->assoclen - mac_len; 1118 1119 /* 1120 * If source and destination are different 1121 * then copy payload to destination 1122 */ 1123 if (req->src != req->dst) { 1124 1125 ptr = kmalloc(inputlen, (req_info->areq->flags & 1126 CRYPTO_TFM_REQ_MAY_SLEEP) ? 1127 GFP_KERNEL : GFP_ATOMIC); 1128 if (!ptr) 1129 return -ENOMEM; 1130 1131 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, 1132 inputlen); 1133 if (status != inputlen) { 1134 status = -EINVAL; 1135 goto error_free; 1136 } 1137 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, 1138 inputlen); 1139 if (status != inputlen) { 1140 status = -EINVAL; 1141 goto error_free; 1142 } 1143 kfree(ptr); 1144 } 1145 1146 if (enc) { 1147 /* 1148 * In an encryption scenario hmac needs 1149 * to be appended after payload 1150 */ 1151 dst = req->dst; 1152 offset = inputlen; 1153 while (offset >= dst->length) { 1154 offset -= dst->length; 1155 dst = sg_next(dst); 1156 if (!dst) 1157 return -ENOENT; 1158 } 1159 1160 update_output_data(req_info, dst, offset, mac_len, &argcnt); 1161 } else { 1162 /* 1163 * In a decryption scenario calculated hmac for received 1164 * payload needs to be compare with hmac received 1165 */ 1166 status = sg_copy_buffer(req->src, sg_nents(req->src), 1167 rctx->fctx.hmac.s.hmac_recv, mac_len, 1168 inputlen, true); 1169 if (status != mac_len) 1170 return -EINVAL; 1171 1172 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; 1173 req_info->out[argcnt].size = mac_len; 1174 argcnt++; 1175 } 1176 1177 req_info->out_cnt = argcnt; 1178 return 0; 1179 1180 error_free: 1181 kfree(ptr); 1182 return status; 1183 } 1184 1185 static int aead_do_fallback(struct aead_request *req, bool is_enc) 1186 { 1187 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1188 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1189 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead); 1190 int ret; 1191 1192 if (ctx->fbk_cipher) { 1193 /* Store the cipher tfm and then use the fallback tfm */ 1194 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher); 1195 aead_request_set_callback(&rctx->fbk_req, req->base.flags, 1196 req->base.complete, req->base.data); 1197 aead_request_set_crypt(&rctx->fbk_req, req->src, 1198 req->dst, req->cryptlen, req->iv); 1199 aead_request_set_ad(&rctx->fbk_req, req->assoclen); 1200 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : 1201 crypto_aead_decrypt(&rctx->fbk_req); 1202 } else { 1203 ret = -EINVAL; 1204 } 1205 1206 return ret; 1207 } 1208 1209 static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) 1210 { 1211 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1212 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1213 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1214 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 1215 struct pci_dev *pdev; 1216 int status, cpu_num; 1217 1218 /* Clear control words */ 1219 rctx->ctrl_word.flags = 0; 1220 rctx->fctx.enc.enc_ctrl.u = 0; 1221 1222 req_info->callback = otx2_cpt_aead_callback; 1223 req_info->areq = &req->base; 1224 req_info->req_type = reg_type; 1225 req_info->is_enc = enc; 1226 req_info->is_trunc_hmac = false; 1227 1228 switch (reg_type) { 1229 case OTX2_CPT_AEAD_ENC_DEC_REQ: 1230 status = create_aead_input_list(req, enc); 1231 if (status) 1232 return status; 1233 create_aead_output_list(req, enc, crypto_aead_authsize(tfm)); 1234 break; 1235 1236 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ: 1237 create_aead_null_input_list(req, enc, 1238 crypto_aead_authsize(tfm)); 1239 status = create_aead_null_output_list(req, enc, 1240 crypto_aead_authsize(tfm)); 1241 if (status) 1242 return status; 1243 break; 1244 1245 default: 1246 return -EINVAL; 1247 } 1248 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len)) 1249 return -EINVAL; 1250 1251 if (!req_info->req.param2 || 1252 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) || 1253 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE)) 1254 return aead_do_fallback(req, enc); 1255 1256 status = get_se_device(&pdev, &cpu_num); 1257 if (status) 1258 return status; 1259 1260 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); 1261 1262 /* 1263 * We perform an asynchronous send and once 1264 * the request is completed the driver would 1265 * intimate through registered call back functions 1266 */ 1267 return otx2_cpt_do_request(pdev, req_info, cpu_num); 1268 } 1269 1270 static int otx2_cpt_aead_encrypt(struct aead_request *req) 1271 { 1272 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true); 1273 } 1274 1275 static int otx2_cpt_aead_decrypt(struct aead_request *req) 1276 { 1277 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false); 1278 } 1279 1280 static int otx2_cpt_aead_null_encrypt(struct aead_request *req) 1281 { 1282 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true); 1283 } 1284 1285 static int otx2_cpt_aead_null_decrypt(struct aead_request *req) 1286 { 1287 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false); 1288 } 1289 1290 static struct skcipher_alg otx2_cpt_skciphers[] = { { 1291 .base.cra_name = "xts(aes)", 1292 .base.cra_driver_name = "cpt_xts_aes", 1293 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1294 .base.cra_blocksize = AES_BLOCK_SIZE, 1295 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1296 .base.cra_alignmask = 7, 1297 .base.cra_priority = 4001, 1298 .base.cra_module = THIS_MODULE, 1299 1300 .init = otx2_cpt_enc_dec_init, 1301 .exit = otx2_cpt_skcipher_exit, 1302 .ivsize = AES_BLOCK_SIZE, 1303 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1304 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1305 .setkey = otx2_cpt_skcipher_xts_setkey, 1306 .encrypt = otx2_cpt_skcipher_encrypt, 1307 .decrypt = otx2_cpt_skcipher_decrypt, 1308 }, { 1309 .base.cra_name = "cbc(aes)", 1310 .base.cra_driver_name = "cpt_cbc_aes", 1311 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1312 .base.cra_blocksize = AES_BLOCK_SIZE, 1313 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1314 .base.cra_alignmask = 7, 1315 .base.cra_priority = 4001, 1316 .base.cra_module = THIS_MODULE, 1317 1318 .init = otx2_cpt_enc_dec_init, 1319 .exit = otx2_cpt_skcipher_exit, 1320 .ivsize = AES_BLOCK_SIZE, 1321 .min_keysize = AES_MIN_KEY_SIZE, 1322 .max_keysize = AES_MAX_KEY_SIZE, 1323 .setkey = otx2_cpt_skcipher_cbc_aes_setkey, 1324 .encrypt = otx2_cpt_skcipher_encrypt, 1325 .decrypt = otx2_cpt_skcipher_decrypt, 1326 }, { 1327 .base.cra_name = "ecb(aes)", 1328 .base.cra_driver_name = "cpt_ecb_aes", 1329 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1330 .base.cra_blocksize = AES_BLOCK_SIZE, 1331 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1332 .base.cra_alignmask = 7, 1333 .base.cra_priority = 4001, 1334 .base.cra_module = THIS_MODULE, 1335 1336 .init = otx2_cpt_enc_dec_init, 1337 .exit = otx2_cpt_skcipher_exit, 1338 .ivsize = 0, 1339 .min_keysize = AES_MIN_KEY_SIZE, 1340 .max_keysize = AES_MAX_KEY_SIZE, 1341 .setkey = otx2_cpt_skcipher_ecb_aes_setkey, 1342 .encrypt = otx2_cpt_skcipher_encrypt, 1343 .decrypt = otx2_cpt_skcipher_decrypt, 1344 }, { 1345 .base.cra_name = "cbc(des3_ede)", 1346 .base.cra_driver_name = "cpt_cbc_des3_ede", 1347 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1348 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1349 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1350 .base.cra_alignmask = 7, 1351 .base.cra_priority = 4001, 1352 .base.cra_module = THIS_MODULE, 1353 1354 .init = otx2_cpt_enc_dec_init, 1355 .exit = otx2_cpt_skcipher_exit, 1356 .min_keysize = DES3_EDE_KEY_SIZE, 1357 .max_keysize = DES3_EDE_KEY_SIZE, 1358 .ivsize = DES_BLOCK_SIZE, 1359 .setkey = otx2_cpt_skcipher_cbc_des3_setkey, 1360 .encrypt = otx2_cpt_skcipher_encrypt, 1361 .decrypt = otx2_cpt_skcipher_decrypt, 1362 }, { 1363 .base.cra_name = "ecb(des3_ede)", 1364 .base.cra_driver_name = "cpt_ecb_des3_ede", 1365 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1366 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1367 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1368 .base.cra_alignmask = 7, 1369 .base.cra_priority = 4001, 1370 .base.cra_module = THIS_MODULE, 1371 1372 .init = otx2_cpt_enc_dec_init, 1373 .exit = otx2_cpt_skcipher_exit, 1374 .min_keysize = DES3_EDE_KEY_SIZE, 1375 .max_keysize = DES3_EDE_KEY_SIZE, 1376 .ivsize = 0, 1377 .setkey = otx2_cpt_skcipher_ecb_des3_setkey, 1378 .encrypt = otx2_cpt_skcipher_encrypt, 1379 .decrypt = otx2_cpt_skcipher_decrypt, 1380 } }; 1381 1382 static struct aead_alg otx2_cpt_aeads[] = { { 1383 .base = { 1384 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1385 .cra_driver_name = "cpt_hmac_sha1_cbc_aes", 1386 .cra_blocksize = AES_BLOCK_SIZE, 1387 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1388 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1389 .cra_priority = 4001, 1390 .cra_alignmask = 0, 1391 .cra_module = THIS_MODULE, 1392 }, 1393 .init = otx2_cpt_aead_cbc_aes_sha1_init, 1394 .exit = otx2_cpt_aead_exit, 1395 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1396 .setauthsize = otx2_cpt_aead_set_authsize, 1397 .encrypt = otx2_cpt_aead_encrypt, 1398 .decrypt = otx2_cpt_aead_decrypt, 1399 .ivsize = AES_BLOCK_SIZE, 1400 .maxauthsize = SHA1_DIGEST_SIZE, 1401 }, { 1402 .base = { 1403 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1404 .cra_driver_name = "cpt_hmac_sha256_cbc_aes", 1405 .cra_blocksize = AES_BLOCK_SIZE, 1406 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1407 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1408 .cra_priority = 4001, 1409 .cra_alignmask = 0, 1410 .cra_module = THIS_MODULE, 1411 }, 1412 .init = otx2_cpt_aead_cbc_aes_sha256_init, 1413 .exit = otx2_cpt_aead_exit, 1414 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1415 .setauthsize = otx2_cpt_aead_set_authsize, 1416 .encrypt = otx2_cpt_aead_encrypt, 1417 .decrypt = otx2_cpt_aead_decrypt, 1418 .ivsize = AES_BLOCK_SIZE, 1419 .maxauthsize = SHA256_DIGEST_SIZE, 1420 }, { 1421 .base = { 1422 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1423 .cra_driver_name = "cpt_hmac_sha384_cbc_aes", 1424 .cra_blocksize = AES_BLOCK_SIZE, 1425 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1426 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1427 .cra_priority = 4001, 1428 .cra_alignmask = 0, 1429 .cra_module = THIS_MODULE, 1430 }, 1431 .init = otx2_cpt_aead_cbc_aes_sha384_init, 1432 .exit = otx2_cpt_aead_exit, 1433 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1434 .setauthsize = otx2_cpt_aead_set_authsize, 1435 .encrypt = otx2_cpt_aead_encrypt, 1436 .decrypt = otx2_cpt_aead_decrypt, 1437 .ivsize = AES_BLOCK_SIZE, 1438 .maxauthsize = SHA384_DIGEST_SIZE, 1439 }, { 1440 .base = { 1441 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1442 .cra_driver_name = "cpt_hmac_sha512_cbc_aes", 1443 .cra_blocksize = AES_BLOCK_SIZE, 1444 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1445 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1446 .cra_priority = 4001, 1447 .cra_alignmask = 0, 1448 .cra_module = THIS_MODULE, 1449 }, 1450 .init = otx2_cpt_aead_cbc_aes_sha512_init, 1451 .exit = otx2_cpt_aead_exit, 1452 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1453 .setauthsize = otx2_cpt_aead_set_authsize, 1454 .encrypt = otx2_cpt_aead_encrypt, 1455 .decrypt = otx2_cpt_aead_decrypt, 1456 .ivsize = AES_BLOCK_SIZE, 1457 .maxauthsize = SHA512_DIGEST_SIZE, 1458 }, { 1459 .base = { 1460 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", 1461 .cra_driver_name = "cpt_hmac_sha1_ecb_null", 1462 .cra_blocksize = 1, 1463 .cra_flags = CRYPTO_ALG_ASYNC, 1464 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1465 .cra_priority = 4001, 1466 .cra_alignmask = 0, 1467 .cra_module = THIS_MODULE, 1468 }, 1469 .init = otx2_cpt_aead_ecb_null_sha1_init, 1470 .exit = otx2_cpt_aead_exit, 1471 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1472 .setauthsize = otx2_cpt_aead_null_set_authsize, 1473 .encrypt = otx2_cpt_aead_null_encrypt, 1474 .decrypt = otx2_cpt_aead_null_decrypt, 1475 .ivsize = 0, 1476 .maxauthsize = SHA1_DIGEST_SIZE, 1477 }, { 1478 .base = { 1479 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", 1480 .cra_driver_name = "cpt_hmac_sha256_ecb_null", 1481 .cra_blocksize = 1, 1482 .cra_flags = CRYPTO_ALG_ASYNC, 1483 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1484 .cra_priority = 4001, 1485 .cra_alignmask = 0, 1486 .cra_module = THIS_MODULE, 1487 }, 1488 .init = otx2_cpt_aead_ecb_null_sha256_init, 1489 .exit = otx2_cpt_aead_exit, 1490 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1491 .setauthsize = otx2_cpt_aead_null_set_authsize, 1492 .encrypt = otx2_cpt_aead_null_encrypt, 1493 .decrypt = otx2_cpt_aead_null_decrypt, 1494 .ivsize = 0, 1495 .maxauthsize = SHA256_DIGEST_SIZE, 1496 }, { 1497 .base = { 1498 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", 1499 .cra_driver_name = "cpt_hmac_sha384_ecb_null", 1500 .cra_blocksize = 1, 1501 .cra_flags = CRYPTO_ALG_ASYNC, 1502 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1503 .cra_priority = 4001, 1504 .cra_alignmask = 0, 1505 .cra_module = THIS_MODULE, 1506 }, 1507 .init = otx2_cpt_aead_ecb_null_sha384_init, 1508 .exit = otx2_cpt_aead_exit, 1509 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1510 .setauthsize = otx2_cpt_aead_null_set_authsize, 1511 .encrypt = otx2_cpt_aead_null_encrypt, 1512 .decrypt = otx2_cpt_aead_null_decrypt, 1513 .ivsize = 0, 1514 .maxauthsize = SHA384_DIGEST_SIZE, 1515 }, { 1516 .base = { 1517 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", 1518 .cra_driver_name = "cpt_hmac_sha512_ecb_null", 1519 .cra_blocksize = 1, 1520 .cra_flags = CRYPTO_ALG_ASYNC, 1521 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1522 .cra_priority = 4001, 1523 .cra_alignmask = 0, 1524 .cra_module = THIS_MODULE, 1525 }, 1526 .init = otx2_cpt_aead_ecb_null_sha512_init, 1527 .exit = otx2_cpt_aead_exit, 1528 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1529 .setauthsize = otx2_cpt_aead_null_set_authsize, 1530 .encrypt = otx2_cpt_aead_null_encrypt, 1531 .decrypt = otx2_cpt_aead_null_decrypt, 1532 .ivsize = 0, 1533 .maxauthsize = SHA512_DIGEST_SIZE, 1534 }, { 1535 .base = { 1536 .cra_name = "rfc4106(gcm(aes))", 1537 .cra_driver_name = "cpt_rfc4106_gcm_aes", 1538 .cra_blocksize = 1, 1539 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1540 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1541 .cra_priority = 4001, 1542 .cra_alignmask = 0, 1543 .cra_module = THIS_MODULE, 1544 }, 1545 .init = otx2_cpt_aead_gcm_aes_init, 1546 .exit = otx2_cpt_aead_exit, 1547 .setkey = otx2_cpt_aead_gcm_aes_setkey, 1548 .setauthsize = otx2_cpt_aead_gcm_set_authsize, 1549 .encrypt = otx2_cpt_aead_encrypt, 1550 .decrypt = otx2_cpt_aead_decrypt, 1551 .ivsize = AES_GCM_IV_SIZE, 1552 .maxauthsize = AES_GCM_ICV_SIZE, 1553 } }; 1554 1555 static inline int cpt_register_algs(void) 1556 { 1557 int i, err = 0; 1558 1559 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++) 1560 otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1561 1562 err = crypto_register_skciphers(otx2_cpt_skciphers, 1563 ARRAY_SIZE(otx2_cpt_skciphers)); 1564 if (err) 1565 return err; 1566 1567 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++) 1568 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1569 1570 err = crypto_register_aeads(otx2_cpt_aeads, 1571 ARRAY_SIZE(otx2_cpt_aeads)); 1572 if (err) { 1573 crypto_unregister_skciphers(otx2_cpt_skciphers, 1574 ARRAY_SIZE(otx2_cpt_skciphers)); 1575 return err; 1576 } 1577 1578 return 0; 1579 } 1580 1581 static inline void cpt_unregister_algs(void) 1582 { 1583 crypto_unregister_skciphers(otx2_cpt_skciphers, 1584 ARRAY_SIZE(otx2_cpt_skciphers)); 1585 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads)); 1586 } 1587 1588 static int compare_func(const void *lptr, const void *rptr) 1589 { 1590 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; 1591 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; 1592 1593 if (ldesc->dev->devfn < rdesc->dev->devfn) 1594 return -1; 1595 if (ldesc->dev->devfn > rdesc->dev->devfn) 1596 return 1; 1597 return 0; 1598 } 1599 1600 static void swap_func(void *lptr, void *rptr, int size) 1601 { 1602 struct cpt_device_desc *ldesc = lptr; 1603 struct cpt_device_desc *rdesc = rptr; 1604 1605 swap(*ldesc, *rdesc); 1606 } 1607 1608 int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, 1609 int num_queues, int num_devices) 1610 { 1611 int ret = 0; 1612 int count; 1613 1614 mutex_lock(&mutex); 1615 count = atomic_read(&se_devices.count); 1616 if (count >= OTX2_CPT_MAX_LFS_NUM) { 1617 dev_err(&pdev->dev, "No space to add a new device\n"); 1618 ret = -ENOSPC; 1619 goto unlock; 1620 } 1621 se_devices.desc[count].num_queues = num_queues; 1622 se_devices.desc[count++].dev = pdev; 1623 atomic_inc(&se_devices.count); 1624 1625 if (atomic_read(&se_devices.count) == num_devices && 1626 is_crypto_registered == false) { 1627 if (cpt_register_algs()) { 1628 dev_err(&pdev->dev, 1629 "Error in registering crypto algorithms\n"); 1630 ret = -EINVAL; 1631 goto unlock; 1632 } 1633 try_module_get(mod); 1634 is_crypto_registered = true; 1635 } 1636 sort(se_devices.desc, count, sizeof(struct cpt_device_desc), 1637 compare_func, swap_func); 1638 1639 unlock: 1640 mutex_unlock(&mutex); 1641 return ret; 1642 } 1643 1644 void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod) 1645 { 1646 struct cpt_device_table *dev_tbl; 1647 bool dev_found = false; 1648 int i, j, count; 1649 1650 mutex_lock(&mutex); 1651 1652 dev_tbl = &se_devices; 1653 count = atomic_read(&dev_tbl->count); 1654 for (i = 0; i < count; i++) { 1655 if (pdev == dev_tbl->desc[i].dev) { 1656 for (j = i; j < count-1; j++) 1657 dev_tbl->desc[j] = dev_tbl->desc[j+1]; 1658 dev_found = true; 1659 break; 1660 } 1661 } 1662 1663 if (!dev_found) { 1664 dev_err(&pdev->dev, "%s device not found\n", __func__); 1665 goto unlock; 1666 } 1667 if (atomic_dec_and_test(&se_devices.count)) { 1668 cpt_unregister_algs(); 1669 module_put(mod); 1670 is_crypto_registered = false; 1671 } 1672 1673 unlock: 1674 mutex_unlock(&mutex); 1675 } 1676