1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Marvell. */ 3 4 #include <crypto/aes.h> 5 #include <crypto/authenc.h> 6 #include <crypto/cryptd.h> 7 #include <crypto/des.h> 8 #include <crypto/internal/aead.h> 9 #include <crypto/sha1.h> 10 #include <crypto/sha2.h> 11 #include <crypto/xts.h> 12 #include <crypto/gcm.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/rtnetlink.h> 15 #include <linux/sort.h> 16 #include <linux/module.h> 17 #include "otx2_cptvf.h" 18 #include "otx2_cptvf_algs.h" 19 #include "otx2_cpt_reqmgr.h" 20 21 /* Size of salt in AES GCM mode */ 22 #define AES_GCM_SALT_SIZE 4 23 /* Size of IV in AES GCM mode */ 24 #define AES_GCM_IV_SIZE 8 25 /* Size of ICV (Integrity Check Value) in AES GCM mode */ 26 #define AES_GCM_ICV_SIZE 16 27 /* Offset of IV in AES GCM mode */ 28 #define AES_GCM_IV_OFFSET 8 29 #define CONTROL_WORD_LEN 8 30 #define KEY2_OFFSET 48 31 #define DMA_MODE_FLAG(dma_mode) \ 32 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0) 33 34 /* Truncated SHA digest size */ 35 #define SHA1_TRUNC_DIGEST_SIZE 12 36 #define SHA256_TRUNC_DIGEST_SIZE 16 37 #define SHA384_TRUNC_DIGEST_SIZE 24 38 #define SHA512_TRUNC_DIGEST_SIZE 32 39 40 static DEFINE_MUTEX(mutex); 41 static int is_crypto_registered; 42 43 struct cpt_device_desc { 44 struct pci_dev *dev; 45 int num_queues; 46 }; 47 48 struct cpt_device_table { 49 atomic_t count; 50 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM]; 51 }; 52 53 static struct cpt_device_table se_devices = { 54 .count = ATOMIC_INIT(0) 55 }; 56 57 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) 58 { 59 int count; 60 61 count = atomic_read(&se_devices.count); 62 if (count < 1) 63 return -ENODEV; 64 65 *cpu_num = get_cpu(); 66 /* 67 * On OcteonTX2 platform CPT instruction queue is bound to each 68 * local function LF, in turn LFs can be attached to PF 69 * or VF therefore we always use first device. We get maximum 70 * performance if one CPT queue is available for each cpu 71 * otherwise CPT queues need to be shared between cpus. 72 */ 73 if (*cpu_num >= se_devices.desc[0].num_queues) 74 *cpu_num %= se_devices.desc[0].num_queues; 75 *pdev = se_devices.desc[0].dev; 76 77 put_cpu(); 78 79 return 0; 80 } 81 82 static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) 83 { 84 struct otx2_cpt_req_ctx *rctx; 85 struct aead_request *req; 86 struct crypto_aead *tfm; 87 88 req = container_of(cpt_req->areq, struct aead_request, base); 89 tfm = crypto_aead_reqtfm(req); 90 rctx = aead_request_ctx(req); 91 if (memcmp(rctx->fctx.hmac.s.hmac_calc, 92 rctx->fctx.hmac.s.hmac_recv, 93 crypto_aead_authsize(tfm)) != 0) 94 return -EBADMSG; 95 96 return 0; 97 } 98 99 static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2) 100 { 101 struct otx2_cpt_inst_info *inst_info = arg2; 102 struct crypto_async_request *areq = arg1; 103 struct otx2_cpt_req_info *cpt_req; 104 struct pci_dev *pdev; 105 106 if (inst_info) { 107 cpt_req = inst_info->req; 108 if (!status) { 109 /* 110 * When selected cipher is NULL we need to manually 111 * verify whether calculated hmac value matches 112 * received hmac value 113 */ 114 if (cpt_req->req_type == 115 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ && 116 !cpt_req->is_enc) 117 status = validate_hmac_cipher_null(cpt_req); 118 } 119 pdev = inst_info->pdev; 120 otx2_cpt_info_destroy(pdev, inst_info); 121 } 122 if (areq) 123 areq->complete(areq, status); 124 } 125 126 static void output_iv_copyback(struct crypto_async_request *areq) 127 { 128 struct otx2_cpt_req_info *req_info; 129 struct otx2_cpt_req_ctx *rctx; 130 struct skcipher_request *sreq; 131 struct crypto_skcipher *stfm; 132 struct otx2_cpt_enc_ctx *ctx; 133 u32 start, ivsize; 134 135 sreq = container_of(areq, struct skcipher_request, base); 136 stfm = crypto_skcipher_reqtfm(sreq); 137 ctx = crypto_skcipher_ctx(stfm); 138 if (ctx->cipher_type == OTX2_CPT_AES_CBC || 139 ctx->cipher_type == OTX2_CPT_DES3_CBC) { 140 rctx = skcipher_request_ctx(sreq); 141 req_info = &rctx->cpt_req; 142 ivsize = crypto_skcipher_ivsize(stfm); 143 start = sreq->cryptlen - ivsize; 144 145 if (req_info->is_enc) { 146 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, 147 ivsize, 0); 148 } else { 149 if (sreq->src != sreq->dst) { 150 scatterwalk_map_and_copy(sreq->iv, sreq->src, 151 start, ivsize, 0); 152 } else { 153 memcpy(sreq->iv, req_info->iv_out, ivsize); 154 kfree(req_info->iv_out); 155 } 156 } 157 } 158 } 159 160 static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2) 161 { 162 struct otx2_cpt_inst_info *inst_info = arg2; 163 struct crypto_async_request *areq = arg1; 164 struct pci_dev *pdev; 165 166 if (areq) { 167 if (!status) 168 output_iv_copyback(areq); 169 if (inst_info) { 170 pdev = inst_info->pdev; 171 otx2_cpt_info_destroy(pdev, inst_info); 172 } 173 areq->complete(areq, status); 174 } 175 } 176 177 static inline void update_input_data(struct otx2_cpt_req_info *req_info, 178 struct scatterlist *inp_sg, 179 u32 nbytes, u32 *argcnt) 180 { 181 req_info->req.dlen += nbytes; 182 183 while (nbytes) { 184 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length; 185 u8 *ptr = sg_virt(inp_sg); 186 187 req_info->in[*argcnt].vptr = (void *)ptr; 188 req_info->in[*argcnt].size = len; 189 nbytes -= len; 190 ++(*argcnt); 191 inp_sg = sg_next(inp_sg); 192 } 193 } 194 195 static inline void update_output_data(struct otx2_cpt_req_info *req_info, 196 struct scatterlist *outp_sg, 197 u32 offset, u32 nbytes, u32 *argcnt) 198 { 199 u32 len, sg_len; 200 u8 *ptr; 201 202 req_info->rlen += nbytes; 203 204 while (nbytes) { 205 sg_len = outp_sg->length - offset; 206 len = (nbytes < sg_len) ? nbytes : sg_len; 207 ptr = sg_virt(outp_sg); 208 209 req_info->out[*argcnt].vptr = (void *) (ptr + offset); 210 req_info->out[*argcnt].size = len; 211 nbytes -= len; 212 ++(*argcnt); 213 offset = 0; 214 outp_sg = sg_next(outp_sg); 215 } 216 } 217 218 static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, 219 u32 *argcnt) 220 { 221 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 222 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); 223 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 224 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 225 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; 226 int ivsize = crypto_skcipher_ivsize(stfm); 227 u32 start = req->cryptlen - ivsize; 228 gfp_t flags; 229 230 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 231 GFP_KERNEL : GFP_ATOMIC; 232 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 233 req_info->ctrl.s.se_req = 1; 234 235 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | 236 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 237 if (enc) { 238 req_info->req.opcode.s.minor = 2; 239 } else { 240 req_info->req.opcode.s.minor = 3; 241 if ((ctx->cipher_type == OTX2_CPT_AES_CBC || 242 ctx->cipher_type == OTX2_CPT_DES3_CBC) && 243 req->src == req->dst) { 244 req_info->iv_out = kmalloc(ivsize, flags); 245 if (!req_info->iv_out) 246 return -ENOMEM; 247 248 scatterwalk_map_and_copy(req_info->iv_out, req->src, 249 start, ivsize, 0); 250 } 251 } 252 /* Encryption data length */ 253 req_info->req.param1 = req->cryptlen; 254 /* Authentication data length */ 255 req_info->req.param2 = 0; 256 257 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 258 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 259 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; 260 261 if (ctx->cipher_type == OTX2_CPT_AES_XTS) 262 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 263 else 264 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 265 266 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); 267 268 cpu_to_be64s(&fctx->enc.enc_ctrl.u); 269 270 /* 271 * Storing Packet Data Information in offset 272 * Control Word First 8 bytes 273 */ 274 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 275 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 276 req_info->req.dlen += CONTROL_WORD_LEN; 277 ++(*argcnt); 278 279 req_info->in[*argcnt].vptr = (u8 *)fctx; 280 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); 281 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); 282 283 ++(*argcnt); 284 285 return 0; 286 } 287 288 static inline int create_input_list(struct skcipher_request *req, u32 enc, 289 u32 enc_iv_len) 290 { 291 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); 292 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 293 u32 argcnt = 0; 294 int ret; 295 296 ret = create_ctx_hdr(req, enc, &argcnt); 297 if (ret) 298 return ret; 299 300 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 301 req_info->in_cnt = argcnt; 302 303 return 0; 304 } 305 306 static inline void create_output_list(struct skcipher_request *req, 307 u32 enc_iv_len) 308 { 309 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); 310 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 311 u32 argcnt = 0; 312 313 /* 314 * OUTPUT Buffer Processing 315 * AES encryption/decryption output would be 316 * received in the following format 317 * 318 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 319 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 320 */ 321 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); 322 req_info->out_cnt = argcnt; 323 } 324 325 static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc) 326 { 327 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 328 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); 329 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 330 int ret; 331 332 if (ctx->fbk_cipher) { 333 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher); 334 skcipher_request_set_callback(&rctx->sk_fbk_req, 335 req->base.flags, 336 req->base.complete, 337 req->base.data); 338 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src, 339 req->dst, req->cryptlen, req->iv); 340 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) : 341 crypto_skcipher_decrypt(&rctx->sk_fbk_req); 342 } else { 343 ret = -EINVAL; 344 } 345 return ret; 346 } 347 348 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) 349 { 350 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 351 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); 352 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 353 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 354 u32 enc_iv_len = crypto_skcipher_ivsize(stfm); 355 struct pci_dev *pdev; 356 int status, cpu_num; 357 358 if (req->cryptlen == 0) 359 return 0; 360 361 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len)) 362 return -EINVAL; 363 364 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE) 365 return skcipher_do_fallback(req, enc); 366 367 /* Clear control words */ 368 rctx->ctrl_word.flags = 0; 369 rctx->fctx.enc.enc_ctrl.u = 0; 370 371 status = create_input_list(req, enc, enc_iv_len); 372 if (status) 373 return status; 374 create_output_list(req, enc_iv_len); 375 376 status = get_se_device(&pdev, &cpu_num); 377 if (status) 378 return status; 379 380 req_info->callback = otx2_cpt_skcipher_callback; 381 req_info->areq = &req->base; 382 req_info->req_type = OTX2_CPT_ENC_DEC_REQ; 383 req_info->is_enc = enc; 384 req_info->is_trunc_hmac = false; 385 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); 386 387 /* 388 * We perform an asynchronous send and once 389 * the request is completed the driver would 390 * intimate through registered call back functions 391 */ 392 status = otx2_cpt_do_request(pdev, req_info, cpu_num); 393 394 return status; 395 } 396 397 static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req) 398 { 399 return cpt_enc_dec(req, true); 400 } 401 402 static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req) 403 { 404 return cpt_enc_dec(req, false); 405 } 406 407 static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, 408 const u8 *key, u32 keylen) 409 { 410 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 411 const u8 *key2 = key + (keylen / 2); 412 const u8 *key1 = key; 413 int ret; 414 415 ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen); 416 if (ret) 417 return ret; 418 ctx->key_len = keylen; 419 ctx->enc_align_len = 1; 420 memcpy(ctx->enc_key, key1, keylen / 2); 421 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 422 ctx->cipher_type = OTX2_CPT_AES_XTS; 423 switch (ctx->key_len) { 424 case 2 * AES_KEYSIZE_128: 425 ctx->key_type = OTX2_CPT_AES_128_BIT; 426 break; 427 case 2 * AES_KEYSIZE_192: 428 ctx->key_type = OTX2_CPT_AES_192_BIT; 429 break; 430 case 2 * AES_KEYSIZE_256: 431 ctx->key_type = OTX2_CPT_AES_256_BIT; 432 break; 433 default: 434 return -EINVAL; 435 } 436 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 437 } 438 439 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 440 u32 keylen, u8 cipher_type) 441 { 442 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 443 444 if (keylen != DES3_EDE_KEY_SIZE) 445 return -EINVAL; 446 447 ctx->key_len = keylen; 448 ctx->cipher_type = cipher_type; 449 ctx->enc_align_len = 8; 450 451 memcpy(ctx->enc_key, key, keylen); 452 453 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 454 } 455 456 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 457 u32 keylen, u8 cipher_type) 458 { 459 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 460 461 switch (keylen) { 462 case AES_KEYSIZE_128: 463 ctx->key_type = OTX2_CPT_AES_128_BIT; 464 break; 465 case AES_KEYSIZE_192: 466 ctx->key_type = OTX2_CPT_AES_192_BIT; 467 break; 468 case AES_KEYSIZE_256: 469 ctx->key_type = OTX2_CPT_AES_256_BIT; 470 break; 471 default: 472 return -EINVAL; 473 } 474 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB) 475 ctx->enc_align_len = 16; 476 else 477 ctx->enc_align_len = 1; 478 479 ctx->key_len = keylen; 480 ctx->cipher_type = cipher_type; 481 482 memcpy(ctx->enc_key, key, keylen); 483 484 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 485 } 486 487 static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, 488 const u8 *key, u32 keylen) 489 { 490 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC); 491 } 492 493 static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, 494 const u8 *key, u32 keylen) 495 { 496 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB); 497 } 498 499 static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, 500 const u8 *key, u32 keylen) 501 { 502 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC); 503 } 504 505 static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, 506 const u8 *key, u32 keylen) 507 { 508 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB); 509 } 510 511 static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx, 512 struct crypto_alg *alg) 513 { 514 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 515 ctx->fbk_cipher = 516 crypto_alloc_skcipher(alg->cra_name, 0, 517 CRYPTO_ALG_ASYNC | 518 CRYPTO_ALG_NEED_FALLBACK); 519 if (IS_ERR(ctx->fbk_cipher)) { 520 pr_err("%s() failed to allocate fallback for %s\n", 521 __func__, alg->cra_name); 522 return PTR_ERR(ctx->fbk_cipher); 523 } 524 } 525 return 0; 526 } 527 528 static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) 529 { 530 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 531 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); 532 struct crypto_alg *alg = tfm->__crt_alg; 533 534 memset(ctx, 0, sizeof(*ctx)); 535 /* 536 * Additional memory for skcipher_request is 537 * allocated since the cryptd daemon uses 538 * this memory for request_ctx information 539 */ 540 crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) + 541 sizeof(struct skcipher_request)); 542 543 return cpt_skcipher_fallback_init(ctx, alg); 544 } 545 546 static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) 547 { 548 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 549 550 if (ctx->fbk_cipher) { 551 crypto_free_skcipher(ctx->fbk_cipher); 552 ctx->fbk_cipher = NULL; 553 } 554 } 555 556 static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, 557 struct crypto_alg *alg) 558 { 559 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 560 ctx->fbk_cipher = 561 crypto_alloc_aead(alg->cra_name, 0, 562 CRYPTO_ALG_ASYNC | 563 CRYPTO_ALG_NEED_FALLBACK); 564 if (IS_ERR(ctx->fbk_cipher)) { 565 pr_err("%s() failed to allocate fallback for %s\n", 566 __func__, alg->cra_name); 567 return PTR_ERR(ctx->fbk_cipher); 568 } 569 } 570 return 0; 571 } 572 573 static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) 574 { 575 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm); 576 struct crypto_tfm *tfm = crypto_aead_tfm(atfm); 577 struct crypto_alg *alg = tfm->__crt_alg; 578 579 ctx->cipher_type = cipher_type; 580 ctx->mac_type = mac_type; 581 582 /* 583 * When selected cipher is NULL we use HMAC opcode instead of 584 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms 585 * for calculating ipad and opad 586 */ 587 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) { 588 switch (ctx->mac_type) { 589 case OTX2_CPT_SHA1: 590 ctx->hashalg = crypto_alloc_shash("sha1", 0, 591 CRYPTO_ALG_ASYNC); 592 if (IS_ERR(ctx->hashalg)) 593 return PTR_ERR(ctx->hashalg); 594 break; 595 596 case OTX2_CPT_SHA256: 597 ctx->hashalg = crypto_alloc_shash("sha256", 0, 598 CRYPTO_ALG_ASYNC); 599 if (IS_ERR(ctx->hashalg)) 600 return PTR_ERR(ctx->hashalg); 601 break; 602 603 case OTX2_CPT_SHA384: 604 ctx->hashalg = crypto_alloc_shash("sha384", 0, 605 CRYPTO_ALG_ASYNC); 606 if (IS_ERR(ctx->hashalg)) 607 return PTR_ERR(ctx->hashalg); 608 break; 609 610 case OTX2_CPT_SHA512: 611 ctx->hashalg = crypto_alloc_shash("sha512", 0, 612 CRYPTO_ALG_ASYNC); 613 if (IS_ERR(ctx->hashalg)) 614 return PTR_ERR(ctx->hashalg); 615 break; 616 } 617 } 618 switch (ctx->cipher_type) { 619 case OTX2_CPT_AES_CBC: 620 case OTX2_CPT_AES_ECB: 621 ctx->enc_align_len = 16; 622 break; 623 case OTX2_CPT_DES3_CBC: 624 case OTX2_CPT_DES3_ECB: 625 ctx->enc_align_len = 8; 626 break; 627 case OTX2_CPT_AES_GCM: 628 case OTX2_CPT_CIPHER_NULL: 629 ctx->enc_align_len = 1; 630 break; 631 } 632 crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx)); 633 634 return cpt_aead_fallback_init(ctx, alg); 635 } 636 637 static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) 638 { 639 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1); 640 } 641 642 static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) 643 { 644 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256); 645 } 646 647 static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) 648 { 649 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384); 650 } 651 652 static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) 653 { 654 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512); 655 } 656 657 static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) 658 { 659 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1); 660 } 661 662 static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) 663 { 664 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256); 665 } 666 667 static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) 668 { 669 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384); 670 } 671 672 static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) 673 { 674 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512); 675 } 676 677 static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) 678 { 679 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL); 680 } 681 682 static void otx2_cpt_aead_exit(struct crypto_aead *tfm) 683 { 684 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); 685 686 kfree(ctx->ipad); 687 kfree(ctx->opad); 688 if (ctx->hashalg) 689 crypto_free_shash(ctx->hashalg); 690 kfree(ctx->sdesc); 691 692 if (ctx->fbk_cipher) { 693 crypto_free_aead(ctx->fbk_cipher); 694 ctx->fbk_cipher = NULL; 695 } 696 } 697 698 static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, 699 unsigned int authsize) 700 { 701 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); 702 703 if (crypto_rfc4106_check_authsize(authsize)) 704 return -EINVAL; 705 706 tfm->authsize = authsize; 707 /* Set authsize for fallback case */ 708 if (ctx->fbk_cipher) 709 ctx->fbk_cipher->authsize = authsize; 710 711 return 0; 712 } 713 714 static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm, 715 unsigned int authsize) 716 { 717 tfm->authsize = authsize; 718 719 return 0; 720 } 721 722 static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm, 723 unsigned int authsize) 724 { 725 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); 726 727 ctx->is_trunc_hmac = true; 728 tfm->authsize = authsize; 729 730 return 0; 731 } 732 733 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) 734 { 735 struct otx2_cpt_sdesc *sdesc; 736 int size; 737 738 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); 739 sdesc = kmalloc(size, GFP_KERNEL); 740 if (!sdesc) 741 return NULL; 742 743 sdesc->shash.tfm = alg; 744 745 return sdesc; 746 } 747 748 static inline void swap_data32(void *buf, u32 len) 749 { 750 cpu_to_be32_array(buf, buf, len / 4); 751 } 752 753 static inline void swap_data64(void *buf, u32 len) 754 { 755 u64 *src = buf; 756 int i = 0; 757 758 for (i = 0 ; i < len / 8; i++, src++) 759 cpu_to_be64s(src); 760 } 761 762 static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad) 763 { 764 struct sha512_state *sha512; 765 struct sha256_state *sha256; 766 struct sha1_state *sha1; 767 768 switch (mac_type) { 769 case OTX2_CPT_SHA1: 770 sha1 = (struct sha1_state *) in_pad; 771 swap_data32(sha1->state, SHA1_DIGEST_SIZE); 772 memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE); 773 break; 774 775 case OTX2_CPT_SHA256: 776 sha256 = (struct sha256_state *) in_pad; 777 swap_data32(sha256->state, SHA256_DIGEST_SIZE); 778 memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE); 779 break; 780 781 case OTX2_CPT_SHA384: 782 case OTX2_CPT_SHA512: 783 sha512 = (struct sha512_state *) in_pad; 784 swap_data64(sha512->state, SHA512_DIGEST_SIZE); 785 memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE); 786 break; 787 788 default: 789 return -EINVAL; 790 } 791 792 return 0; 793 } 794 795 static int aead_hmac_init(struct crypto_aead *cipher) 796 { 797 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); 798 int state_size = crypto_shash_statesize(ctx->hashalg); 799 int ds = crypto_shash_digestsize(ctx->hashalg); 800 int bs = crypto_shash_blocksize(ctx->hashalg); 801 int authkeylen = ctx->auth_key_len; 802 u8 *ipad = NULL, *opad = NULL; 803 int ret = 0, icount = 0; 804 805 ctx->sdesc = alloc_sdesc(ctx->hashalg); 806 if (!ctx->sdesc) 807 return -ENOMEM; 808 809 ctx->ipad = kzalloc(bs, GFP_KERNEL); 810 if (!ctx->ipad) { 811 ret = -ENOMEM; 812 goto calc_fail; 813 } 814 815 ctx->opad = kzalloc(bs, GFP_KERNEL); 816 if (!ctx->opad) { 817 ret = -ENOMEM; 818 goto calc_fail; 819 } 820 821 ipad = kzalloc(state_size, GFP_KERNEL); 822 if (!ipad) { 823 ret = -ENOMEM; 824 goto calc_fail; 825 } 826 827 opad = kzalloc(state_size, GFP_KERNEL); 828 if (!opad) { 829 ret = -ENOMEM; 830 goto calc_fail; 831 } 832 833 if (authkeylen > bs) { 834 ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key, 835 authkeylen, ipad); 836 if (ret) 837 goto calc_fail; 838 839 authkeylen = ds; 840 } else { 841 memcpy(ipad, ctx->key, authkeylen); 842 } 843 844 memset(ipad + authkeylen, 0, bs - authkeylen); 845 memcpy(opad, ipad, bs); 846 847 for (icount = 0; icount < bs; icount++) { 848 ipad[icount] ^= 0x36; 849 opad[icount] ^= 0x5c; 850 } 851 852 /* 853 * Partial Hash calculated from the software 854 * algorithm is retrieved for IPAD & OPAD 855 */ 856 857 /* IPAD Calculation */ 858 crypto_shash_init(&ctx->sdesc->shash); 859 crypto_shash_update(&ctx->sdesc->shash, ipad, bs); 860 crypto_shash_export(&ctx->sdesc->shash, ipad); 861 ret = copy_pad(ctx->mac_type, ctx->ipad, ipad); 862 if (ret) 863 goto calc_fail; 864 865 /* OPAD Calculation */ 866 crypto_shash_init(&ctx->sdesc->shash); 867 crypto_shash_update(&ctx->sdesc->shash, opad, bs); 868 crypto_shash_export(&ctx->sdesc->shash, opad); 869 ret = copy_pad(ctx->mac_type, ctx->opad, opad); 870 if (ret) 871 goto calc_fail; 872 873 kfree(ipad); 874 kfree(opad); 875 876 return 0; 877 878 calc_fail: 879 kfree(ctx->ipad); 880 ctx->ipad = NULL; 881 kfree(ctx->opad); 882 ctx->opad = NULL; 883 kfree(ipad); 884 kfree(opad); 885 kfree(ctx->sdesc); 886 ctx->sdesc = NULL; 887 888 return ret; 889 } 890 891 static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, 892 const unsigned char *key, 893 unsigned int keylen) 894 { 895 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); 896 struct crypto_authenc_key_param *param; 897 int enckeylen = 0, authkeylen = 0; 898 struct rtattr *rta = (void *)key; 899 int status; 900 901 if (!RTA_OK(rta, keylen)) 902 return -EINVAL; 903 904 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 905 return -EINVAL; 906 907 if (RTA_PAYLOAD(rta) < sizeof(*param)) 908 return -EINVAL; 909 910 param = RTA_DATA(rta); 911 enckeylen = be32_to_cpu(param->enckeylen); 912 key += RTA_ALIGN(rta->rta_len); 913 keylen -= RTA_ALIGN(rta->rta_len); 914 if (keylen < enckeylen) 915 return -EINVAL; 916 917 if (keylen > OTX2_CPT_MAX_KEY_SIZE) 918 return -EINVAL; 919 920 authkeylen = keylen - enckeylen; 921 memcpy(ctx->key, key, keylen); 922 923 switch (enckeylen) { 924 case AES_KEYSIZE_128: 925 ctx->key_type = OTX2_CPT_AES_128_BIT; 926 break; 927 case AES_KEYSIZE_192: 928 ctx->key_type = OTX2_CPT_AES_192_BIT; 929 break; 930 case AES_KEYSIZE_256: 931 ctx->key_type = OTX2_CPT_AES_256_BIT; 932 break; 933 default: 934 /* Invalid key length */ 935 return -EINVAL; 936 } 937 938 ctx->enc_key_len = enckeylen; 939 ctx->auth_key_len = authkeylen; 940 941 status = aead_hmac_init(cipher); 942 if (status) 943 return status; 944 945 return 0; 946 } 947 948 static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, 949 const unsigned char *key, 950 unsigned int keylen) 951 { 952 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); 953 struct crypto_authenc_key_param *param; 954 struct rtattr *rta = (void *)key; 955 int enckeylen = 0; 956 957 if (!RTA_OK(rta, keylen)) 958 return -EINVAL; 959 960 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 961 return -EINVAL; 962 963 if (RTA_PAYLOAD(rta) < sizeof(*param)) 964 return -EINVAL; 965 966 param = RTA_DATA(rta); 967 enckeylen = be32_to_cpu(param->enckeylen); 968 key += RTA_ALIGN(rta->rta_len); 969 keylen -= RTA_ALIGN(rta->rta_len); 970 if (enckeylen != 0) 971 return -EINVAL; 972 973 if (keylen > OTX2_CPT_MAX_KEY_SIZE) 974 return -EINVAL; 975 976 memcpy(ctx->key, key, keylen); 977 ctx->enc_key_len = enckeylen; 978 ctx->auth_key_len = keylen; 979 980 return 0; 981 } 982 983 static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, 984 const unsigned char *key, 985 unsigned int keylen) 986 { 987 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); 988 989 /* 990 * For aes gcm we expect to get encryption key (16, 24, 32 bytes) 991 * and salt (4 bytes) 992 */ 993 switch (keylen) { 994 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: 995 ctx->key_type = OTX2_CPT_AES_128_BIT; 996 ctx->enc_key_len = AES_KEYSIZE_128; 997 break; 998 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: 999 ctx->key_type = OTX2_CPT_AES_192_BIT; 1000 ctx->enc_key_len = AES_KEYSIZE_192; 1001 break; 1002 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: 1003 ctx->key_type = OTX2_CPT_AES_256_BIT; 1004 ctx->enc_key_len = AES_KEYSIZE_256; 1005 break; 1006 default: 1007 /* Invalid key and salt length */ 1008 return -EINVAL; 1009 } 1010 1011 /* Store encryption key and salt */ 1012 memcpy(ctx->key, key, keylen); 1013 1014 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen); 1015 } 1016 1017 static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc, 1018 u32 *argcnt) 1019 { 1020 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1021 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1022 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); 1023 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1024 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; 1025 int mac_len = crypto_aead_authsize(tfm); 1026 int ds; 1027 1028 rctx->ctrl_word.e.enc_data_offset = req->assoclen; 1029 1030 switch (ctx->cipher_type) { 1031 case OTX2_CPT_AES_CBC: 1032 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8)) 1033 return -EINVAL; 1034 1035 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; 1036 /* Copy encryption key to context */ 1037 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, 1038 ctx->enc_key_len); 1039 /* Copy IV to context */ 1040 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); 1041 1042 ds = crypto_shash_digestsize(ctx->hashalg); 1043 if (ctx->mac_type == OTX2_CPT_SHA384) 1044 ds = SHA512_DIGEST_SIZE; 1045 if (ctx->ipad) 1046 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); 1047 if (ctx->opad) 1048 memcpy(fctx->hmac.e.opad, ctx->opad, ds); 1049 break; 1050 1051 case OTX2_CPT_AES_GCM: 1052 if (crypto_ipsec_check_assoclen(req->assoclen)) 1053 return -EINVAL; 1054 1055 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR; 1056 /* Copy encryption key to context */ 1057 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); 1058 /* Copy salt to context */ 1059 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, 1060 AES_GCM_SALT_SIZE); 1061 1062 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; 1063 break; 1064 1065 default: 1066 /* Unknown cipher type */ 1067 return -EINVAL; 1068 } 1069 cpu_to_be64s(&rctx->ctrl_word.flags); 1070 1071 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 1072 req_info->ctrl.s.se_req = 1; 1073 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | 1074 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 1075 if (enc) { 1076 req_info->req.opcode.s.minor = 2; 1077 req_info->req.param1 = req->cryptlen; 1078 req_info->req.param2 = req->cryptlen + req->assoclen; 1079 } else { 1080 req_info->req.opcode.s.minor = 3; 1081 req_info->req.param1 = req->cryptlen - mac_len; 1082 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; 1083 } 1084 1085 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 1086 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 1087 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; 1088 fctx->enc.enc_ctrl.e.mac_len = mac_len; 1089 cpu_to_be64s(&fctx->enc.enc_ctrl.u); 1090 1091 /* 1092 * Storing Packet Data Information in offset 1093 * Control Word First 8 bytes 1094 */ 1095 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 1096 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 1097 req_info->req.dlen += CONTROL_WORD_LEN; 1098 ++(*argcnt); 1099 1100 req_info->in[*argcnt].vptr = (u8 *)fctx; 1101 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); 1102 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); 1103 ++(*argcnt); 1104 1105 return 0; 1106 } 1107 1108 static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, 1109 u32 enc) 1110 { 1111 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1112 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1113 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); 1114 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1115 1116 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 1117 req_info->ctrl.s.se_req = 1; 1118 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC | 1119 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 1120 req_info->is_trunc_hmac = ctx->is_trunc_hmac; 1121 1122 req_info->req.opcode.s.minor = 0; 1123 req_info->req.param1 = ctx->auth_key_len; 1124 req_info->req.param2 = ctx->mac_type << 8; 1125 1126 /* Add authentication key */ 1127 req_info->in[*argcnt].vptr = ctx->key; 1128 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); 1129 req_info->req.dlen += round_up(ctx->auth_key_len, 8); 1130 ++(*argcnt); 1131 } 1132 1133 static inline int create_aead_input_list(struct aead_request *req, u32 enc) 1134 { 1135 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1136 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1137 u32 inputlen = req->cryptlen + req->assoclen; 1138 u32 status, argcnt = 0; 1139 1140 status = create_aead_ctx_hdr(req, enc, &argcnt); 1141 if (status) 1142 return status; 1143 update_input_data(req_info, req->src, inputlen, &argcnt); 1144 req_info->in_cnt = argcnt; 1145 1146 return 0; 1147 } 1148 1149 static inline void create_aead_output_list(struct aead_request *req, u32 enc, 1150 u32 mac_len) 1151 { 1152 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1153 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1154 u32 argcnt = 0, outputlen = 0; 1155 1156 if (enc) 1157 outputlen = req->cryptlen + req->assoclen + mac_len; 1158 else 1159 outputlen = req->cryptlen + req->assoclen - mac_len; 1160 1161 update_output_data(req_info, req->dst, 0, outputlen, &argcnt); 1162 req_info->out_cnt = argcnt; 1163 } 1164 1165 static inline void create_aead_null_input_list(struct aead_request *req, 1166 u32 enc, u32 mac_len) 1167 { 1168 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1169 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1170 u32 inputlen, argcnt = 0; 1171 1172 if (enc) 1173 inputlen = req->cryptlen + req->assoclen; 1174 else 1175 inputlen = req->cryptlen + req->assoclen - mac_len; 1176 1177 create_hmac_ctx_hdr(req, &argcnt, enc); 1178 update_input_data(req_info, req->src, inputlen, &argcnt); 1179 req_info->in_cnt = argcnt; 1180 } 1181 1182 static inline int create_aead_null_output_list(struct aead_request *req, 1183 u32 enc, u32 mac_len) 1184 { 1185 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1186 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1187 struct scatterlist *dst; 1188 u8 *ptr = NULL; 1189 int argcnt = 0, status, offset; 1190 u32 inputlen; 1191 1192 if (enc) 1193 inputlen = req->cryptlen + req->assoclen; 1194 else 1195 inputlen = req->cryptlen + req->assoclen - mac_len; 1196 1197 /* 1198 * If source and destination are different 1199 * then copy payload to destination 1200 */ 1201 if (req->src != req->dst) { 1202 1203 ptr = kmalloc(inputlen, (req_info->areq->flags & 1204 CRYPTO_TFM_REQ_MAY_SLEEP) ? 1205 GFP_KERNEL : GFP_ATOMIC); 1206 if (!ptr) 1207 return -ENOMEM; 1208 1209 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, 1210 inputlen); 1211 if (status != inputlen) { 1212 status = -EINVAL; 1213 goto error_free; 1214 } 1215 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, 1216 inputlen); 1217 if (status != inputlen) { 1218 status = -EINVAL; 1219 goto error_free; 1220 } 1221 kfree(ptr); 1222 } 1223 1224 if (enc) { 1225 /* 1226 * In an encryption scenario hmac needs 1227 * to be appended after payload 1228 */ 1229 dst = req->dst; 1230 offset = inputlen; 1231 while (offset >= dst->length) { 1232 offset -= dst->length; 1233 dst = sg_next(dst); 1234 if (!dst) 1235 return -ENOENT; 1236 } 1237 1238 update_output_data(req_info, dst, offset, mac_len, &argcnt); 1239 } else { 1240 /* 1241 * In a decryption scenario calculated hmac for received 1242 * payload needs to be compare with hmac received 1243 */ 1244 status = sg_copy_buffer(req->src, sg_nents(req->src), 1245 rctx->fctx.hmac.s.hmac_recv, mac_len, 1246 inputlen, true); 1247 if (status != mac_len) 1248 return -EINVAL; 1249 1250 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; 1251 req_info->out[argcnt].size = mac_len; 1252 argcnt++; 1253 } 1254 1255 req_info->out_cnt = argcnt; 1256 return 0; 1257 1258 error_free: 1259 kfree(ptr); 1260 return status; 1261 } 1262 1263 static int aead_do_fallback(struct aead_request *req, bool is_enc) 1264 { 1265 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1266 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1267 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead); 1268 int ret; 1269 1270 if (ctx->fbk_cipher) { 1271 /* Store the cipher tfm and then use the fallback tfm */ 1272 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher); 1273 aead_request_set_callback(&rctx->fbk_req, req->base.flags, 1274 req->base.complete, req->base.data); 1275 aead_request_set_crypt(&rctx->fbk_req, req->src, 1276 req->dst, req->cryptlen, req->iv); 1277 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : 1278 crypto_aead_decrypt(&rctx->fbk_req); 1279 } else { 1280 ret = -EINVAL; 1281 } 1282 1283 return ret; 1284 } 1285 1286 static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) 1287 { 1288 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); 1289 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1290 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1291 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); 1292 struct pci_dev *pdev; 1293 int status, cpu_num; 1294 1295 /* Clear control words */ 1296 rctx->ctrl_word.flags = 0; 1297 rctx->fctx.enc.enc_ctrl.u = 0; 1298 1299 req_info->callback = otx2_cpt_aead_callback; 1300 req_info->areq = &req->base; 1301 req_info->req_type = reg_type; 1302 req_info->is_enc = enc; 1303 req_info->is_trunc_hmac = false; 1304 1305 switch (reg_type) { 1306 case OTX2_CPT_AEAD_ENC_DEC_REQ: 1307 status = create_aead_input_list(req, enc); 1308 if (status) 1309 return status; 1310 create_aead_output_list(req, enc, crypto_aead_authsize(tfm)); 1311 break; 1312 1313 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ: 1314 create_aead_null_input_list(req, enc, 1315 crypto_aead_authsize(tfm)); 1316 status = create_aead_null_output_list(req, enc, 1317 crypto_aead_authsize(tfm)); 1318 if (status) 1319 return status; 1320 break; 1321 1322 default: 1323 return -EINVAL; 1324 } 1325 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len)) 1326 return -EINVAL; 1327 1328 if (!req_info->req.param2 || 1329 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) || 1330 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE)) 1331 return aead_do_fallback(req, enc); 1332 1333 status = get_se_device(&pdev, &cpu_num); 1334 if (status) 1335 return status; 1336 1337 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); 1338 1339 /* 1340 * We perform an asynchronous send and once 1341 * the request is completed the driver would 1342 * intimate through registered call back functions 1343 */ 1344 return otx2_cpt_do_request(pdev, req_info, cpu_num); 1345 } 1346 1347 static int otx2_cpt_aead_encrypt(struct aead_request *req) 1348 { 1349 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true); 1350 } 1351 1352 static int otx2_cpt_aead_decrypt(struct aead_request *req) 1353 { 1354 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false); 1355 } 1356 1357 static int otx2_cpt_aead_null_encrypt(struct aead_request *req) 1358 { 1359 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true); 1360 } 1361 1362 static int otx2_cpt_aead_null_decrypt(struct aead_request *req) 1363 { 1364 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false); 1365 } 1366 1367 static struct skcipher_alg otx2_cpt_skciphers[] = { { 1368 .base.cra_name = "xts(aes)", 1369 .base.cra_driver_name = "cpt_xts_aes", 1370 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1371 .base.cra_blocksize = AES_BLOCK_SIZE, 1372 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1373 .base.cra_alignmask = 7, 1374 .base.cra_priority = 4001, 1375 .base.cra_module = THIS_MODULE, 1376 1377 .init = otx2_cpt_enc_dec_init, 1378 .exit = otx2_cpt_skcipher_exit, 1379 .ivsize = AES_BLOCK_SIZE, 1380 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1381 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1382 .setkey = otx2_cpt_skcipher_xts_setkey, 1383 .encrypt = otx2_cpt_skcipher_encrypt, 1384 .decrypt = otx2_cpt_skcipher_decrypt, 1385 }, { 1386 .base.cra_name = "cbc(aes)", 1387 .base.cra_driver_name = "cpt_cbc_aes", 1388 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1389 .base.cra_blocksize = AES_BLOCK_SIZE, 1390 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1391 .base.cra_alignmask = 7, 1392 .base.cra_priority = 4001, 1393 .base.cra_module = THIS_MODULE, 1394 1395 .init = otx2_cpt_enc_dec_init, 1396 .exit = otx2_cpt_skcipher_exit, 1397 .ivsize = AES_BLOCK_SIZE, 1398 .min_keysize = AES_MIN_KEY_SIZE, 1399 .max_keysize = AES_MAX_KEY_SIZE, 1400 .setkey = otx2_cpt_skcipher_cbc_aes_setkey, 1401 .encrypt = otx2_cpt_skcipher_encrypt, 1402 .decrypt = otx2_cpt_skcipher_decrypt, 1403 }, { 1404 .base.cra_name = "ecb(aes)", 1405 .base.cra_driver_name = "cpt_ecb_aes", 1406 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1407 .base.cra_blocksize = AES_BLOCK_SIZE, 1408 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1409 .base.cra_alignmask = 7, 1410 .base.cra_priority = 4001, 1411 .base.cra_module = THIS_MODULE, 1412 1413 .init = otx2_cpt_enc_dec_init, 1414 .exit = otx2_cpt_skcipher_exit, 1415 .ivsize = 0, 1416 .min_keysize = AES_MIN_KEY_SIZE, 1417 .max_keysize = AES_MAX_KEY_SIZE, 1418 .setkey = otx2_cpt_skcipher_ecb_aes_setkey, 1419 .encrypt = otx2_cpt_skcipher_encrypt, 1420 .decrypt = otx2_cpt_skcipher_decrypt, 1421 }, { 1422 .base.cra_name = "cbc(des3_ede)", 1423 .base.cra_driver_name = "cpt_cbc_des3_ede", 1424 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1425 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1426 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1427 .base.cra_alignmask = 7, 1428 .base.cra_priority = 4001, 1429 .base.cra_module = THIS_MODULE, 1430 1431 .init = otx2_cpt_enc_dec_init, 1432 .exit = otx2_cpt_skcipher_exit, 1433 .min_keysize = DES3_EDE_KEY_SIZE, 1434 .max_keysize = DES3_EDE_KEY_SIZE, 1435 .ivsize = DES_BLOCK_SIZE, 1436 .setkey = otx2_cpt_skcipher_cbc_des3_setkey, 1437 .encrypt = otx2_cpt_skcipher_encrypt, 1438 .decrypt = otx2_cpt_skcipher_decrypt, 1439 }, { 1440 .base.cra_name = "ecb(des3_ede)", 1441 .base.cra_driver_name = "cpt_ecb_des3_ede", 1442 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1443 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1444 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1445 .base.cra_alignmask = 7, 1446 .base.cra_priority = 4001, 1447 .base.cra_module = THIS_MODULE, 1448 1449 .init = otx2_cpt_enc_dec_init, 1450 .exit = otx2_cpt_skcipher_exit, 1451 .min_keysize = DES3_EDE_KEY_SIZE, 1452 .max_keysize = DES3_EDE_KEY_SIZE, 1453 .ivsize = 0, 1454 .setkey = otx2_cpt_skcipher_ecb_des3_setkey, 1455 .encrypt = otx2_cpt_skcipher_encrypt, 1456 .decrypt = otx2_cpt_skcipher_decrypt, 1457 } }; 1458 1459 static struct aead_alg otx2_cpt_aeads[] = { { 1460 .base = { 1461 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1462 .cra_driver_name = "cpt_hmac_sha1_cbc_aes", 1463 .cra_blocksize = AES_BLOCK_SIZE, 1464 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1465 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1466 .cra_priority = 4001, 1467 .cra_alignmask = 0, 1468 .cra_module = THIS_MODULE, 1469 }, 1470 .init = otx2_cpt_aead_cbc_aes_sha1_init, 1471 .exit = otx2_cpt_aead_exit, 1472 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1473 .setauthsize = otx2_cpt_aead_set_authsize, 1474 .encrypt = otx2_cpt_aead_encrypt, 1475 .decrypt = otx2_cpt_aead_decrypt, 1476 .ivsize = AES_BLOCK_SIZE, 1477 .maxauthsize = SHA1_DIGEST_SIZE, 1478 }, { 1479 .base = { 1480 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1481 .cra_driver_name = "cpt_hmac_sha256_cbc_aes", 1482 .cra_blocksize = AES_BLOCK_SIZE, 1483 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1484 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1485 .cra_priority = 4001, 1486 .cra_alignmask = 0, 1487 .cra_module = THIS_MODULE, 1488 }, 1489 .init = otx2_cpt_aead_cbc_aes_sha256_init, 1490 .exit = otx2_cpt_aead_exit, 1491 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1492 .setauthsize = otx2_cpt_aead_set_authsize, 1493 .encrypt = otx2_cpt_aead_encrypt, 1494 .decrypt = otx2_cpt_aead_decrypt, 1495 .ivsize = AES_BLOCK_SIZE, 1496 .maxauthsize = SHA256_DIGEST_SIZE, 1497 }, { 1498 .base = { 1499 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1500 .cra_driver_name = "cpt_hmac_sha384_cbc_aes", 1501 .cra_blocksize = AES_BLOCK_SIZE, 1502 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1503 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1504 .cra_priority = 4001, 1505 .cra_alignmask = 0, 1506 .cra_module = THIS_MODULE, 1507 }, 1508 .init = otx2_cpt_aead_cbc_aes_sha384_init, 1509 .exit = otx2_cpt_aead_exit, 1510 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1511 .setauthsize = otx2_cpt_aead_set_authsize, 1512 .encrypt = otx2_cpt_aead_encrypt, 1513 .decrypt = otx2_cpt_aead_decrypt, 1514 .ivsize = AES_BLOCK_SIZE, 1515 .maxauthsize = SHA384_DIGEST_SIZE, 1516 }, { 1517 .base = { 1518 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1519 .cra_driver_name = "cpt_hmac_sha512_cbc_aes", 1520 .cra_blocksize = AES_BLOCK_SIZE, 1521 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1522 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1523 .cra_priority = 4001, 1524 .cra_alignmask = 0, 1525 .cra_module = THIS_MODULE, 1526 }, 1527 .init = otx2_cpt_aead_cbc_aes_sha512_init, 1528 .exit = otx2_cpt_aead_exit, 1529 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1530 .setauthsize = otx2_cpt_aead_set_authsize, 1531 .encrypt = otx2_cpt_aead_encrypt, 1532 .decrypt = otx2_cpt_aead_decrypt, 1533 .ivsize = AES_BLOCK_SIZE, 1534 .maxauthsize = SHA512_DIGEST_SIZE, 1535 }, { 1536 .base = { 1537 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", 1538 .cra_driver_name = "cpt_hmac_sha1_ecb_null", 1539 .cra_blocksize = 1, 1540 .cra_flags = CRYPTO_ALG_ASYNC, 1541 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1542 .cra_priority = 4001, 1543 .cra_alignmask = 0, 1544 .cra_module = THIS_MODULE, 1545 }, 1546 .init = otx2_cpt_aead_ecb_null_sha1_init, 1547 .exit = otx2_cpt_aead_exit, 1548 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1549 .setauthsize = otx2_cpt_aead_null_set_authsize, 1550 .encrypt = otx2_cpt_aead_null_encrypt, 1551 .decrypt = otx2_cpt_aead_null_decrypt, 1552 .ivsize = 0, 1553 .maxauthsize = SHA1_DIGEST_SIZE, 1554 }, { 1555 .base = { 1556 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", 1557 .cra_driver_name = "cpt_hmac_sha256_ecb_null", 1558 .cra_blocksize = 1, 1559 .cra_flags = CRYPTO_ALG_ASYNC, 1560 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1561 .cra_priority = 4001, 1562 .cra_alignmask = 0, 1563 .cra_module = THIS_MODULE, 1564 }, 1565 .init = otx2_cpt_aead_ecb_null_sha256_init, 1566 .exit = otx2_cpt_aead_exit, 1567 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1568 .setauthsize = otx2_cpt_aead_null_set_authsize, 1569 .encrypt = otx2_cpt_aead_null_encrypt, 1570 .decrypt = otx2_cpt_aead_null_decrypt, 1571 .ivsize = 0, 1572 .maxauthsize = SHA256_DIGEST_SIZE, 1573 }, { 1574 .base = { 1575 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", 1576 .cra_driver_name = "cpt_hmac_sha384_ecb_null", 1577 .cra_blocksize = 1, 1578 .cra_flags = CRYPTO_ALG_ASYNC, 1579 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1580 .cra_priority = 4001, 1581 .cra_alignmask = 0, 1582 .cra_module = THIS_MODULE, 1583 }, 1584 .init = otx2_cpt_aead_ecb_null_sha384_init, 1585 .exit = otx2_cpt_aead_exit, 1586 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1587 .setauthsize = otx2_cpt_aead_null_set_authsize, 1588 .encrypt = otx2_cpt_aead_null_encrypt, 1589 .decrypt = otx2_cpt_aead_null_decrypt, 1590 .ivsize = 0, 1591 .maxauthsize = SHA384_DIGEST_SIZE, 1592 }, { 1593 .base = { 1594 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", 1595 .cra_driver_name = "cpt_hmac_sha512_ecb_null", 1596 .cra_blocksize = 1, 1597 .cra_flags = CRYPTO_ALG_ASYNC, 1598 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1599 .cra_priority = 4001, 1600 .cra_alignmask = 0, 1601 .cra_module = THIS_MODULE, 1602 }, 1603 .init = otx2_cpt_aead_ecb_null_sha512_init, 1604 .exit = otx2_cpt_aead_exit, 1605 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1606 .setauthsize = otx2_cpt_aead_null_set_authsize, 1607 .encrypt = otx2_cpt_aead_null_encrypt, 1608 .decrypt = otx2_cpt_aead_null_decrypt, 1609 .ivsize = 0, 1610 .maxauthsize = SHA512_DIGEST_SIZE, 1611 }, { 1612 .base = { 1613 .cra_name = "rfc4106(gcm(aes))", 1614 .cra_driver_name = "cpt_rfc4106_gcm_aes", 1615 .cra_blocksize = 1, 1616 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1617 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), 1618 .cra_priority = 4001, 1619 .cra_alignmask = 0, 1620 .cra_module = THIS_MODULE, 1621 }, 1622 .init = otx2_cpt_aead_gcm_aes_init, 1623 .exit = otx2_cpt_aead_exit, 1624 .setkey = otx2_cpt_aead_gcm_aes_setkey, 1625 .setauthsize = otx2_cpt_aead_gcm_set_authsize, 1626 .encrypt = otx2_cpt_aead_encrypt, 1627 .decrypt = otx2_cpt_aead_decrypt, 1628 .ivsize = AES_GCM_IV_SIZE, 1629 .maxauthsize = AES_GCM_ICV_SIZE, 1630 } }; 1631 1632 static inline int cpt_register_algs(void) 1633 { 1634 int i, err = 0; 1635 1636 if (!IS_ENABLED(CONFIG_DM_CRYPT)) { 1637 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++) 1638 otx2_cpt_skciphers[i].base.cra_flags &= 1639 ~CRYPTO_ALG_DEAD; 1640 1641 err = crypto_register_skciphers(otx2_cpt_skciphers, 1642 ARRAY_SIZE(otx2_cpt_skciphers)); 1643 if (err) 1644 return err; 1645 } 1646 1647 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++) 1648 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1649 1650 err = crypto_register_aeads(otx2_cpt_aeads, 1651 ARRAY_SIZE(otx2_cpt_aeads)); 1652 if (err) { 1653 crypto_unregister_skciphers(otx2_cpt_skciphers, 1654 ARRAY_SIZE(otx2_cpt_skciphers)); 1655 return err; 1656 } 1657 1658 return 0; 1659 } 1660 1661 static inline void cpt_unregister_algs(void) 1662 { 1663 crypto_unregister_skciphers(otx2_cpt_skciphers, 1664 ARRAY_SIZE(otx2_cpt_skciphers)); 1665 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads)); 1666 } 1667 1668 static int compare_func(const void *lptr, const void *rptr) 1669 { 1670 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; 1671 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; 1672 1673 if (ldesc->dev->devfn < rdesc->dev->devfn) 1674 return -1; 1675 if (ldesc->dev->devfn > rdesc->dev->devfn) 1676 return 1; 1677 return 0; 1678 } 1679 1680 static void swap_func(void *lptr, void *rptr, int size) 1681 { 1682 struct cpt_device_desc *ldesc = lptr; 1683 struct cpt_device_desc *rdesc = rptr; 1684 struct cpt_device_desc desc; 1685 1686 desc = *ldesc; 1687 *ldesc = *rdesc; 1688 *rdesc = desc; 1689 } 1690 1691 int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, 1692 int num_queues, int num_devices) 1693 { 1694 int ret = 0; 1695 int count; 1696 1697 mutex_lock(&mutex); 1698 count = atomic_read(&se_devices.count); 1699 if (count >= OTX2_CPT_MAX_LFS_NUM) { 1700 dev_err(&pdev->dev, "No space to add a new device\n"); 1701 ret = -ENOSPC; 1702 goto unlock; 1703 } 1704 se_devices.desc[count].num_queues = num_queues; 1705 se_devices.desc[count++].dev = pdev; 1706 atomic_inc(&se_devices.count); 1707 1708 if (atomic_read(&se_devices.count) == num_devices && 1709 is_crypto_registered == false) { 1710 if (cpt_register_algs()) { 1711 dev_err(&pdev->dev, 1712 "Error in registering crypto algorithms\n"); 1713 ret = -EINVAL; 1714 goto unlock; 1715 } 1716 try_module_get(mod); 1717 is_crypto_registered = true; 1718 } 1719 sort(se_devices.desc, count, sizeof(struct cpt_device_desc), 1720 compare_func, swap_func); 1721 1722 unlock: 1723 mutex_unlock(&mutex); 1724 return ret; 1725 } 1726 1727 void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod) 1728 { 1729 struct cpt_device_table *dev_tbl; 1730 bool dev_found = false; 1731 int i, j, count; 1732 1733 mutex_lock(&mutex); 1734 1735 dev_tbl = &se_devices; 1736 count = atomic_read(&dev_tbl->count); 1737 for (i = 0; i < count; i++) { 1738 if (pdev == dev_tbl->desc[i].dev) { 1739 for (j = i; j < count-1; j++) 1740 dev_tbl->desc[j] = dev_tbl->desc[j+1]; 1741 dev_found = true; 1742 break; 1743 } 1744 } 1745 1746 if (!dev_found) { 1747 dev_err(&pdev->dev, "%s device not found\n", __func__); 1748 goto unlock; 1749 } 1750 if (atomic_dec_and_test(&se_devices.count)) { 1751 cpt_unregister_algs(); 1752 module_put(mod); 1753 is_crypto_registered = false; 1754 } 1755 1756 unlock: 1757 mutex_unlock(&mutex); 1758 } 1759