1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright (C) 2016 Cavium, Inc. 5 */ 6 7 #include <crypto/aes.h> 8 #include <crypto/algapi.h> 9 #include <crypto/authenc.h> 10 #include <crypto/crypto_wq.h> 11 #include <crypto/des.h> 12 #include <crypto/xts.h> 13 #include <linux/crypto.h> 14 #include <linux/err.h> 15 #include <linux/list.h> 16 #include <linux/scatterlist.h> 17 18 #include "cptvf.h" 19 #include "cptvf_algs.h" 20 21 struct cpt_device_handle { 22 void *cdev[MAX_DEVICES]; 23 u32 dev_count; 24 }; 25 26 static struct cpt_device_handle dev_handle; 27 28 static void cvm_callback(u32 status, void *arg) 29 { 30 struct crypto_async_request *req = (struct crypto_async_request *)arg; 31 32 req->complete(req, !status); 33 } 34 35 static inline void update_input_iv(struct cpt_request_info *req_info, 36 u8 *iv, u32 enc_iv_len, 37 u32 *argcnt) 38 { 39 /* Setting the iv information */ 40 req_info->in[*argcnt].vptr = (void *)iv; 41 req_info->in[*argcnt].size = enc_iv_len; 42 req_info->req.dlen += enc_iv_len; 43 44 ++(*argcnt); 45 } 46 47 static inline void update_output_iv(struct cpt_request_info *req_info, 48 u8 *iv, u32 enc_iv_len, 49 u32 *argcnt) 50 { 51 /* Setting the iv information */ 52 req_info->out[*argcnt].vptr = (void *)iv; 53 req_info->out[*argcnt].size = enc_iv_len; 54 req_info->rlen += enc_iv_len; 55 56 ++(*argcnt); 57 } 58 59 static inline void update_input_data(struct cpt_request_info *req_info, 60 struct scatterlist *inp_sg, 61 u32 nbytes, u32 *argcnt) 62 { 63 req_info->req.dlen += nbytes; 64 65 while (nbytes) { 66 u32 len = min(nbytes, inp_sg->length); 67 u8 *ptr = sg_virt(inp_sg); 68 69 req_info->in[*argcnt].vptr = (void *)ptr; 70 req_info->in[*argcnt].size = len; 71 nbytes -= len; 72 73 ++(*argcnt); 74 ++inp_sg; 75 } 76 } 77 78 static inline void update_output_data(struct cpt_request_info *req_info, 79 struct scatterlist *outp_sg, 80 u32 nbytes, u32 *argcnt) 81 { 82 req_info->rlen += nbytes; 83 84 while (nbytes) { 85 u32 len = min(nbytes, outp_sg->length); 86 u8 *ptr = sg_virt(outp_sg); 87 88 req_info->out[*argcnt].vptr = (void *)ptr; 89 req_info->out[*argcnt].size = len; 90 nbytes -= len; 91 ++(*argcnt); 92 ++outp_sg; 93 } 94 } 95 96 static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc, 97 u32 *argcnt) 98 { 99 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 100 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm); 101 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); 102 struct fc_context *fctx = &rctx->fctx; 103 u64 *offset_control = &rctx->control_word; 104 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm); 105 struct cpt_request_info *req_info = &rctx->cpt_req; 106 u64 *ctrl_flags = NULL; 107 108 req_info->ctrl.s.grp = 0; 109 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; 110 req_info->ctrl.s.se_req = SE_CORE_REQ; 111 112 req_info->req.opcode.s.major = MAJOR_OP_FC | 113 DMA_MODE_FLAG(DMA_GATHER_SCATTER); 114 if (enc) 115 req_info->req.opcode.s.minor = 2; 116 else 117 req_info->req.opcode.s.minor = 3; 118 119 req_info->req.param1 = req->nbytes; /* Encryption Data length */ 120 req_info->req.param2 = 0; /*Auth data length */ 121 122 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 123 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 124 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; 125 126 if (ctx->cipher_type == AES_XTS) 127 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 128 else 129 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 130 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags; 131 *ctrl_flags = cpu_to_be64(*ctrl_flags); 132 133 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); 134 /* Storing Packet Data Information in offset 135 * Control Word First 8 bytes 136 */ 137 req_info->in[*argcnt].vptr = (u8 *)offset_control; 138 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 139 req_info->req.dlen += CONTROL_WORD_LEN; 140 ++(*argcnt); 141 142 req_info->in[*argcnt].vptr = (u8 *)fctx; 143 req_info->in[*argcnt].size = sizeof(struct fc_context); 144 req_info->req.dlen += sizeof(struct fc_context); 145 146 ++(*argcnt); 147 148 return 0; 149 } 150 151 static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc, 152 u32 enc_iv_len) 153 { 154 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); 155 struct cpt_request_info *req_info = &rctx->cpt_req; 156 u32 argcnt = 0; 157 158 create_ctx_hdr(req, enc, &argcnt); 159 update_input_iv(req_info, req->info, enc_iv_len, &argcnt); 160 update_input_data(req_info, req->src, req->nbytes, &argcnt); 161 req_info->incnt = argcnt; 162 163 return 0; 164 } 165 166 static inline void store_cb_info(struct ablkcipher_request *req, 167 struct cpt_request_info *req_info) 168 { 169 req_info->callback = (void *)cvm_callback; 170 req_info->callback_arg = (void *)&req->base; 171 } 172 173 static inline void create_output_list(struct ablkcipher_request *req, 174 u32 enc_iv_len) 175 { 176 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); 177 struct cpt_request_info *req_info = &rctx->cpt_req; 178 u32 argcnt = 0; 179 180 /* OUTPUT Buffer Processing 181 * AES encryption/decryption output would be 182 * received in the following format 183 * 184 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 185 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 186 */ 187 /* Reading IV information */ 188 update_output_iv(req_info, req->info, enc_iv_len, &argcnt); 189 update_output_data(req_info, req->dst, req->nbytes, &argcnt); 190 req_info->outcnt = argcnt; 191 } 192 193 static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc) 194 { 195 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 196 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); 197 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm); 198 struct fc_context *fctx = &rctx->fctx; 199 struct cpt_request_info *req_info = &rctx->cpt_req; 200 void *cdev = NULL; 201 int status; 202 203 memset(req_info, 0, sizeof(struct cpt_request_info)); 204 memset(fctx, 0, sizeof(struct fc_context)); 205 create_input_list(req, enc, enc_iv_len); 206 create_output_list(req, enc_iv_len); 207 store_cb_info(req, req_info); 208 cdev = dev_handle.cdev[smp_processor_id()]; 209 status = cptvf_do_request(cdev, req_info); 210 /* We perform an asynchronous send and once 211 * the request is completed the driver would 212 * intimate through registered call back functions 213 */ 214 215 if (status) 216 return status; 217 else 218 return -EINPROGRESS; 219 } 220 221 static int cvm_encrypt(struct ablkcipher_request *req) 222 { 223 return cvm_enc_dec(req, true); 224 } 225 226 static int cvm_decrypt(struct ablkcipher_request *req) 227 { 228 return cvm_enc_dec(req, false); 229 } 230 231 static int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 232 u32 keylen) 233 { 234 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 235 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); 236 int err; 237 const u8 *key1 = key; 238 const u8 *key2 = key + (keylen / 2); 239 240 err = xts_check_key(tfm, key, keylen); 241 if (err) 242 return err; 243 ctx->key_len = keylen; 244 memcpy(ctx->enc_key, key1, keylen / 2); 245 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 246 ctx->cipher_type = AES_XTS; 247 switch (ctx->key_len) { 248 case 32: 249 ctx->key_type = AES_128_BIT; 250 break; 251 case 64: 252 ctx->key_type = AES_256_BIT; 253 break; 254 default: 255 return -EINVAL; 256 } 257 258 return 0; 259 } 260 261 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) 262 { 263 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { 264 ctx->key_len = keylen; 265 switch (ctx->key_len) { 266 case 16: 267 ctx->key_type = AES_128_BIT; 268 break; 269 case 24: 270 ctx->key_type = AES_192_BIT; 271 break; 272 case 32: 273 ctx->key_type = AES_256_BIT; 274 break; 275 default: 276 return -EINVAL; 277 } 278 279 if (ctx->cipher_type == DES3_CBC) 280 ctx->key_type = 0; 281 282 return 0; 283 } 284 285 return -EINVAL; 286 } 287 288 static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 289 u32 keylen, u8 cipher_type) 290 { 291 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 292 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); 293 294 ctx->cipher_type = cipher_type; 295 if (!cvm_validate_keylen(ctx, keylen)) { 296 memcpy(ctx->enc_key, key, keylen); 297 return 0; 298 } else { 299 crypto_ablkcipher_set_flags(cipher, 300 CRYPTO_TFM_RES_BAD_KEY_LEN); 301 return -EINVAL; 302 } 303 } 304 305 static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 306 u32 keylen) 307 { 308 return cvm_setkey(cipher, key, keylen, AES_CBC); 309 } 310 311 static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 312 u32 keylen) 313 { 314 return cvm_setkey(cipher, key, keylen, AES_ECB); 315 } 316 317 static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 318 u32 keylen) 319 { 320 return cvm_setkey(cipher, key, keylen, AES_CFB); 321 } 322 323 static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 324 u32 keylen) 325 { 326 u32 flags = crypto_ablkcipher_get_flags(cipher); 327 int err; 328 329 err = __des3_verify_key(&flags, key); 330 if (unlikely(err)) { 331 crypto_ablkcipher_set_flags(cipher, flags); 332 return err; 333 } 334 335 return cvm_setkey(cipher, key, keylen, DES3_CBC); 336 } 337 338 static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 339 u32 keylen) 340 { 341 u32 flags = crypto_ablkcipher_get_flags(cipher); 342 int err; 343 344 err = __des3_verify_key(&flags, key); 345 if (unlikely(err)) { 346 crypto_ablkcipher_set_flags(cipher, flags); 347 return err; 348 } 349 350 return cvm_setkey(cipher, key, keylen, DES3_ECB); 351 } 352 353 static int cvm_enc_dec_init(struct crypto_tfm *tfm) 354 { 355 tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx); 356 return 0; 357 } 358 359 static struct crypto_alg algs[] = { { 360 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 361 .cra_blocksize = AES_BLOCK_SIZE, 362 .cra_ctxsize = sizeof(struct cvm_enc_ctx), 363 .cra_alignmask = 7, 364 .cra_priority = 4001, 365 .cra_name = "xts(aes)", 366 .cra_driver_name = "cavium-xts-aes", 367 .cra_type = &crypto_ablkcipher_type, 368 .cra_u = { 369 .ablkcipher = { 370 .ivsize = AES_BLOCK_SIZE, 371 .min_keysize = 2 * AES_MIN_KEY_SIZE, 372 .max_keysize = 2 * AES_MAX_KEY_SIZE, 373 .setkey = cvm_xts_setkey, 374 .encrypt = cvm_encrypt, 375 .decrypt = cvm_decrypt, 376 }, 377 }, 378 .cra_init = cvm_enc_dec_init, 379 .cra_module = THIS_MODULE, 380 }, { 381 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 382 .cra_blocksize = AES_BLOCK_SIZE, 383 .cra_ctxsize = sizeof(struct cvm_enc_ctx), 384 .cra_alignmask = 7, 385 .cra_priority = 4001, 386 .cra_name = "cbc(aes)", 387 .cra_driver_name = "cavium-cbc-aes", 388 .cra_type = &crypto_ablkcipher_type, 389 .cra_u = { 390 .ablkcipher = { 391 .ivsize = AES_BLOCK_SIZE, 392 .min_keysize = AES_MIN_KEY_SIZE, 393 .max_keysize = AES_MAX_KEY_SIZE, 394 .setkey = cvm_cbc_aes_setkey, 395 .encrypt = cvm_encrypt, 396 .decrypt = cvm_decrypt, 397 }, 398 }, 399 .cra_init = cvm_enc_dec_init, 400 .cra_module = THIS_MODULE, 401 }, { 402 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 403 .cra_blocksize = AES_BLOCK_SIZE, 404 .cra_ctxsize = sizeof(struct cvm_enc_ctx), 405 .cra_alignmask = 7, 406 .cra_priority = 4001, 407 .cra_name = "ecb(aes)", 408 .cra_driver_name = "cavium-ecb-aes", 409 .cra_type = &crypto_ablkcipher_type, 410 .cra_u = { 411 .ablkcipher = { 412 .ivsize = AES_BLOCK_SIZE, 413 .min_keysize = AES_MIN_KEY_SIZE, 414 .max_keysize = AES_MAX_KEY_SIZE, 415 .setkey = cvm_ecb_aes_setkey, 416 .encrypt = cvm_encrypt, 417 .decrypt = cvm_decrypt, 418 }, 419 }, 420 .cra_init = cvm_enc_dec_init, 421 .cra_module = THIS_MODULE, 422 }, { 423 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 424 .cra_blocksize = AES_BLOCK_SIZE, 425 .cra_ctxsize = sizeof(struct cvm_enc_ctx), 426 .cra_alignmask = 7, 427 .cra_priority = 4001, 428 .cra_name = "cfb(aes)", 429 .cra_driver_name = "cavium-cfb-aes", 430 .cra_type = &crypto_ablkcipher_type, 431 .cra_u = { 432 .ablkcipher = { 433 .ivsize = AES_BLOCK_SIZE, 434 .min_keysize = AES_MIN_KEY_SIZE, 435 .max_keysize = AES_MAX_KEY_SIZE, 436 .setkey = cvm_cfb_aes_setkey, 437 .encrypt = cvm_encrypt, 438 .decrypt = cvm_decrypt, 439 }, 440 }, 441 .cra_init = cvm_enc_dec_init, 442 .cra_module = THIS_MODULE, 443 }, { 444 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 445 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 446 .cra_ctxsize = sizeof(struct cvm_des3_ctx), 447 .cra_alignmask = 7, 448 .cra_priority = 4001, 449 .cra_name = "cbc(des3_ede)", 450 .cra_driver_name = "cavium-cbc-des3_ede", 451 .cra_type = &crypto_ablkcipher_type, 452 .cra_u = { 453 .ablkcipher = { 454 .min_keysize = DES3_EDE_KEY_SIZE, 455 .max_keysize = DES3_EDE_KEY_SIZE, 456 .ivsize = DES_BLOCK_SIZE, 457 .setkey = cvm_cbc_des3_setkey, 458 .encrypt = cvm_encrypt, 459 .decrypt = cvm_decrypt, 460 }, 461 }, 462 .cra_init = cvm_enc_dec_init, 463 .cra_module = THIS_MODULE, 464 }, { 465 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 466 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 467 .cra_ctxsize = sizeof(struct cvm_des3_ctx), 468 .cra_alignmask = 7, 469 .cra_priority = 4001, 470 .cra_name = "ecb(des3_ede)", 471 .cra_driver_name = "cavium-ecb-des3_ede", 472 .cra_type = &crypto_ablkcipher_type, 473 .cra_u = { 474 .ablkcipher = { 475 .min_keysize = DES3_EDE_KEY_SIZE, 476 .max_keysize = DES3_EDE_KEY_SIZE, 477 .ivsize = DES_BLOCK_SIZE, 478 .setkey = cvm_ecb_des3_setkey, 479 .encrypt = cvm_encrypt, 480 .decrypt = cvm_decrypt, 481 }, 482 }, 483 .cra_init = cvm_enc_dec_init, 484 .cra_module = THIS_MODULE, 485 } }; 486 487 static inline int cav_register_algs(void) 488 { 489 int err = 0; 490 491 err = crypto_register_algs(algs, ARRAY_SIZE(algs)); 492 if (err) 493 return err; 494 495 return 0; 496 } 497 498 static inline void cav_unregister_algs(void) 499 { 500 crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 501 } 502 503 int cvm_crypto_init(struct cpt_vf *cptvf) 504 { 505 struct pci_dev *pdev = cptvf->pdev; 506 u32 dev_count; 507 508 dev_count = dev_handle.dev_count; 509 dev_handle.cdev[dev_count] = cptvf; 510 dev_handle.dev_count++; 511 512 if (dev_count == 3) { 513 if (cav_register_algs()) { 514 dev_err(&pdev->dev, "Error in registering crypto algorithms\n"); 515 return -EINVAL; 516 } 517 } 518 519 return 0; 520 } 521 522 void cvm_crypto_exit(void) 523 { 524 u32 dev_count; 525 526 dev_count = --dev_handle.dev_count; 527 if (!dev_count) 528 cav_unregister_algs(); 529 } 530