1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/crypto.h> 3 #include <linux/kernel.h> 4 #include <linux/module.h> 5 #include <linux/printk.h> 6 7 #include <crypto/aes.h> 8 #include <crypto/skcipher.h> 9 #include <crypto/ctr.h> 10 #include <crypto/des.h> 11 #include <crypto/xts.h> 12 13 #include "nitrox_dev.h" 14 #include "nitrox_common.h" 15 #include "nitrox_req.h" 16 17 #define PRIO 4001 18 19 struct nitrox_cipher { 20 const char *name; 21 enum flexi_cipher value; 22 }; 23 24 /** 25 * supported cipher list 26 */ 27 static const struct nitrox_cipher flexi_cipher_table[] = { 28 { "null", CIPHER_NULL }, 29 { "cbc(des3_ede)", CIPHER_3DES_CBC }, 30 { "ecb(des3_ede)", CIPHER_3DES_ECB }, 31 { "cbc(aes)", CIPHER_AES_CBC }, 32 { "ecb(aes)", CIPHER_AES_ECB }, 33 { "cfb(aes)", CIPHER_AES_CFB }, 34 { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, 35 { "xts(aes)", CIPHER_AES_XTS }, 36 { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, 37 { NULL, CIPHER_INVALID } 38 }; 39 40 static enum flexi_cipher flexi_cipher_type(const char *name) 41 { 42 const struct nitrox_cipher *cipher = flexi_cipher_table; 43 44 while (cipher->name) { 45 if (!strcmp(cipher->name, name)) 46 break; 47 cipher++; 48 } 49 return cipher->value; 50 } 51 52 static int flexi_aes_keylen(int keylen) 53 { 54 int aes_keylen; 55 56 switch (keylen) { 57 case AES_KEYSIZE_128: 58 aes_keylen = 1; 59 break; 60 case AES_KEYSIZE_192: 61 aes_keylen = 2; 62 break; 63 case AES_KEYSIZE_256: 64 aes_keylen = 3; 65 break; 66 default: 67 aes_keylen = -EINVAL; 68 break; 69 } 70 return aes_keylen; 71 } 72 73 static int nitrox_skcipher_init(struct crypto_skcipher *tfm) 74 { 75 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 76 struct crypto_ctx_hdr *chdr; 77 78 /* get the first device */ 79 nctx->ndev = nitrox_get_first_device(); 80 if (!nctx->ndev) 81 return -ENODEV; 82 83 /* allocate nitrox crypto context */ 84 chdr = crypto_alloc_context(nctx->ndev); 85 if (!chdr) { 86 nitrox_put_device(nctx->ndev); 87 return -ENOMEM; 88 } 89 nctx->chdr = chdr; 90 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + 91 sizeof(struct ctx_hdr)); 92 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + 93 sizeof(struct nitrox_kcrypt_request)); 94 return 0; 95 } 96 97 static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) 98 { 99 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 100 101 /* free the nitrox crypto context */ 102 if (nctx->u.ctx_handle) { 103 struct flexi_crypto_context *fctx = nctx->u.fctx; 104 105 memset(&fctx->crypto, 0, sizeof(struct crypto_keys)); 106 memset(&fctx->auth, 0, sizeof(struct auth_keys)); 107 crypto_free_context((void *)nctx->chdr); 108 } 109 nitrox_put_device(nctx->ndev); 110 111 nctx->u.ctx_handle = 0; 112 nctx->ndev = NULL; 113 } 114 115 static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, 116 int aes_keylen, const u8 *key, 117 unsigned int keylen) 118 { 119 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 120 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 121 struct flexi_crypto_context *fctx; 122 enum flexi_cipher cipher_type; 123 const char *name; 124 125 name = crypto_tfm_alg_name(tfm); 126 cipher_type = flexi_cipher_type(name); 127 if (unlikely(cipher_type == CIPHER_INVALID)) { 128 pr_err("unsupported cipher: %s\n", name); 129 return -EINVAL; 130 } 131 132 /* fill crypto context */ 133 fctx = nctx->u.fctx; 134 fctx->flags = 0; 135 fctx->w0.cipher_type = cipher_type; 136 fctx->w0.aes_keylen = aes_keylen; 137 fctx->w0.iv_source = IV_FROM_DPTR; 138 fctx->flags = cpu_to_be64(*(u64 *)&fctx->w0); 139 /* copy the key to context */ 140 memcpy(fctx->crypto.u.key, key, keylen); 141 142 return 0; 143 } 144 145 static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 146 unsigned int keylen) 147 { 148 int aes_keylen; 149 150 aes_keylen = flexi_aes_keylen(keylen); 151 if (aes_keylen < 0) { 152 crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 153 return -EINVAL; 154 } 155 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 156 } 157 158 static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize) 159 { 160 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 161 int nents = sg_nents(skreq->src) + 1; 162 struct se_crypto_request *creq = &nkreq->creq; 163 char *iv; 164 struct scatterlist *sg; 165 166 /* Allocate buffer to hold IV and input scatterlist array */ 167 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp); 168 if (!nkreq->src) 169 return -ENOMEM; 170 171 /* copy iv */ 172 iv = nkreq->src; 173 memcpy(iv, skreq->iv, ivsize); 174 175 sg = (struct scatterlist *)(iv + ivsize); 176 creq->src = sg; 177 sg_init_table(sg, nents); 178 179 /* Input format: 180 * +----+----------------+ 181 * | IV | SRC sg entries | 182 * +----+----------------+ 183 */ 184 185 /* IV */ 186 sg = create_single_sg(sg, iv, ivsize); 187 /* SRC entries */ 188 create_multi_sg(sg, skreq->src); 189 190 return 0; 191 } 192 193 static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize) 194 { 195 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 196 int nents = sg_nents(skreq->dst) + 3; 197 int extralen = ORH_HLEN + COMP_HLEN; 198 struct se_crypto_request *creq = &nkreq->creq; 199 struct scatterlist *sg; 200 char *iv = nkreq->src; 201 202 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist 203 * array 204 */ 205 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp); 206 if (!nkreq->dst) 207 return -ENOMEM; 208 209 creq->orh = (u64 *)(nkreq->dst); 210 set_orh_value(creq->orh); 211 212 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN); 213 set_comp_value(creq->comp); 214 215 sg = (struct scatterlist *)(nkreq->dst + ORH_HLEN + COMP_HLEN); 216 creq->dst = sg; 217 sg_init_table(sg, nents); 218 219 /* Output format: 220 * +-----+----+----------------+-----------------+ 221 * | ORH | IV | DST sg entries | COMPLETION Bytes| 222 * +-----+----+----------------+-----------------+ 223 */ 224 225 /* ORH */ 226 sg = create_single_sg(sg, creq->orh, ORH_HLEN); 227 /* IV */ 228 sg = create_single_sg(sg, iv, ivsize); 229 /* DST entries */ 230 sg = create_multi_sg(sg, skreq->dst); 231 /* COMPLETION Bytes */ 232 create_single_sg(sg, creq->comp, COMP_HLEN); 233 234 return 0; 235 } 236 237 static void free_src_sglist(struct skcipher_request *skreq) 238 { 239 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 240 241 kfree(nkreq->src); 242 } 243 244 static void free_dst_sglist(struct skcipher_request *skreq) 245 { 246 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 247 248 kfree(nkreq->dst); 249 } 250 251 static void nitrox_skcipher_callback(struct skcipher_request *skreq, 252 int err) 253 { 254 free_src_sglist(skreq); 255 free_dst_sglist(skreq); 256 if (err) { 257 pr_err_ratelimited("request failed status 0x%0x\n", err); 258 err = -EINVAL; 259 } 260 261 skcipher_request_complete(skreq, err); 262 } 263 264 static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) 265 { 266 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 267 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); 268 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 269 int ivsize = crypto_skcipher_ivsize(cipher); 270 struct se_crypto_request *creq; 271 int ret; 272 273 creq = &nkreq->creq; 274 creq->flags = skreq->base.flags; 275 creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 276 GFP_KERNEL : GFP_ATOMIC; 277 278 /* fill the request */ 279 creq->ctrl.value = 0; 280 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; 281 creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); 282 /* param0: length of the data to be encrypted */ 283 creq->gph.param0 = cpu_to_be16(skreq->cryptlen); 284 creq->gph.param1 = 0; 285 /* param2: encryption data offset */ 286 creq->gph.param2 = cpu_to_be16(ivsize); 287 creq->gph.param3 = 0; 288 289 creq->ctx_handle = nctx->u.ctx_handle; 290 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); 291 292 ret = alloc_src_sglist(skreq, ivsize); 293 if (ret) 294 return ret; 295 296 ret = alloc_dst_sglist(skreq, ivsize); 297 if (ret) { 298 free_src_sglist(skreq); 299 return ret; 300 } 301 302 nkreq->nctx = nctx; 303 nkreq->skreq = skreq; 304 305 /* send the crypto request */ 306 return nitrox_process_se_request(nctx->ndev, creq, 307 nitrox_skcipher_callback, skreq); 308 } 309 310 static int nitrox_aes_encrypt(struct skcipher_request *skreq) 311 { 312 return nitrox_skcipher_crypt(skreq, true); 313 } 314 315 static int nitrox_aes_decrypt(struct skcipher_request *skreq) 316 { 317 return nitrox_skcipher_crypt(skreq, false); 318 } 319 320 static int nitrox_3des_setkey(struct crypto_skcipher *cipher, 321 const u8 *key, unsigned int keylen) 322 { 323 if (keylen != DES3_EDE_KEY_SIZE) { 324 crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 325 return -EINVAL; 326 } 327 328 return nitrox_skcipher_setkey(cipher, 0, key, keylen); 329 } 330 331 static int nitrox_3des_encrypt(struct skcipher_request *skreq) 332 { 333 return nitrox_skcipher_crypt(skreq, true); 334 } 335 336 static int nitrox_3des_decrypt(struct skcipher_request *skreq) 337 { 338 return nitrox_skcipher_crypt(skreq, false); 339 } 340 341 static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, 342 const u8 *key, unsigned int keylen) 343 { 344 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 345 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 346 struct flexi_crypto_context *fctx; 347 int aes_keylen, ret; 348 349 ret = xts_check_key(tfm, key, keylen); 350 if (ret) 351 return ret; 352 353 keylen /= 2; 354 355 aes_keylen = flexi_aes_keylen(keylen); 356 if (aes_keylen < 0) { 357 crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 358 return -EINVAL; 359 } 360 361 fctx = nctx->u.fctx; 362 /* copy KEY2 */ 363 memcpy(fctx->auth.u.key2, (key + keylen), keylen); 364 365 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 366 } 367 368 static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, 369 const u8 *key, unsigned int keylen) 370 { 371 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 372 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 373 struct flexi_crypto_context *fctx; 374 int aes_keylen; 375 376 if (keylen < CTR_RFC3686_NONCE_SIZE) 377 return -EINVAL; 378 379 fctx = nctx->u.fctx; 380 381 memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), 382 CTR_RFC3686_NONCE_SIZE); 383 384 keylen -= CTR_RFC3686_NONCE_SIZE; 385 386 aes_keylen = flexi_aes_keylen(keylen); 387 if (aes_keylen < 0) { 388 crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 389 return -EINVAL; 390 } 391 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 392 } 393 394 static struct skcipher_alg nitrox_skciphers[] = { { 395 .base = { 396 .cra_name = "cbc(aes)", 397 .cra_driver_name = "n5_cbc(aes)", 398 .cra_priority = PRIO, 399 .cra_flags = CRYPTO_ALG_ASYNC, 400 .cra_blocksize = AES_BLOCK_SIZE, 401 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 402 .cra_alignmask = 0, 403 .cra_module = THIS_MODULE, 404 }, 405 .min_keysize = AES_MIN_KEY_SIZE, 406 .max_keysize = AES_MAX_KEY_SIZE, 407 .ivsize = AES_BLOCK_SIZE, 408 .setkey = nitrox_aes_setkey, 409 .encrypt = nitrox_aes_encrypt, 410 .decrypt = nitrox_aes_decrypt, 411 .init = nitrox_skcipher_init, 412 .exit = nitrox_skcipher_exit, 413 }, { 414 .base = { 415 .cra_name = "ecb(aes)", 416 .cra_driver_name = "n5_ecb(aes)", 417 .cra_priority = PRIO, 418 .cra_flags = CRYPTO_ALG_ASYNC, 419 .cra_blocksize = AES_BLOCK_SIZE, 420 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 421 .cra_alignmask = 0, 422 .cra_module = THIS_MODULE, 423 }, 424 .min_keysize = AES_MIN_KEY_SIZE, 425 .max_keysize = AES_MAX_KEY_SIZE, 426 .ivsize = AES_BLOCK_SIZE, 427 .setkey = nitrox_aes_setkey, 428 .encrypt = nitrox_aes_encrypt, 429 .decrypt = nitrox_aes_decrypt, 430 .init = nitrox_skcipher_init, 431 .exit = nitrox_skcipher_exit, 432 }, { 433 .base = { 434 .cra_name = "cfb(aes)", 435 .cra_driver_name = "n5_cfb(aes)", 436 .cra_priority = PRIO, 437 .cra_flags = CRYPTO_ALG_ASYNC, 438 .cra_blocksize = AES_BLOCK_SIZE, 439 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 440 .cra_alignmask = 0, 441 .cra_module = THIS_MODULE, 442 }, 443 .min_keysize = AES_MIN_KEY_SIZE, 444 .max_keysize = AES_MAX_KEY_SIZE, 445 .ivsize = AES_BLOCK_SIZE, 446 .setkey = nitrox_aes_setkey, 447 .encrypt = nitrox_aes_encrypt, 448 .decrypt = nitrox_aes_decrypt, 449 .init = nitrox_skcipher_init, 450 .exit = nitrox_skcipher_exit, 451 }, { 452 .base = { 453 .cra_name = "xts(aes)", 454 .cra_driver_name = "n5_xts(aes)", 455 .cra_priority = PRIO, 456 .cra_flags = CRYPTO_ALG_ASYNC, 457 .cra_blocksize = AES_BLOCK_SIZE, 458 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 459 .cra_alignmask = 0, 460 .cra_module = THIS_MODULE, 461 }, 462 .min_keysize = 2 * AES_MIN_KEY_SIZE, 463 .max_keysize = 2 * AES_MAX_KEY_SIZE, 464 .ivsize = AES_BLOCK_SIZE, 465 .setkey = nitrox_aes_xts_setkey, 466 .encrypt = nitrox_aes_encrypt, 467 .decrypt = nitrox_aes_decrypt, 468 .init = nitrox_skcipher_init, 469 .exit = nitrox_skcipher_exit, 470 }, { 471 .base = { 472 .cra_name = "rfc3686(ctr(aes))", 473 .cra_driver_name = "n5_rfc3686(ctr(aes))", 474 .cra_priority = PRIO, 475 .cra_flags = CRYPTO_ALG_ASYNC, 476 .cra_blocksize = 1, 477 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 478 .cra_alignmask = 0, 479 .cra_module = THIS_MODULE, 480 }, 481 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 482 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 483 .ivsize = CTR_RFC3686_IV_SIZE, 484 .init = nitrox_skcipher_init, 485 .exit = nitrox_skcipher_exit, 486 .setkey = nitrox_aes_ctr_rfc3686_setkey, 487 .encrypt = nitrox_aes_encrypt, 488 .decrypt = nitrox_aes_decrypt, 489 }, { 490 .base = { 491 .cra_name = "cts(cbc(aes))", 492 .cra_driver_name = "n5_cts(cbc(aes))", 493 .cra_priority = PRIO, 494 .cra_flags = CRYPTO_ALG_ASYNC, 495 .cra_blocksize = AES_BLOCK_SIZE, 496 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 497 .cra_alignmask = 0, 498 .cra_type = &crypto_ablkcipher_type, 499 .cra_module = THIS_MODULE, 500 }, 501 .min_keysize = AES_MIN_KEY_SIZE, 502 .max_keysize = AES_MAX_KEY_SIZE, 503 .ivsize = AES_BLOCK_SIZE, 504 .setkey = nitrox_aes_setkey, 505 .encrypt = nitrox_aes_encrypt, 506 .decrypt = nitrox_aes_decrypt, 507 .init = nitrox_skcipher_init, 508 .exit = nitrox_skcipher_exit, 509 }, { 510 .base = { 511 .cra_name = "cbc(des3_ede)", 512 .cra_driver_name = "n5_cbc(des3_ede)", 513 .cra_priority = PRIO, 514 .cra_flags = CRYPTO_ALG_ASYNC, 515 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 516 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 517 .cra_alignmask = 0, 518 .cra_module = THIS_MODULE, 519 }, 520 .min_keysize = DES3_EDE_KEY_SIZE, 521 .max_keysize = DES3_EDE_KEY_SIZE, 522 .ivsize = DES3_EDE_BLOCK_SIZE, 523 .setkey = nitrox_3des_setkey, 524 .encrypt = nitrox_3des_encrypt, 525 .decrypt = nitrox_3des_decrypt, 526 .init = nitrox_skcipher_init, 527 .exit = nitrox_skcipher_exit, 528 }, { 529 .base = { 530 .cra_name = "ecb(des3_ede)", 531 .cra_driver_name = "n5_ecb(des3_ede)", 532 .cra_priority = PRIO, 533 .cra_flags = CRYPTO_ALG_ASYNC, 534 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 535 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 536 .cra_alignmask = 0, 537 .cra_module = THIS_MODULE, 538 }, 539 .min_keysize = DES3_EDE_KEY_SIZE, 540 .max_keysize = DES3_EDE_KEY_SIZE, 541 .ivsize = DES3_EDE_BLOCK_SIZE, 542 .setkey = nitrox_3des_setkey, 543 .encrypt = nitrox_3des_encrypt, 544 .decrypt = nitrox_3des_decrypt, 545 .init = nitrox_skcipher_init, 546 .exit = nitrox_skcipher_exit, 547 } 548 549 }; 550 551 int nitrox_crypto_register(void) 552 { 553 return crypto_register_skciphers(nitrox_skciphers, 554 ARRAY_SIZE(nitrox_skciphers)); 555 } 556 557 void nitrox_crypto_unregister(void) 558 { 559 crypto_unregister_skciphers(nitrox_skciphers, 560 ARRAY_SIZE(nitrox_skciphers)); 561 } 562