1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/printk.h> 4 #include <linux/crypto.h> 5 #include <linux/rtnetlink.h> 6 7 #include <crypto/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/des.h> 10 #include <crypto/sha.h> 11 #include <crypto/internal/aead.h> 12 #include <crypto/scatterwalk.h> 13 #include <crypto/gcm.h> 14 15 #include "nitrox_dev.h" 16 #include "nitrox_common.h" 17 #include "nitrox_req.h" 18 19 #define GCM_AES_SALT_SIZE 4 20 21 union gph_p3 { 22 struct { 23 #ifdef __BIG_ENDIAN_BITFIELD 24 u16 iv_offset : 8; 25 u16 auth_offset : 8; 26 #else 27 u16 auth_offset : 8; 28 u16 iv_offset : 8; 29 #endif 30 }; 31 u16 param; 32 }; 33 34 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, 35 unsigned int keylen) 36 { 37 int aes_keylen; 38 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 39 struct flexi_crypto_context *fctx; 40 union fc_ctx_flags flags; 41 42 aes_keylen = flexi_aes_keylen(keylen); 43 if (aes_keylen < 0) { 44 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 45 return -EINVAL; 46 } 47 48 /* fill crypto context */ 49 fctx = nctx->u.fctx; 50 flags.f = be64_to_cpu(fctx->flags.f); 51 flags.w0.aes_keylen = aes_keylen; 52 fctx->flags.f = cpu_to_be64(flags.f); 53 54 /* copy enc key to context */ 55 memset(&fctx->crypto, 0, sizeof(fctx->crypto)); 56 memcpy(fctx->crypto.u.key, key, keylen); 57 58 return 0; 59 } 60 61 static int nitrox_aead_setauthsize(struct crypto_aead *aead, 62 unsigned int authsize) 63 { 64 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 65 struct flexi_crypto_context *fctx = nctx->u.fctx; 66 union fc_ctx_flags flags; 67 68 flags.f = be64_to_cpu(fctx->flags.f); 69 flags.w0.mac_len = authsize; 70 fctx->flags.f = cpu_to_be64(flags.f); 71 72 aead->authsize = authsize; 73 74 return 0; 75 } 76 77 static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq, 78 struct scatterlist *src, char *iv, int ivsize, 79 int buflen) 80 { 81 int nents = sg_nents_for_len(src, buflen); 82 int ret; 83 84 if (nents < 0) 85 return nents; 86 87 /* IV entry */ 88 nents += 1; 89 /* Allocate buffer to hold IV and input scatterlist array */ 90 ret = alloc_src_req_buf(nkreq, nents, ivsize); 91 if (ret) 92 return ret; 93 94 nitrox_creq_copy_iv(nkreq->src, iv, ivsize); 95 nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen); 96 97 return 0; 98 } 99 100 static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq, 101 struct scatterlist *dst, int ivsize, int buflen) 102 { 103 int nents = sg_nents_for_len(dst, buflen); 104 int ret; 105 106 if (nents < 0) 107 return nents; 108 109 /* IV, ORH, COMPLETION entries */ 110 nents += 3; 111 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist 112 * array 113 */ 114 ret = alloc_dst_req_buf(nkreq, nents); 115 if (ret) 116 return ret; 117 118 nitrox_creq_set_orh(nkreq); 119 nitrox_creq_set_comp(nkreq); 120 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen); 121 122 return 0; 123 } 124 125 static void free_src_sglist(struct nitrox_kcrypt_request *nkreq) 126 { 127 kfree(nkreq->src); 128 } 129 130 static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq) 131 { 132 kfree(nkreq->dst); 133 } 134 135 static int nitrox_set_creq(struct nitrox_aead_rctx *rctx) 136 { 137 struct se_crypto_request *creq = &rctx->nkreq.creq; 138 union gph_p3 param3; 139 int ret; 140 141 creq->flags = rctx->flags; 142 creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : 143 GFP_ATOMIC; 144 145 creq->ctrl.value = 0; 146 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; 147 creq->ctrl.s.arg = rctx->ctrl_arg; 148 149 creq->gph.param0 = cpu_to_be16(rctx->cryptlen); 150 creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen); 151 creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen); 152 param3.iv_offset = 0; 153 param3.auth_offset = rctx->ivsize; 154 creq->gph.param3 = cpu_to_be16(param3.param); 155 156 creq->ctx_handle = rctx->ctx_handle; 157 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); 158 159 ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize, 160 rctx->srclen); 161 if (ret) 162 return ret; 163 164 ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize, 165 rctx->dstlen); 166 if (ret) { 167 free_src_sglist(&rctx->nkreq); 168 return ret; 169 } 170 171 return 0; 172 } 173 174 static void nitrox_aead_callback(void *arg, int err) 175 { 176 struct aead_request *areq = arg; 177 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); 178 179 free_src_sglist(&rctx->nkreq); 180 free_dst_sglist(&rctx->nkreq); 181 if (err) { 182 pr_err_ratelimited("request failed status 0x%0x\n", err); 183 err = -EINVAL; 184 } 185 186 areq->base.complete(&areq->base, err); 187 } 188 189 static int nitrox_aes_gcm_enc(struct aead_request *areq) 190 { 191 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 192 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 193 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); 194 struct se_crypto_request *creq = &rctx->nkreq.creq; 195 struct flexi_crypto_context *fctx = nctx->u.fctx; 196 int ret; 197 198 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); 199 200 rctx->cryptlen = areq->cryptlen; 201 rctx->assoclen = areq->assoclen; 202 rctx->srclen = areq->assoclen + areq->cryptlen; 203 rctx->dstlen = rctx->srclen + aead->authsize; 204 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; 205 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; 206 rctx->flags = areq->base.flags; 207 rctx->ctx_handle = nctx->u.ctx_handle; 208 rctx->src = areq->src; 209 rctx->dst = areq->dst; 210 rctx->ctrl_arg = ENCRYPT; 211 ret = nitrox_set_creq(rctx); 212 if (ret) 213 return ret; 214 215 /* send the crypto request */ 216 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, 217 areq); 218 } 219 220 static int nitrox_aes_gcm_dec(struct aead_request *areq) 221 { 222 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 223 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 224 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); 225 struct se_crypto_request *creq = &rctx->nkreq.creq; 226 struct flexi_crypto_context *fctx = nctx->u.fctx; 227 int ret; 228 229 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); 230 231 rctx->cryptlen = areq->cryptlen - aead->authsize; 232 rctx->assoclen = areq->assoclen; 233 rctx->srclen = areq->cryptlen + areq->assoclen; 234 rctx->dstlen = rctx->srclen - aead->authsize; 235 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; 236 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; 237 rctx->flags = areq->base.flags; 238 rctx->ctx_handle = nctx->u.ctx_handle; 239 rctx->src = areq->src; 240 rctx->dst = areq->dst; 241 rctx->ctrl_arg = DECRYPT; 242 ret = nitrox_set_creq(rctx); 243 if (ret) 244 return ret; 245 246 /* send the crypto request */ 247 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, 248 areq); 249 } 250 251 static int nitrox_aead_init(struct crypto_aead *aead) 252 { 253 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 254 struct crypto_ctx_hdr *chdr; 255 256 /* get the first device */ 257 nctx->ndev = nitrox_get_first_device(); 258 if (!nctx->ndev) 259 return -ENODEV; 260 261 /* allocate nitrox crypto context */ 262 chdr = crypto_alloc_context(nctx->ndev); 263 if (!chdr) { 264 nitrox_put_device(nctx->ndev); 265 return -ENOMEM; 266 } 267 nctx->chdr = chdr; 268 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + 269 sizeof(struct ctx_hdr)); 270 nctx->u.fctx->flags.f = 0; 271 272 return 0; 273 } 274 275 static int nitrox_gcm_common_init(struct crypto_aead *aead) 276 { 277 int ret; 278 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 279 union fc_ctx_flags *flags; 280 281 ret = nitrox_aead_init(aead); 282 if (ret) 283 return ret; 284 285 flags = &nctx->u.fctx->flags; 286 flags->w0.cipher_type = CIPHER_AES_GCM; 287 flags->w0.hash_type = AUTH_NULL; 288 flags->w0.iv_source = IV_FROM_DPTR; 289 /* ask microcode to calculate ipad/opad */ 290 flags->w0.auth_input_type = 1; 291 flags->f = be64_to_cpu(flags->f); 292 293 return 0; 294 } 295 296 static int nitrox_aes_gcm_init(struct crypto_aead *aead) 297 { 298 int ret; 299 300 ret = nitrox_gcm_common_init(aead); 301 if (ret) 302 return ret; 303 304 crypto_aead_set_reqsize(aead, 305 sizeof(struct aead_request) + 306 sizeof(struct nitrox_aead_rctx)); 307 308 return 0; 309 } 310 311 static void nitrox_aead_exit(struct crypto_aead *aead) 312 { 313 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 314 315 /* free the nitrox crypto context */ 316 if (nctx->u.ctx_handle) { 317 struct flexi_crypto_context *fctx = nctx->u.fctx; 318 319 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); 320 memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); 321 crypto_free_context((void *)nctx->chdr); 322 } 323 nitrox_put_device(nctx->ndev); 324 325 nctx->u.ctx_handle = 0; 326 nctx->ndev = NULL; 327 } 328 329 static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key, 330 unsigned int keylen) 331 { 332 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 333 struct flexi_crypto_context *fctx = nctx->u.fctx; 334 int ret; 335 336 if (keylen < GCM_AES_SALT_SIZE) 337 return -EINVAL; 338 339 keylen -= GCM_AES_SALT_SIZE; 340 ret = nitrox_aes_gcm_setkey(aead, key, keylen); 341 if (ret) 342 return ret; 343 344 memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE); 345 return 0; 346 } 347 348 static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead, 349 unsigned int authsize) 350 { 351 switch (authsize) { 352 case 8: 353 case 12: 354 case 16: 355 break; 356 default: 357 return -EINVAL; 358 } 359 360 return nitrox_aead_setauthsize(aead, authsize); 361 } 362 363 static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq) 364 { 365 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); 366 struct nitrox_aead_rctx *aead_rctx = &rctx->base; 367 unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; 368 struct scatterlist *sg; 369 370 if (areq->assoclen != 16 && areq->assoclen != 20) 371 return -EINVAL; 372 373 scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0); 374 sg_init_table(rctx->src, 3); 375 sg_set_buf(rctx->src, rctx->assoc, assoclen); 376 sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen); 377 if (sg != rctx->src + 1) 378 sg_chain(rctx->src, 2, sg); 379 380 if (areq->src != areq->dst) { 381 sg_init_table(rctx->dst, 3); 382 sg_set_buf(rctx->dst, rctx->assoc, assoclen); 383 sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen); 384 if (sg != rctx->dst + 1) 385 sg_chain(rctx->dst, 2, sg); 386 } 387 388 aead_rctx->src = rctx->src; 389 aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst; 390 391 return 0; 392 } 393 394 static void nitrox_rfc4106_callback(void *arg, int err) 395 { 396 struct aead_request *areq = arg; 397 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); 398 struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq; 399 400 free_src_sglist(nkreq); 401 free_dst_sglist(nkreq); 402 if (err) { 403 pr_err_ratelimited("request failed status 0x%0x\n", err); 404 err = -EINVAL; 405 } 406 407 areq->base.complete(&areq->base, err); 408 } 409 410 static int nitrox_rfc4106_enc(struct aead_request *areq) 411 { 412 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 413 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 414 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); 415 struct nitrox_aead_rctx *aead_rctx = &rctx->base; 416 struct se_crypto_request *creq = &aead_rctx->nkreq.creq; 417 int ret; 418 419 aead_rctx->cryptlen = areq->cryptlen; 420 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; 421 aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen; 422 aead_rctx->dstlen = aead_rctx->srclen + aead->authsize; 423 aead_rctx->iv = areq->iv; 424 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; 425 aead_rctx->flags = areq->base.flags; 426 aead_rctx->ctx_handle = nctx->u.ctx_handle; 427 aead_rctx->ctrl_arg = ENCRYPT; 428 429 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); 430 if (ret) 431 return ret; 432 433 ret = nitrox_set_creq(aead_rctx); 434 if (ret) 435 return ret; 436 437 /* send the crypto request */ 438 return nitrox_process_se_request(nctx->ndev, creq, 439 nitrox_rfc4106_callback, areq); 440 } 441 442 static int nitrox_rfc4106_dec(struct aead_request *areq) 443 { 444 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 445 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 446 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); 447 struct nitrox_aead_rctx *aead_rctx = &rctx->base; 448 struct se_crypto_request *creq = &aead_rctx->nkreq.creq; 449 int ret; 450 451 aead_rctx->cryptlen = areq->cryptlen - aead->authsize; 452 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; 453 aead_rctx->srclen = 454 areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen; 455 aead_rctx->dstlen = aead_rctx->srclen - aead->authsize; 456 aead_rctx->iv = areq->iv; 457 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; 458 aead_rctx->flags = areq->base.flags; 459 aead_rctx->ctx_handle = nctx->u.ctx_handle; 460 aead_rctx->ctrl_arg = DECRYPT; 461 462 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); 463 if (ret) 464 return ret; 465 466 ret = nitrox_set_creq(aead_rctx); 467 if (ret) 468 return ret; 469 470 /* send the crypto request */ 471 return nitrox_process_se_request(nctx->ndev, creq, 472 nitrox_rfc4106_callback, areq); 473 } 474 475 static int nitrox_rfc4106_init(struct crypto_aead *aead) 476 { 477 int ret; 478 479 ret = nitrox_gcm_common_init(aead); 480 if (ret) 481 return ret; 482 483 crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + 484 sizeof(struct nitrox_rfc4106_rctx)); 485 486 return 0; 487 } 488 489 static struct aead_alg nitrox_aeads[] = { { 490 .base = { 491 .cra_name = "gcm(aes)", 492 .cra_driver_name = "n5_aes_gcm", 493 .cra_priority = PRIO, 494 .cra_flags = CRYPTO_ALG_ASYNC, 495 .cra_blocksize = AES_BLOCK_SIZE, 496 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 497 .cra_alignmask = 0, 498 .cra_module = THIS_MODULE, 499 }, 500 .setkey = nitrox_aes_gcm_setkey, 501 .setauthsize = nitrox_aead_setauthsize, 502 .encrypt = nitrox_aes_gcm_enc, 503 .decrypt = nitrox_aes_gcm_dec, 504 .init = nitrox_aes_gcm_init, 505 .exit = nitrox_aead_exit, 506 .ivsize = GCM_AES_IV_SIZE, 507 .maxauthsize = AES_BLOCK_SIZE, 508 }, { 509 .base = { 510 .cra_name = "rfc4106(gcm(aes))", 511 .cra_driver_name = "n5_rfc4106", 512 .cra_priority = PRIO, 513 .cra_flags = CRYPTO_ALG_ASYNC, 514 .cra_blocksize = AES_BLOCK_SIZE, 515 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 516 .cra_alignmask = 0, 517 .cra_module = THIS_MODULE, 518 }, 519 .setkey = nitrox_rfc4106_setkey, 520 .setauthsize = nitrox_rfc4106_setauthsize, 521 .encrypt = nitrox_rfc4106_enc, 522 .decrypt = nitrox_rfc4106_dec, 523 .init = nitrox_rfc4106_init, 524 .exit = nitrox_aead_exit, 525 .ivsize = GCM_RFC4106_IV_SIZE, 526 .maxauthsize = AES_BLOCK_SIZE, 527 } }; 528 529 int nitrox_register_aeads(void) 530 { 531 return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); 532 } 533 534 void nitrox_unregister_aeads(void) 535 { 536 crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); 537 } 538