1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AES CCM routines supporting the Power 7+ Nest Accelerators driver 4 * 5 * Copyright (C) 2012 International Business Machines Inc. 6 * 7 * Author: Kent Yoder <yoder1@us.ibm.com> 8 */ 9 10 #include <crypto/internal/aead.h> 11 #include <crypto/aes.h> 12 #include <crypto/algapi.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/module.h> 15 #include <linux/types.h> 16 #include <linux/crypto.h> 17 #include <asm/vio.h> 18 19 #include "nx_csbcpb.h" 20 #include "nx.h" 21 22 23 static int ccm_aes_nx_set_key(struct crypto_aead *tfm, 24 const u8 *in_key, 25 unsigned int key_len) 26 { 27 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); 28 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 29 struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; 30 31 nx_ctx_init(nx_ctx, HCOP_FC_AES); 32 33 switch (key_len) { 34 case AES_KEYSIZE_128: 35 NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128); 36 NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128); 37 nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128]; 38 break; 39 default: 40 return -EINVAL; 41 } 42 43 csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM; 44 memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len); 45 46 csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA; 47 memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len); 48 49 return 0; 50 51 } 52 53 static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm, 54 const u8 *in_key, 55 unsigned int key_len) 56 { 57 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); 58 59 if (key_len < 3) 60 return -EINVAL; 61 62 key_len -= 3; 63 64 memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3); 65 66 return ccm_aes_nx_set_key(tfm, in_key, key_len); 67 } 68 69 static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm, 70 unsigned int authsize) 71 { 72 switch (authsize) { 73 case 4: 74 case 6: 75 case 8: 76 case 10: 77 case 12: 78 case 14: 79 case 16: 80 break; 81 default: 82 return -EINVAL; 83 } 84 85 return 0; 86 } 87 88 static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm, 89 unsigned int authsize) 90 { 91 switch (authsize) { 92 case 8: 93 case 12: 94 case 16: 95 break; 96 default: 97 return -EINVAL; 98 } 99 100 return 0; 101 } 102 103 /* taken from crypto/ccm.c */ 104 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 105 { 106 __be32 data; 107 108 memset(block, 0, csize); 109 block += csize; 110 111 if (csize >= 4) 112 csize = 4; 113 else if (msglen > (unsigned int)(1 << (8 * csize))) 114 return -EOVERFLOW; 115 116 data = cpu_to_be32(msglen); 117 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 118 119 return 0; 120 } 121 122 /* taken from crypto/ccm.c */ 123 static inline int crypto_ccm_check_iv(const u8 *iv) 124 { 125 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 126 if (1 > iv[0] || iv[0] > 7) 127 return -EINVAL; 128 129 return 0; 130 } 131 132 /* based on code from crypto/ccm.c */ 133 static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, 134 unsigned int cryptlen, u8 *b0) 135 { 136 unsigned int l, lp, m = authsize; 137 int rc; 138 139 memcpy(b0, iv, 16); 140 141 lp = b0[0]; 142 l = lp + 1; 143 144 /* set m, bits 3-5 */ 145 *b0 |= (8 * ((m - 2) / 2)); 146 147 /* set adata, bit 6, if associated data is used */ 148 if (assoclen) 149 *b0 |= 64; 150 151 rc = set_msg_len(b0 + 16 - l, cryptlen, l); 152 153 return rc; 154 } 155 156 static int generate_pat(u8 *iv, 157 struct aead_request *req, 158 struct nx_crypto_ctx *nx_ctx, 159 unsigned int authsize, 160 unsigned int nbytes, 161 unsigned int assoclen, 162 u8 *out) 163 { 164 struct nx_sg *nx_insg = nx_ctx->in_sg; 165 struct nx_sg *nx_outsg = nx_ctx->out_sg; 166 unsigned int iauth_len = 0; 167 u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL; 168 int rc; 169 unsigned int max_sg_len; 170 171 /* zero the ctr value */ 172 memset(iv + 15 - iv[0], 0, iv[0] + 1); 173 174 /* page 78 of nx_wb.pdf has, 175 * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes 176 * in length. If a full message is used, the AES CCA implementation 177 * restricts the maximum AAD length to 2^32 -1 bytes. 178 * If partial messages are used, the implementation supports 179 * 2^64 -1 bytes maximum AAD length. 180 * 181 * However, in the cryptoapi's aead_request structure, 182 * assoclen is an unsigned int, thus it cannot hold a length 183 * value greater than 2^32 - 1. 184 * Thus the AAD is further constrained by this and is never 185 * greater than 2^32. 186 */ 187 188 if (!assoclen) { 189 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; 190 } else if (assoclen <= 14) { 191 /* if associated data is 14 bytes or less, we do 1 GCM 192 * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1, 193 * which is fed in through the source buffers here */ 194 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; 195 b1 = nx_ctx->priv.ccm.iauth_tag; 196 iauth_len = assoclen; 197 } else if (assoclen <= 65280) { 198 /* if associated data is less than (2^16 - 2^8), we construct 199 * B1 differently and feed in the associated data to a CCA 200 * operation */ 201 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0; 202 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1; 203 iauth_len = 14; 204 } else { 205 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0; 206 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1; 207 iauth_len = 10; 208 } 209 210 /* generate B0 */ 211 rc = generate_b0(iv, assoclen, authsize, nbytes, b0); 212 if (rc) 213 return rc; 214 215 /* generate B1: 216 * add control info for associated data 217 * RFC 3610 and NIST Special Publication 800-38C 218 */ 219 if (b1) { 220 memset(b1, 0, 16); 221 if (assoclen <= 65280) { 222 *(u16 *)b1 = assoclen; 223 scatterwalk_map_and_copy(b1 + 2, req->src, 0, 224 iauth_len, SCATTERWALK_FROM_SG); 225 } else { 226 *(u16 *)b1 = (u16)(0xfffe); 227 *(u32 *)&b1[2] = assoclen; 228 scatterwalk_map_and_copy(b1 + 6, req->src, 0, 229 iauth_len, SCATTERWALK_FROM_SG); 230 } 231 } 232 233 /* now copy any remaining AAD to scatterlist and call nx... */ 234 if (!assoclen) { 235 return rc; 236 } else if (assoclen <= 14) { 237 unsigned int len = 16; 238 239 nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen); 240 241 if (len != 16) 242 return -EINVAL; 243 244 nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len, 245 nx_ctx->ap->sglen); 246 247 if (len != 16) 248 return -EINVAL; 249 250 /* inlen should be negative, indicating to phyp that its a 251 * pointer to an sg list */ 252 nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) * 253 sizeof(struct nx_sg); 254 nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) * 255 sizeof(struct nx_sg); 256 257 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT; 258 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE; 259 260 result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac; 261 262 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 263 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 264 if (rc) 265 return rc; 266 267 atomic_inc(&(nx_ctx->stats->aes_ops)); 268 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); 269 270 } else { 271 unsigned int processed = 0, to_process; 272 273 processed += iauth_len; 274 275 /* page_limit: number of sg entries that fit on one page */ 276 max_sg_len = min_t(u64, nx_ctx->ap->sglen, 277 nx_driver.of.max_sg_len/sizeof(struct nx_sg)); 278 max_sg_len = min_t(u64, max_sg_len, 279 nx_ctx->ap->databytelen/NX_PAGE_SIZE); 280 281 do { 282 to_process = min_t(u32, assoclen - processed, 283 nx_ctx->ap->databytelen); 284 285 nx_insg = nx_walk_and_build(nx_ctx->in_sg, 286 nx_ctx->ap->sglen, 287 req->src, processed, 288 &to_process); 289 290 if ((to_process + processed) < assoclen) { 291 NX_CPB_FDM(nx_ctx->csbcpb_aead) |= 292 NX_FDM_INTERMEDIATE; 293 } else { 294 NX_CPB_FDM(nx_ctx->csbcpb_aead) &= 295 ~NX_FDM_INTERMEDIATE; 296 } 297 298 299 nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) * 300 sizeof(struct nx_sg); 301 302 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; 303 304 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead, 305 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 306 if (rc) 307 return rc; 308 309 memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0, 310 nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0, 311 AES_BLOCK_SIZE); 312 313 NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION; 314 315 atomic_inc(&(nx_ctx->stats->aes_ops)); 316 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); 317 318 processed += to_process; 319 } while (processed < assoclen); 320 321 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; 322 } 323 324 memcpy(out, result, AES_BLOCK_SIZE); 325 326 return rc; 327 } 328 329 static int ccm_nx_decrypt(struct aead_request *req, 330 u8 *iv, 331 unsigned int assoclen) 332 { 333 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 334 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 335 unsigned int nbytes = req->cryptlen; 336 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 337 struct nx_ccm_priv *priv = &nx_ctx->priv.ccm; 338 unsigned long irq_flags; 339 unsigned int processed = 0, to_process; 340 int rc = -1; 341 342 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 343 344 nbytes -= authsize; 345 346 /* copy out the auth tag to compare with later */ 347 scatterwalk_map_and_copy(priv->oauth_tag, 348 req->src, nbytes + req->assoclen, authsize, 349 SCATTERWALK_FROM_SG); 350 351 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, 352 csbcpb->cpb.aes_ccm.in_pat_or_b0); 353 if (rc) 354 goto out; 355 356 do { 357 358 /* to_process: the AES_BLOCK_SIZE data chunk to process in this 359 * update. This value is bound by sg list limits. 360 */ 361 to_process = nbytes - processed; 362 363 if ((to_process + processed) < nbytes) 364 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; 365 else 366 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 367 368 NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; 369 370 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, 371 &to_process, processed + req->assoclen, 372 csbcpb->cpb.aes_ccm.iv_or_ctr); 373 if (rc) 374 goto out; 375 376 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 377 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 378 if (rc) 379 goto out; 380 381 /* for partial completion, copy following for next 382 * entry into loop... 383 */ 384 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE); 385 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0, 386 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE); 387 memcpy(csbcpb->cpb.aes_ccm.in_s0, 388 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE); 389 390 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 391 392 /* update stats */ 393 atomic_inc(&(nx_ctx->stats->aes_ops)); 394 atomic64_add(csbcpb->csb.processed_byte_count, 395 &(nx_ctx->stats->aes_bytes)); 396 397 processed += to_process; 398 } while (processed < nbytes); 399 400 rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag, 401 authsize) ? -EBADMSG : 0; 402 out: 403 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 404 return rc; 405 } 406 407 static int ccm_nx_encrypt(struct aead_request *req, 408 u8 *iv, 409 unsigned int assoclen) 410 { 411 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 412 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 413 unsigned int nbytes = req->cryptlen; 414 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 415 unsigned long irq_flags; 416 unsigned int processed = 0, to_process; 417 int rc = -1; 418 419 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 420 421 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen, 422 csbcpb->cpb.aes_ccm.in_pat_or_b0); 423 if (rc) 424 goto out; 425 426 do { 427 /* to process: the AES_BLOCK_SIZE data chunk to process in this 428 * update. This value is bound by sg list limits. 429 */ 430 to_process = nbytes - processed; 431 432 if ((to_process + processed) < nbytes) 433 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; 434 else 435 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 436 437 NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; 438 439 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src, 440 &to_process, processed + req->assoclen, 441 csbcpb->cpb.aes_ccm.iv_or_ctr); 442 if (rc) 443 goto out; 444 445 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 446 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 447 if (rc) 448 goto out; 449 450 /* for partial completion, copy following for next 451 * entry into loop... 452 */ 453 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE); 454 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0, 455 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE); 456 memcpy(csbcpb->cpb.aes_ccm.in_s0, 457 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE); 458 459 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 460 461 /* update stats */ 462 atomic_inc(&(nx_ctx->stats->aes_ops)); 463 atomic64_add(csbcpb->csb.processed_byte_count, 464 &(nx_ctx->stats->aes_bytes)); 465 466 processed += to_process; 467 468 } while (processed < nbytes); 469 470 /* copy out the auth tag */ 471 scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac, 472 req->dst, nbytes + req->assoclen, authsize, 473 SCATTERWALK_TO_SG); 474 475 out: 476 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 477 return rc; 478 } 479 480 static int ccm4309_aes_nx_encrypt(struct aead_request *req) 481 { 482 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 483 struct nx_gcm_rctx *rctx = aead_request_ctx(req); 484 u8 *iv = rctx->iv; 485 486 iv[0] = 3; 487 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3); 488 memcpy(iv + 4, req->iv, 8); 489 490 return ccm_nx_encrypt(req, iv, req->assoclen - 8); 491 } 492 493 static int ccm_aes_nx_encrypt(struct aead_request *req) 494 { 495 int rc; 496 497 rc = crypto_ccm_check_iv(req->iv); 498 if (rc) 499 return rc; 500 501 return ccm_nx_encrypt(req, req->iv, req->assoclen); 502 } 503 504 static int ccm4309_aes_nx_decrypt(struct aead_request *req) 505 { 506 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); 507 struct nx_gcm_rctx *rctx = aead_request_ctx(req); 508 u8 *iv = rctx->iv; 509 510 iv[0] = 3; 511 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3); 512 memcpy(iv + 4, req->iv, 8); 513 514 return ccm_nx_decrypt(req, iv, req->assoclen - 8); 515 } 516 517 static int ccm_aes_nx_decrypt(struct aead_request *req) 518 { 519 int rc; 520 521 rc = crypto_ccm_check_iv(req->iv); 522 if (rc) 523 return rc; 524 525 return ccm_nx_decrypt(req, req->iv, req->assoclen); 526 } 527 528 struct aead_alg nx_ccm_aes_alg = { 529 .base = { 530 .cra_name = "ccm(aes)", 531 .cra_driver_name = "ccm-aes-nx", 532 .cra_priority = 300, 533 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 534 .cra_blocksize = 1, 535 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 536 .cra_module = THIS_MODULE, 537 }, 538 .init = nx_crypto_ctx_aes_ccm_init, 539 .exit = nx_crypto_ctx_aead_exit, 540 .ivsize = AES_BLOCK_SIZE, 541 .maxauthsize = AES_BLOCK_SIZE, 542 .setkey = ccm_aes_nx_set_key, 543 .setauthsize = ccm_aes_nx_setauthsize, 544 .encrypt = ccm_aes_nx_encrypt, 545 .decrypt = ccm_aes_nx_decrypt, 546 }; 547 548 struct aead_alg nx_ccm4309_aes_alg = { 549 .base = { 550 .cra_name = "rfc4309(ccm(aes))", 551 .cra_driver_name = "rfc4309-ccm-aes-nx", 552 .cra_priority = 300, 553 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 554 .cra_blocksize = 1, 555 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 556 .cra_module = THIS_MODULE, 557 }, 558 .init = nx_crypto_ctx_aes_ccm_init, 559 .exit = nx_crypto_ctx_aead_exit, 560 .ivsize = 8, 561 .maxauthsize = AES_BLOCK_SIZE, 562 .setkey = ccm4309_aes_nx_set_key, 563 .setauthsize = ccm4309_aes_nx_setauthsize, 564 .encrypt = ccm4309_aes_nx_encrypt, 565 .decrypt = ccm4309_aes_nx_decrypt, 566 }; 567