1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * s390 implementation of the AES Cipher Algorithm with protected keys. 6 * 7 * s390 Version: 8 * Copyright IBM Corp. 2017,2019 9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 10 * Harald Freudenberger <freude@de.ibm.com> 11 */ 12 13 #define KMSG_COMPONENT "paes_s390" 14 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 15 16 #include <crypto/aes.h> 17 #include <crypto/algapi.h> 18 #include <linux/bug.h> 19 #include <linux/err.h> 20 #include <linux/module.h> 21 #include <linux/cpufeature.h> 22 #include <linux/init.h> 23 #include <linux/spinlock.h> 24 #include <crypto/xts.h> 25 #include <asm/cpacf.h> 26 #include <asm/pkey.h> 27 28 /* 29 * Key blobs smaller/bigger than these defines are rejected 30 * by the common code even before the individual setkey function 31 * is called. As paes can handle different kinds of key blobs 32 * and padding is also possible, the limits need to be generous. 33 */ 34 #define PAES_MIN_KEYSIZE 64 35 #define PAES_MAX_KEYSIZE 256 36 37 static u8 *ctrblk; 38 static DEFINE_SPINLOCK(ctrblk_lock); 39 40 static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; 41 42 struct key_blob { 43 /* 44 * Small keys will be stored in the keybuf. Larger keys are 45 * stored in extra allocated memory. In both cases does 46 * key point to the memory where the key is stored. 47 * The code distinguishes by checking keylen against 48 * sizeof(keybuf). See the two following helper functions. 49 */ 50 u8 *key; 51 u8 keybuf[128]; 52 unsigned int keylen; 53 }; 54 55 static inline int _copy_key_to_kb(struct key_blob *kb, 56 const u8 *key, 57 unsigned int keylen) 58 { 59 if (keylen <= sizeof(kb->keybuf)) 60 kb->key = kb->keybuf; 61 else { 62 kb->key = kmalloc(keylen, GFP_KERNEL); 63 if (!kb->key) 64 return -ENOMEM; 65 } 66 memcpy(kb->key, key, keylen); 67 kb->keylen = keylen; 68 69 return 0; 70 } 71 72 static inline void _free_kb_keybuf(struct key_blob *kb) 73 { 74 if (kb->key && kb->key != kb->keybuf 75 && kb->keylen > sizeof(kb->keybuf)) { 76 kfree(kb->key); 77 kb->key = NULL; 78 } 79 } 80 81 struct s390_paes_ctx { 82 struct key_blob kb; 83 struct pkey_protkey pk; 84 unsigned long fc; 85 }; 86 87 struct s390_pxts_ctx { 88 struct key_blob kb[2]; 89 struct pkey_protkey pk[2]; 90 unsigned long fc; 91 }; 92 93 static inline int __paes_convert_key(struct key_blob *kb, 94 struct pkey_protkey *pk) 95 { 96 int i, ret; 97 98 /* try three times in case of failure */ 99 for (i = 0; i < 3; i++) { 100 ret = pkey_keyblob2pkey(kb->key, kb->keylen, pk); 101 if (ret == 0) 102 break; 103 } 104 105 return ret; 106 } 107 108 static int __paes_set_key(struct s390_paes_ctx *ctx) 109 { 110 unsigned long fc; 111 112 if (__paes_convert_key(&ctx->kb, &ctx->pk)) 113 return -EINVAL; 114 115 /* Pick the correct function code based on the protected key type */ 116 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 : 117 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 : 118 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0; 119 120 /* Check if the function code is available */ 121 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 122 123 return ctx->fc ? 0 : -EINVAL; 124 } 125 126 static int ecb_paes_init(struct crypto_tfm *tfm) 127 { 128 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 129 130 ctx->kb.key = NULL; 131 132 return 0; 133 } 134 135 static void ecb_paes_exit(struct crypto_tfm *tfm) 136 { 137 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 138 139 _free_kb_keybuf(&ctx->kb); 140 } 141 142 static int ecb_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 143 unsigned int key_len) 144 { 145 int rc; 146 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 147 148 _free_kb_keybuf(&ctx->kb); 149 rc = _copy_key_to_kb(&ctx->kb, in_key, key_len); 150 if (rc) 151 return rc; 152 153 if (__paes_set_key(ctx)) { 154 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 155 return -EINVAL; 156 } 157 return 0; 158 } 159 160 static int ecb_paes_crypt(struct blkcipher_desc *desc, 161 unsigned long modifier, 162 struct blkcipher_walk *walk) 163 { 164 struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 165 unsigned int nbytes, n, k; 166 int ret; 167 168 ret = blkcipher_walk_virt(desc, walk); 169 while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 170 /* only use complete blocks */ 171 n = nbytes & ~(AES_BLOCK_SIZE - 1); 172 k = cpacf_km(ctx->fc | modifier, ctx->pk.protkey, 173 walk->dst.virt.addr, walk->src.virt.addr, n); 174 if (k) 175 ret = blkcipher_walk_done(desc, walk, nbytes - k); 176 if (k < n) { 177 if (__paes_set_key(ctx) != 0) 178 return blkcipher_walk_done(desc, walk, -EIO); 179 } 180 } 181 return ret; 182 } 183 184 static int ecb_paes_encrypt(struct blkcipher_desc *desc, 185 struct scatterlist *dst, struct scatterlist *src, 186 unsigned int nbytes) 187 { 188 struct blkcipher_walk walk; 189 190 blkcipher_walk_init(&walk, dst, src, nbytes); 191 return ecb_paes_crypt(desc, CPACF_ENCRYPT, &walk); 192 } 193 194 static int ecb_paes_decrypt(struct blkcipher_desc *desc, 195 struct scatterlist *dst, struct scatterlist *src, 196 unsigned int nbytes) 197 { 198 struct blkcipher_walk walk; 199 200 blkcipher_walk_init(&walk, dst, src, nbytes); 201 return ecb_paes_crypt(desc, CPACF_DECRYPT, &walk); 202 } 203 204 static struct crypto_alg ecb_paes_alg = { 205 .cra_name = "ecb(paes)", 206 .cra_driver_name = "ecb-paes-s390", 207 .cra_priority = 401, /* combo: aes + ecb + 1 */ 208 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 209 .cra_blocksize = AES_BLOCK_SIZE, 210 .cra_ctxsize = sizeof(struct s390_paes_ctx), 211 .cra_type = &crypto_blkcipher_type, 212 .cra_module = THIS_MODULE, 213 .cra_list = LIST_HEAD_INIT(ecb_paes_alg.cra_list), 214 .cra_init = ecb_paes_init, 215 .cra_exit = ecb_paes_exit, 216 .cra_u = { 217 .blkcipher = { 218 .min_keysize = PAES_MIN_KEYSIZE, 219 .max_keysize = PAES_MAX_KEYSIZE, 220 .setkey = ecb_paes_set_key, 221 .encrypt = ecb_paes_encrypt, 222 .decrypt = ecb_paes_decrypt, 223 } 224 } 225 }; 226 227 static int cbc_paes_init(struct crypto_tfm *tfm) 228 { 229 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 230 231 ctx->kb.key = NULL; 232 233 return 0; 234 } 235 236 static void cbc_paes_exit(struct crypto_tfm *tfm) 237 { 238 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 239 240 _free_kb_keybuf(&ctx->kb); 241 } 242 243 static int __cbc_paes_set_key(struct s390_paes_ctx *ctx) 244 { 245 unsigned long fc; 246 247 if (__paes_convert_key(&ctx->kb, &ctx->pk)) 248 return -EINVAL; 249 250 /* Pick the correct function code based on the protected key type */ 251 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 : 252 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 : 253 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0; 254 255 /* Check if the function code is available */ 256 ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0; 257 258 return ctx->fc ? 0 : -EINVAL; 259 } 260 261 static int cbc_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 262 unsigned int key_len) 263 { 264 int rc; 265 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 266 267 _free_kb_keybuf(&ctx->kb); 268 rc = _copy_key_to_kb(&ctx->kb, in_key, key_len); 269 if (rc) 270 return rc; 271 272 if (__cbc_paes_set_key(ctx)) { 273 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 274 return -EINVAL; 275 } 276 return 0; 277 } 278 279 static int cbc_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, 280 struct blkcipher_walk *walk) 281 { 282 struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 283 unsigned int nbytes, n, k; 284 int ret; 285 struct { 286 u8 iv[AES_BLOCK_SIZE]; 287 u8 key[MAXPROTKEYSIZE]; 288 } param; 289 290 ret = blkcipher_walk_virt(desc, walk); 291 memcpy(param.iv, walk->iv, AES_BLOCK_SIZE); 292 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 293 while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 294 /* only use complete blocks */ 295 n = nbytes & ~(AES_BLOCK_SIZE - 1); 296 k = cpacf_kmc(ctx->fc | modifier, ¶m, 297 walk->dst.virt.addr, walk->src.virt.addr, n); 298 if (k) 299 ret = blkcipher_walk_done(desc, walk, nbytes - k); 300 if (k < n) { 301 if (__cbc_paes_set_key(ctx) != 0) 302 return blkcipher_walk_done(desc, walk, -EIO); 303 memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 304 } 305 } 306 memcpy(walk->iv, param.iv, AES_BLOCK_SIZE); 307 return ret; 308 } 309 310 static int cbc_paes_encrypt(struct blkcipher_desc *desc, 311 struct scatterlist *dst, struct scatterlist *src, 312 unsigned int nbytes) 313 { 314 struct blkcipher_walk walk; 315 316 blkcipher_walk_init(&walk, dst, src, nbytes); 317 return cbc_paes_crypt(desc, 0, &walk); 318 } 319 320 static int cbc_paes_decrypt(struct blkcipher_desc *desc, 321 struct scatterlist *dst, struct scatterlist *src, 322 unsigned int nbytes) 323 { 324 struct blkcipher_walk walk; 325 326 blkcipher_walk_init(&walk, dst, src, nbytes); 327 return cbc_paes_crypt(desc, CPACF_DECRYPT, &walk); 328 } 329 330 static struct crypto_alg cbc_paes_alg = { 331 .cra_name = "cbc(paes)", 332 .cra_driver_name = "cbc-paes-s390", 333 .cra_priority = 402, /* ecb-paes-s390 + 1 */ 334 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 335 .cra_blocksize = AES_BLOCK_SIZE, 336 .cra_ctxsize = sizeof(struct s390_paes_ctx), 337 .cra_type = &crypto_blkcipher_type, 338 .cra_module = THIS_MODULE, 339 .cra_list = LIST_HEAD_INIT(cbc_paes_alg.cra_list), 340 .cra_init = cbc_paes_init, 341 .cra_exit = cbc_paes_exit, 342 .cra_u = { 343 .blkcipher = { 344 .min_keysize = PAES_MIN_KEYSIZE, 345 .max_keysize = PAES_MAX_KEYSIZE, 346 .ivsize = AES_BLOCK_SIZE, 347 .setkey = cbc_paes_set_key, 348 .encrypt = cbc_paes_encrypt, 349 .decrypt = cbc_paes_decrypt, 350 } 351 } 352 }; 353 354 static int xts_paes_init(struct crypto_tfm *tfm) 355 { 356 struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm); 357 358 ctx->kb[0].key = NULL; 359 ctx->kb[1].key = NULL; 360 361 return 0; 362 } 363 364 static void xts_paes_exit(struct crypto_tfm *tfm) 365 { 366 struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm); 367 368 _free_kb_keybuf(&ctx->kb[0]); 369 _free_kb_keybuf(&ctx->kb[1]); 370 } 371 372 static int __xts_paes_set_key(struct s390_pxts_ctx *ctx) 373 { 374 unsigned long fc; 375 376 if (__paes_convert_key(&ctx->kb[0], &ctx->pk[0]) || 377 __paes_convert_key(&ctx->kb[1], &ctx->pk[1])) 378 return -EINVAL; 379 380 if (ctx->pk[0].type != ctx->pk[1].type) 381 return -EINVAL; 382 383 /* Pick the correct function code based on the protected key type */ 384 fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 : 385 (ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ? 386 CPACF_KM_PXTS_256 : 0; 387 388 /* Check if the function code is available */ 389 ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 390 391 return ctx->fc ? 0 : -EINVAL; 392 } 393 394 static int xts_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 395 unsigned int xts_key_len) 396 { 397 int rc; 398 struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm); 399 u8 ckey[2 * AES_MAX_KEY_SIZE]; 400 unsigned int ckey_len, key_len; 401 402 if (xts_key_len % 2) 403 return -EINVAL; 404 405 key_len = xts_key_len / 2; 406 407 _free_kb_keybuf(&ctx->kb[0]); 408 _free_kb_keybuf(&ctx->kb[1]); 409 rc = _copy_key_to_kb(&ctx->kb[0], in_key, key_len); 410 if (rc) 411 return rc; 412 rc = _copy_key_to_kb(&ctx->kb[1], in_key + key_len, key_len); 413 if (rc) 414 return rc; 415 416 if (__xts_paes_set_key(ctx)) { 417 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 418 return -EINVAL; 419 } 420 421 /* 422 * xts_check_key verifies the key length is not odd and makes 423 * sure that the two keys are not the same. This can be done 424 * on the two protected keys as well 425 */ 426 ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 427 AES_KEYSIZE_128 : AES_KEYSIZE_256; 428 memcpy(ckey, ctx->pk[0].protkey, ckey_len); 429 memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len); 430 return xts_check_key(tfm, ckey, 2*ckey_len); 431 } 432 433 static int xts_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, 434 struct blkcipher_walk *walk) 435 { 436 struct s390_pxts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 437 unsigned int keylen, offset, nbytes, n, k; 438 int ret; 439 struct { 440 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 441 u8 tweak[16]; 442 u8 block[16]; 443 u8 bit[16]; 444 u8 xts[16]; 445 } pcc_param; 446 struct { 447 u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 448 u8 init[16]; 449 } xts_param; 450 451 ret = blkcipher_walk_virt(desc, walk); 452 keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64; 453 offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0; 454 retry: 455 memset(&pcc_param, 0, sizeof(pcc_param)); 456 memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak)); 457 memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen); 458 cpacf_pcc(ctx->fc, pcc_param.key + offset); 459 460 memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen); 461 memcpy(xts_param.init, pcc_param.xts, 16); 462 463 while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 464 /* only use complete blocks */ 465 n = nbytes & ~(AES_BLOCK_SIZE - 1); 466 k = cpacf_km(ctx->fc | modifier, xts_param.key + offset, 467 walk->dst.virt.addr, walk->src.virt.addr, n); 468 if (k) 469 ret = blkcipher_walk_done(desc, walk, nbytes - k); 470 if (k < n) { 471 if (__xts_paes_set_key(ctx) != 0) 472 return blkcipher_walk_done(desc, walk, -EIO); 473 goto retry; 474 } 475 } 476 return ret; 477 } 478 479 static int xts_paes_encrypt(struct blkcipher_desc *desc, 480 struct scatterlist *dst, struct scatterlist *src, 481 unsigned int nbytes) 482 { 483 struct blkcipher_walk walk; 484 485 blkcipher_walk_init(&walk, dst, src, nbytes); 486 return xts_paes_crypt(desc, 0, &walk); 487 } 488 489 static int xts_paes_decrypt(struct blkcipher_desc *desc, 490 struct scatterlist *dst, struct scatterlist *src, 491 unsigned int nbytes) 492 { 493 struct blkcipher_walk walk; 494 495 blkcipher_walk_init(&walk, dst, src, nbytes); 496 return xts_paes_crypt(desc, CPACF_DECRYPT, &walk); 497 } 498 499 static struct crypto_alg xts_paes_alg = { 500 .cra_name = "xts(paes)", 501 .cra_driver_name = "xts-paes-s390", 502 .cra_priority = 402, /* ecb-paes-s390 + 1 */ 503 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 504 .cra_blocksize = AES_BLOCK_SIZE, 505 .cra_ctxsize = sizeof(struct s390_pxts_ctx), 506 .cra_type = &crypto_blkcipher_type, 507 .cra_module = THIS_MODULE, 508 .cra_list = LIST_HEAD_INIT(xts_paes_alg.cra_list), 509 .cra_init = xts_paes_init, 510 .cra_exit = xts_paes_exit, 511 .cra_u = { 512 .blkcipher = { 513 .min_keysize = 2 * PAES_MIN_KEYSIZE, 514 .max_keysize = 2 * PAES_MAX_KEYSIZE, 515 .ivsize = AES_BLOCK_SIZE, 516 .setkey = xts_paes_set_key, 517 .encrypt = xts_paes_encrypt, 518 .decrypt = xts_paes_decrypt, 519 } 520 } 521 }; 522 523 static int ctr_paes_init(struct crypto_tfm *tfm) 524 { 525 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 526 527 ctx->kb.key = NULL; 528 529 return 0; 530 } 531 532 static void ctr_paes_exit(struct crypto_tfm *tfm) 533 { 534 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 535 536 _free_kb_keybuf(&ctx->kb); 537 } 538 539 static int __ctr_paes_set_key(struct s390_paes_ctx *ctx) 540 { 541 unsigned long fc; 542 543 if (__paes_convert_key(&ctx->kb, &ctx->pk)) 544 return -EINVAL; 545 546 /* Pick the correct function code based on the protected key type */ 547 fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 : 548 (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 : 549 (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? 550 CPACF_KMCTR_PAES_256 : 0; 551 552 /* Check if the function code is available */ 553 ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0; 554 555 return ctx->fc ? 0 : -EINVAL; 556 } 557 558 static int ctr_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 559 unsigned int key_len) 560 { 561 int rc; 562 struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 563 564 _free_kb_keybuf(&ctx->kb); 565 rc = _copy_key_to_kb(&ctx->kb, in_key, key_len); 566 if (rc) 567 return rc; 568 569 if (__ctr_paes_set_key(ctx)) { 570 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 571 return -EINVAL; 572 } 573 return 0; 574 } 575 576 static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) 577 { 578 unsigned int i, n; 579 580 /* only use complete blocks, max. PAGE_SIZE */ 581 memcpy(ctrptr, iv, AES_BLOCK_SIZE); 582 n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1); 583 for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) { 584 memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE); 585 crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE); 586 ctrptr += AES_BLOCK_SIZE; 587 } 588 return n; 589 } 590 591 static int ctr_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, 592 struct blkcipher_walk *walk) 593 { 594 struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 595 u8 buf[AES_BLOCK_SIZE], *ctrptr; 596 unsigned int nbytes, n, k; 597 int ret, locked; 598 599 locked = spin_trylock(&ctrblk_lock); 600 601 ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE); 602 while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 603 n = AES_BLOCK_SIZE; 604 if (nbytes >= 2*AES_BLOCK_SIZE && locked) 605 n = __ctrblk_init(ctrblk, walk->iv, nbytes); 606 ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; 607 k = cpacf_kmctr(ctx->fc | modifier, ctx->pk.protkey, 608 walk->dst.virt.addr, walk->src.virt.addr, 609 n, ctrptr); 610 if (k) { 611 if (ctrptr == ctrblk) 612 memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE, 613 AES_BLOCK_SIZE); 614 crypto_inc(walk->iv, AES_BLOCK_SIZE); 615 ret = blkcipher_walk_done(desc, walk, nbytes - n); 616 } 617 if (k < n) { 618 if (__ctr_paes_set_key(ctx) != 0) { 619 if (locked) 620 spin_unlock(&ctrblk_lock); 621 return blkcipher_walk_done(desc, walk, -EIO); 622 } 623 } 624 } 625 if (locked) 626 spin_unlock(&ctrblk_lock); 627 /* 628 * final block may be < AES_BLOCK_SIZE, copy only nbytes 629 */ 630 if (nbytes) { 631 while (1) { 632 if (cpacf_kmctr(ctx->fc | modifier, 633 ctx->pk.protkey, buf, 634 walk->src.virt.addr, AES_BLOCK_SIZE, 635 walk->iv) == AES_BLOCK_SIZE) 636 break; 637 if (__ctr_paes_set_key(ctx) != 0) 638 return blkcipher_walk_done(desc, walk, -EIO); 639 } 640 memcpy(walk->dst.virt.addr, buf, nbytes); 641 crypto_inc(walk->iv, AES_BLOCK_SIZE); 642 ret = blkcipher_walk_done(desc, walk, 0); 643 } 644 645 return ret; 646 } 647 648 static int ctr_paes_encrypt(struct blkcipher_desc *desc, 649 struct scatterlist *dst, struct scatterlist *src, 650 unsigned int nbytes) 651 { 652 struct blkcipher_walk walk; 653 654 blkcipher_walk_init(&walk, dst, src, nbytes); 655 return ctr_paes_crypt(desc, 0, &walk); 656 } 657 658 static int ctr_paes_decrypt(struct blkcipher_desc *desc, 659 struct scatterlist *dst, struct scatterlist *src, 660 unsigned int nbytes) 661 { 662 struct blkcipher_walk walk; 663 664 blkcipher_walk_init(&walk, dst, src, nbytes); 665 return ctr_paes_crypt(desc, CPACF_DECRYPT, &walk); 666 } 667 668 static struct crypto_alg ctr_paes_alg = { 669 .cra_name = "ctr(paes)", 670 .cra_driver_name = "ctr-paes-s390", 671 .cra_priority = 402, /* ecb-paes-s390 + 1 */ 672 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 673 .cra_blocksize = 1, 674 .cra_ctxsize = sizeof(struct s390_paes_ctx), 675 .cra_type = &crypto_blkcipher_type, 676 .cra_module = THIS_MODULE, 677 .cra_list = LIST_HEAD_INIT(ctr_paes_alg.cra_list), 678 .cra_init = ctr_paes_init, 679 .cra_exit = ctr_paes_exit, 680 .cra_u = { 681 .blkcipher = { 682 .min_keysize = PAES_MIN_KEYSIZE, 683 .max_keysize = PAES_MAX_KEYSIZE, 684 .ivsize = AES_BLOCK_SIZE, 685 .setkey = ctr_paes_set_key, 686 .encrypt = ctr_paes_encrypt, 687 .decrypt = ctr_paes_decrypt, 688 } 689 } 690 }; 691 692 static inline void __crypto_unregister_alg(struct crypto_alg *alg) 693 { 694 if (!list_empty(&alg->cra_list)) 695 crypto_unregister_alg(alg); 696 } 697 698 static void paes_s390_fini(void) 699 { 700 if (ctrblk) 701 free_page((unsigned long) ctrblk); 702 __crypto_unregister_alg(&ctr_paes_alg); 703 __crypto_unregister_alg(&xts_paes_alg); 704 __crypto_unregister_alg(&cbc_paes_alg); 705 __crypto_unregister_alg(&ecb_paes_alg); 706 } 707 708 static int __init paes_s390_init(void) 709 { 710 int ret; 711 712 /* Query available functions for KM, KMC and KMCTR */ 713 cpacf_query(CPACF_KM, &km_functions); 714 cpacf_query(CPACF_KMC, &kmc_functions); 715 cpacf_query(CPACF_KMCTR, &kmctr_functions); 716 717 if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) || 718 cpacf_test_func(&km_functions, CPACF_KM_PAES_192) || 719 cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) { 720 ret = crypto_register_alg(&ecb_paes_alg); 721 if (ret) 722 goto out_err; 723 } 724 725 if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) || 726 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) || 727 cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) { 728 ret = crypto_register_alg(&cbc_paes_alg); 729 if (ret) 730 goto out_err; 731 } 732 733 if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) || 734 cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) { 735 ret = crypto_register_alg(&xts_paes_alg); 736 if (ret) 737 goto out_err; 738 } 739 740 if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) || 741 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) || 742 cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) { 743 ret = crypto_register_alg(&ctr_paes_alg); 744 if (ret) 745 goto out_err; 746 ctrblk = (u8 *) __get_free_page(GFP_KERNEL); 747 if (!ctrblk) { 748 ret = -ENOMEM; 749 goto out_err; 750 } 751 } 752 753 return 0; 754 out_err: 755 paes_s390_fini(); 756 return ret; 757 } 758 759 module_init(paes_s390_init); 760 module_exit(paes_s390_fini); 761 762 MODULE_ALIAS_CRYPTO("paes"); 763 764 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys"); 765 MODULE_LICENSE("GPL"); 766