1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * SM4 Cipher Algorithm, using ARMv8 Crypto Extensions 4 * as specified in 5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html 6 * 7 * Copyright (C) 2022, Alibaba Group. 8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/crypto.h> 13 #include <linux/kernel.h> 14 #include <linux/cpufeature.h> 15 #include <asm/neon.h> 16 #include <asm/simd.h> 17 #include <crypto/b128ops.h> 18 #include <crypto/internal/simd.h> 19 #include <crypto/internal/skcipher.h> 20 #include <crypto/internal/hash.h> 21 #include <crypto/scatterwalk.h> 22 #include <crypto/xts.h> 23 #include <crypto/sm4.h> 24 25 #define BYTES2BLKS(nbytes) ((nbytes) >> 4) 26 27 asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec, 28 const u32 *fk, const u32 *ck); 29 asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src); 30 asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src, 31 unsigned int nblks); 32 asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src, 33 u8 *iv, unsigned int nblocks); 34 asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src, 35 u8 *iv, unsigned int nblocks); 36 asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src, 37 u8 *iv, unsigned int nbytes); 38 asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src, 39 u8 *iv, unsigned int nbytes); 40 asmlinkage void sm4_ce_cfb_enc(const u32 *rkey, u8 *dst, const u8 *src, 41 u8 *iv, unsigned int nblks); 42 asmlinkage void sm4_ce_cfb_dec(const u32 *rkey, u8 *dst, const u8 *src, 43 u8 *iv, unsigned int nblks); 44 asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src, 45 u8 *iv, unsigned int nblks); 46 asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src, 47 u8 *tweak, unsigned int nbytes, 48 const u32 *rkey2_enc); 49 asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src, 50 u8 *tweak, unsigned int nbytes, 51 const u32 *rkey2_enc); 52 asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest, 53 const u8 *src, unsigned int nblocks, 54 bool enc_before, bool enc_after); 55 56 EXPORT_SYMBOL(sm4_ce_expand_key); 57 EXPORT_SYMBOL(sm4_ce_crypt_block); 58 EXPORT_SYMBOL(sm4_ce_cbc_enc); 59 EXPORT_SYMBOL(sm4_ce_cfb_enc); 60 61 struct sm4_xts_ctx { 62 struct sm4_ctx key1; 63 struct sm4_ctx key2; 64 }; 65 66 struct sm4_mac_tfm_ctx { 67 struct sm4_ctx key; 68 u8 __aligned(8) consts[]; 69 }; 70 71 struct sm4_mac_desc_ctx { 72 unsigned int len; 73 u8 digest[SM4_BLOCK_SIZE]; 74 }; 75 76 static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key, 77 unsigned int key_len) 78 { 79 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 80 81 if (key_len != SM4_KEY_SIZE) 82 return -EINVAL; 83 84 kernel_neon_begin(); 85 sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec, 86 crypto_sm4_fk, crypto_sm4_ck); 87 kernel_neon_end(); 88 return 0; 89 } 90 91 static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, 92 unsigned int key_len) 93 { 94 struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 95 int ret; 96 97 if (key_len != SM4_KEY_SIZE * 2) 98 return -EINVAL; 99 100 ret = xts_verify_key(tfm, key, key_len); 101 if (ret) 102 return ret; 103 104 kernel_neon_begin(); 105 sm4_ce_expand_key(key, ctx->key1.rkey_enc, 106 ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck); 107 sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc, 108 ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck); 109 kernel_neon_end(); 110 111 return 0; 112 } 113 114 static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey) 115 { 116 struct skcipher_walk walk; 117 unsigned int nbytes; 118 int err; 119 120 err = skcipher_walk_virt(&walk, req, false); 121 122 while ((nbytes = walk.nbytes) > 0) { 123 const u8 *src = walk.src.virt.addr; 124 u8 *dst = walk.dst.virt.addr; 125 unsigned int nblks; 126 127 kernel_neon_begin(); 128 129 nblks = BYTES2BLKS(nbytes); 130 if (nblks) { 131 sm4_ce_crypt(rkey, dst, src, nblks); 132 nbytes -= nblks * SM4_BLOCK_SIZE; 133 } 134 135 kernel_neon_end(); 136 137 err = skcipher_walk_done(&walk, nbytes); 138 } 139 140 return err; 141 } 142 143 static int sm4_ecb_encrypt(struct skcipher_request *req) 144 { 145 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 146 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 147 148 return sm4_ecb_do_crypt(req, ctx->rkey_enc); 149 } 150 151 static int sm4_ecb_decrypt(struct skcipher_request *req) 152 { 153 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 154 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 155 156 return sm4_ecb_do_crypt(req, ctx->rkey_dec); 157 } 158 159 static int sm4_cbc_crypt(struct skcipher_request *req, 160 struct sm4_ctx *ctx, bool encrypt) 161 { 162 struct skcipher_walk walk; 163 unsigned int nbytes; 164 int err; 165 166 err = skcipher_walk_virt(&walk, req, false); 167 if (err) 168 return err; 169 170 while ((nbytes = walk.nbytes) > 0) { 171 const u8 *src = walk.src.virt.addr; 172 u8 *dst = walk.dst.virt.addr; 173 unsigned int nblocks; 174 175 nblocks = nbytes / SM4_BLOCK_SIZE; 176 if (nblocks) { 177 kernel_neon_begin(); 178 179 if (encrypt) 180 sm4_ce_cbc_enc(ctx->rkey_enc, dst, src, 181 walk.iv, nblocks); 182 else 183 sm4_ce_cbc_dec(ctx->rkey_dec, dst, src, 184 walk.iv, nblocks); 185 186 kernel_neon_end(); 187 } 188 189 err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE); 190 } 191 192 return err; 193 } 194 195 static int sm4_cbc_encrypt(struct skcipher_request *req) 196 { 197 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 198 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 199 200 return sm4_cbc_crypt(req, ctx, true); 201 } 202 203 static int sm4_cbc_decrypt(struct skcipher_request *req) 204 { 205 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 206 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 207 208 return sm4_cbc_crypt(req, ctx, false); 209 } 210 211 static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt) 212 { 213 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 214 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 215 struct scatterlist *src = req->src; 216 struct scatterlist *dst = req->dst; 217 struct scatterlist sg_src[2], sg_dst[2]; 218 struct skcipher_request subreq; 219 struct skcipher_walk walk; 220 int cbc_blocks; 221 int err; 222 223 if (req->cryptlen < SM4_BLOCK_SIZE) 224 return -EINVAL; 225 226 if (req->cryptlen == SM4_BLOCK_SIZE) 227 return sm4_cbc_crypt(req, ctx, encrypt); 228 229 skcipher_request_set_tfm(&subreq, tfm); 230 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 231 NULL, NULL); 232 233 /* handle the CBC cryption part */ 234 cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2; 235 if (cbc_blocks) { 236 skcipher_request_set_crypt(&subreq, src, dst, 237 cbc_blocks * SM4_BLOCK_SIZE, 238 req->iv); 239 240 err = sm4_cbc_crypt(&subreq, ctx, encrypt); 241 if (err) 242 return err; 243 244 dst = src = scatterwalk_ffwd(sg_src, src, subreq.cryptlen); 245 if (req->dst != req->src) 246 dst = scatterwalk_ffwd(sg_dst, req->dst, 247 subreq.cryptlen); 248 } 249 250 /* handle ciphertext stealing */ 251 skcipher_request_set_crypt(&subreq, src, dst, 252 req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE, 253 req->iv); 254 255 err = skcipher_walk_virt(&walk, &subreq, false); 256 if (err) 257 return err; 258 259 kernel_neon_begin(); 260 261 if (encrypt) 262 sm4_ce_cbc_cts_enc(ctx->rkey_enc, walk.dst.virt.addr, 263 walk.src.virt.addr, walk.iv, walk.nbytes); 264 else 265 sm4_ce_cbc_cts_dec(ctx->rkey_dec, walk.dst.virt.addr, 266 walk.src.virt.addr, walk.iv, walk.nbytes); 267 268 kernel_neon_end(); 269 270 return skcipher_walk_done(&walk, 0); 271 } 272 273 static int sm4_cbc_cts_encrypt(struct skcipher_request *req) 274 { 275 return sm4_cbc_cts_crypt(req, true); 276 } 277 278 static int sm4_cbc_cts_decrypt(struct skcipher_request *req) 279 { 280 return sm4_cbc_cts_crypt(req, false); 281 } 282 283 static int sm4_cfb_encrypt(struct skcipher_request *req) 284 { 285 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 286 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 287 struct skcipher_walk walk; 288 unsigned int nbytes; 289 int err; 290 291 err = skcipher_walk_virt(&walk, req, false); 292 293 while ((nbytes = walk.nbytes) > 0) { 294 const u8 *src = walk.src.virt.addr; 295 u8 *dst = walk.dst.virt.addr; 296 unsigned int nblks; 297 298 kernel_neon_begin(); 299 300 nblks = BYTES2BLKS(nbytes); 301 if (nblks) { 302 sm4_ce_cfb_enc(ctx->rkey_enc, dst, src, walk.iv, nblks); 303 dst += nblks * SM4_BLOCK_SIZE; 304 src += nblks * SM4_BLOCK_SIZE; 305 nbytes -= nblks * SM4_BLOCK_SIZE; 306 } 307 308 /* tail */ 309 if (walk.nbytes == walk.total && nbytes > 0) { 310 u8 keystream[SM4_BLOCK_SIZE]; 311 312 sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv); 313 crypto_xor_cpy(dst, src, keystream, nbytes); 314 nbytes = 0; 315 } 316 317 kernel_neon_end(); 318 319 err = skcipher_walk_done(&walk, nbytes); 320 } 321 322 return err; 323 } 324 325 static int sm4_cfb_decrypt(struct skcipher_request *req) 326 { 327 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 328 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 329 struct skcipher_walk walk; 330 unsigned int nbytes; 331 int err; 332 333 err = skcipher_walk_virt(&walk, req, false); 334 335 while ((nbytes = walk.nbytes) > 0) { 336 const u8 *src = walk.src.virt.addr; 337 u8 *dst = walk.dst.virt.addr; 338 unsigned int nblks; 339 340 kernel_neon_begin(); 341 342 nblks = BYTES2BLKS(nbytes); 343 if (nblks) { 344 sm4_ce_cfb_dec(ctx->rkey_enc, dst, src, walk.iv, nblks); 345 dst += nblks * SM4_BLOCK_SIZE; 346 src += nblks * SM4_BLOCK_SIZE; 347 nbytes -= nblks * SM4_BLOCK_SIZE; 348 } 349 350 /* tail */ 351 if (walk.nbytes == walk.total && nbytes > 0) { 352 u8 keystream[SM4_BLOCK_SIZE]; 353 354 sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv); 355 crypto_xor_cpy(dst, src, keystream, nbytes); 356 nbytes = 0; 357 } 358 359 kernel_neon_end(); 360 361 err = skcipher_walk_done(&walk, nbytes); 362 } 363 364 return err; 365 } 366 367 static int sm4_ctr_crypt(struct skcipher_request *req) 368 { 369 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 370 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); 371 struct skcipher_walk walk; 372 unsigned int nbytes; 373 int err; 374 375 err = skcipher_walk_virt(&walk, req, false); 376 377 while ((nbytes = walk.nbytes) > 0) { 378 const u8 *src = walk.src.virt.addr; 379 u8 *dst = walk.dst.virt.addr; 380 unsigned int nblks; 381 382 kernel_neon_begin(); 383 384 nblks = BYTES2BLKS(nbytes); 385 if (nblks) { 386 sm4_ce_ctr_enc(ctx->rkey_enc, dst, src, walk.iv, nblks); 387 dst += nblks * SM4_BLOCK_SIZE; 388 src += nblks * SM4_BLOCK_SIZE; 389 nbytes -= nblks * SM4_BLOCK_SIZE; 390 } 391 392 /* tail */ 393 if (walk.nbytes == walk.total && nbytes > 0) { 394 u8 keystream[SM4_BLOCK_SIZE]; 395 396 sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv); 397 crypto_inc(walk.iv, SM4_BLOCK_SIZE); 398 crypto_xor_cpy(dst, src, keystream, nbytes); 399 nbytes = 0; 400 } 401 402 kernel_neon_end(); 403 404 err = skcipher_walk_done(&walk, nbytes); 405 } 406 407 return err; 408 } 409 410 static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt) 411 { 412 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 413 struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 414 int tail = req->cryptlen % SM4_BLOCK_SIZE; 415 const u32 *rkey2_enc = ctx->key2.rkey_enc; 416 struct scatterlist sg_src[2], sg_dst[2]; 417 struct skcipher_request subreq; 418 struct scatterlist *src, *dst; 419 struct skcipher_walk walk; 420 unsigned int nbytes; 421 int err; 422 423 if (req->cryptlen < SM4_BLOCK_SIZE) 424 return -EINVAL; 425 426 err = skcipher_walk_virt(&walk, req, false); 427 if (err) 428 return err; 429 430 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 431 int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2; 432 433 skcipher_walk_abort(&walk); 434 435 skcipher_request_set_tfm(&subreq, tfm); 436 skcipher_request_set_callback(&subreq, 437 skcipher_request_flags(req), 438 NULL, NULL); 439 skcipher_request_set_crypt(&subreq, req->src, req->dst, 440 nblocks * SM4_BLOCK_SIZE, req->iv); 441 442 err = skcipher_walk_virt(&walk, &subreq, false); 443 if (err) 444 return err; 445 } else { 446 tail = 0; 447 } 448 449 while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) { 450 if (nbytes < walk.total) 451 nbytes &= ~(SM4_BLOCK_SIZE - 1); 452 453 kernel_neon_begin(); 454 455 if (encrypt) 456 sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr, 457 walk.src.virt.addr, walk.iv, nbytes, 458 rkey2_enc); 459 else 460 sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr, 461 walk.src.virt.addr, walk.iv, nbytes, 462 rkey2_enc); 463 464 kernel_neon_end(); 465 466 rkey2_enc = NULL; 467 468 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 469 if (err) 470 return err; 471 } 472 473 if (likely(tail == 0)) 474 return 0; 475 476 /* handle ciphertext stealing */ 477 478 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 479 if (req->dst != req->src) 480 dst = scatterwalk_ffwd(sg_dst, req->dst, subreq.cryptlen); 481 482 skcipher_request_set_crypt(&subreq, src, dst, SM4_BLOCK_SIZE + tail, 483 req->iv); 484 485 err = skcipher_walk_virt(&walk, &subreq, false); 486 if (err) 487 return err; 488 489 kernel_neon_begin(); 490 491 if (encrypt) 492 sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr, 493 walk.src.virt.addr, walk.iv, walk.nbytes, 494 rkey2_enc); 495 else 496 sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr, 497 walk.src.virt.addr, walk.iv, walk.nbytes, 498 rkey2_enc); 499 500 kernel_neon_end(); 501 502 return skcipher_walk_done(&walk, 0); 503 } 504 505 static int sm4_xts_encrypt(struct skcipher_request *req) 506 { 507 return sm4_xts_crypt(req, true); 508 } 509 510 static int sm4_xts_decrypt(struct skcipher_request *req) 511 { 512 return sm4_xts_crypt(req, false); 513 } 514 515 static struct skcipher_alg sm4_algs[] = { 516 { 517 .base = { 518 .cra_name = "ecb(sm4)", 519 .cra_driver_name = "ecb-sm4-ce", 520 .cra_priority = 400, 521 .cra_blocksize = SM4_BLOCK_SIZE, 522 .cra_ctxsize = sizeof(struct sm4_ctx), 523 .cra_module = THIS_MODULE, 524 }, 525 .min_keysize = SM4_KEY_SIZE, 526 .max_keysize = SM4_KEY_SIZE, 527 .setkey = sm4_setkey, 528 .encrypt = sm4_ecb_encrypt, 529 .decrypt = sm4_ecb_decrypt, 530 }, { 531 .base = { 532 .cra_name = "cbc(sm4)", 533 .cra_driver_name = "cbc-sm4-ce", 534 .cra_priority = 400, 535 .cra_blocksize = SM4_BLOCK_SIZE, 536 .cra_ctxsize = sizeof(struct sm4_ctx), 537 .cra_module = THIS_MODULE, 538 }, 539 .min_keysize = SM4_KEY_SIZE, 540 .max_keysize = SM4_KEY_SIZE, 541 .ivsize = SM4_BLOCK_SIZE, 542 .setkey = sm4_setkey, 543 .encrypt = sm4_cbc_encrypt, 544 .decrypt = sm4_cbc_decrypt, 545 }, { 546 .base = { 547 .cra_name = "cfb(sm4)", 548 .cra_driver_name = "cfb-sm4-ce", 549 .cra_priority = 400, 550 .cra_blocksize = 1, 551 .cra_ctxsize = sizeof(struct sm4_ctx), 552 .cra_module = THIS_MODULE, 553 }, 554 .min_keysize = SM4_KEY_SIZE, 555 .max_keysize = SM4_KEY_SIZE, 556 .ivsize = SM4_BLOCK_SIZE, 557 .chunksize = SM4_BLOCK_SIZE, 558 .setkey = sm4_setkey, 559 .encrypt = sm4_cfb_encrypt, 560 .decrypt = sm4_cfb_decrypt, 561 }, { 562 .base = { 563 .cra_name = "ctr(sm4)", 564 .cra_driver_name = "ctr-sm4-ce", 565 .cra_priority = 400, 566 .cra_blocksize = 1, 567 .cra_ctxsize = sizeof(struct sm4_ctx), 568 .cra_module = THIS_MODULE, 569 }, 570 .min_keysize = SM4_KEY_SIZE, 571 .max_keysize = SM4_KEY_SIZE, 572 .ivsize = SM4_BLOCK_SIZE, 573 .chunksize = SM4_BLOCK_SIZE, 574 .setkey = sm4_setkey, 575 .encrypt = sm4_ctr_crypt, 576 .decrypt = sm4_ctr_crypt, 577 }, { 578 .base = { 579 .cra_name = "cts(cbc(sm4))", 580 .cra_driver_name = "cts-cbc-sm4-ce", 581 .cra_priority = 400, 582 .cra_blocksize = SM4_BLOCK_SIZE, 583 .cra_ctxsize = sizeof(struct sm4_ctx), 584 .cra_module = THIS_MODULE, 585 }, 586 .min_keysize = SM4_KEY_SIZE, 587 .max_keysize = SM4_KEY_SIZE, 588 .ivsize = SM4_BLOCK_SIZE, 589 .walksize = SM4_BLOCK_SIZE * 2, 590 .setkey = sm4_setkey, 591 .encrypt = sm4_cbc_cts_encrypt, 592 .decrypt = sm4_cbc_cts_decrypt, 593 }, { 594 .base = { 595 .cra_name = "xts(sm4)", 596 .cra_driver_name = "xts-sm4-ce", 597 .cra_priority = 400, 598 .cra_blocksize = SM4_BLOCK_SIZE, 599 .cra_ctxsize = sizeof(struct sm4_xts_ctx), 600 .cra_module = THIS_MODULE, 601 }, 602 .min_keysize = SM4_KEY_SIZE * 2, 603 .max_keysize = SM4_KEY_SIZE * 2, 604 .ivsize = SM4_BLOCK_SIZE, 605 .walksize = SM4_BLOCK_SIZE * 2, 606 .setkey = sm4_xts_setkey, 607 .encrypt = sm4_xts_encrypt, 608 .decrypt = sm4_xts_decrypt, 609 } 610 }; 611 612 static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key, 613 unsigned int key_len) 614 { 615 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 616 617 if (key_len != SM4_KEY_SIZE) 618 return -EINVAL; 619 620 kernel_neon_begin(); 621 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec, 622 crypto_sm4_fk, crypto_sm4_ck); 623 kernel_neon_end(); 624 625 return 0; 626 } 627 628 static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key, 629 unsigned int key_len) 630 { 631 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 632 be128 *consts = (be128 *)ctx->consts; 633 u64 a, b; 634 635 if (key_len != SM4_KEY_SIZE) 636 return -EINVAL; 637 638 memset(consts, 0, SM4_BLOCK_SIZE); 639 640 kernel_neon_begin(); 641 642 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec, 643 crypto_sm4_fk, crypto_sm4_ck); 644 645 /* encrypt the zero block */ 646 sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts); 647 648 kernel_neon_end(); 649 650 /* gf(2^128) multiply zero-ciphertext with u and u^2 */ 651 a = be64_to_cpu(consts[0].a); 652 b = be64_to_cpu(consts[0].b); 653 consts[0].a = cpu_to_be64((a << 1) | (b >> 63)); 654 consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); 655 656 a = be64_to_cpu(consts[0].a); 657 b = be64_to_cpu(consts[0].b); 658 consts[1].a = cpu_to_be64((a << 1) | (b >> 63)); 659 consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); 660 661 return 0; 662 } 663 664 static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key, 665 unsigned int key_len) 666 { 667 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 668 u8 __aligned(8) key2[SM4_BLOCK_SIZE]; 669 static u8 const ks[3][SM4_BLOCK_SIZE] = { 670 { [0 ... SM4_BLOCK_SIZE - 1] = 0x1}, 671 { [0 ... SM4_BLOCK_SIZE - 1] = 0x2}, 672 { [0 ... SM4_BLOCK_SIZE - 1] = 0x3}, 673 }; 674 675 if (key_len != SM4_KEY_SIZE) 676 return -EINVAL; 677 678 kernel_neon_begin(); 679 680 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec, 681 crypto_sm4_fk, crypto_sm4_ck); 682 683 sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]); 684 sm4_ce_crypt(ctx->key.rkey_enc, ctx->consts, ks[1], 2); 685 686 sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec, 687 crypto_sm4_fk, crypto_sm4_ck); 688 689 kernel_neon_end(); 690 691 return 0; 692 } 693 694 static int sm4_mac_init(struct shash_desc *desc) 695 { 696 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); 697 698 memset(ctx->digest, 0, SM4_BLOCK_SIZE); 699 ctx->len = 0; 700 701 return 0; 702 } 703 704 static int sm4_mac_update(struct shash_desc *desc, const u8 *p, 705 unsigned int len) 706 { 707 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 708 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); 709 unsigned int l, nblocks; 710 711 if (len == 0) 712 return 0; 713 714 if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) { 715 l = min(len, SM4_BLOCK_SIZE - ctx->len); 716 717 crypto_xor(ctx->digest + ctx->len, p, l); 718 ctx->len += l; 719 len -= l; 720 p += l; 721 } 722 723 if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) { 724 kernel_neon_begin(); 725 726 if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) { 727 sm4_ce_crypt_block(tctx->key.rkey_enc, 728 ctx->digest, ctx->digest); 729 ctx->len = 0; 730 } else { 731 nblocks = len / SM4_BLOCK_SIZE; 732 len %= SM4_BLOCK_SIZE; 733 734 sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p, 735 nblocks, (ctx->len == SM4_BLOCK_SIZE), 736 (len != 0)); 737 738 p += nblocks * SM4_BLOCK_SIZE; 739 740 if (len == 0) 741 ctx->len = SM4_BLOCK_SIZE; 742 } 743 744 kernel_neon_end(); 745 746 if (len) { 747 crypto_xor(ctx->digest, p, len); 748 ctx->len = len; 749 } 750 } 751 752 return 0; 753 } 754 755 static int sm4_cmac_final(struct shash_desc *desc, u8 *out) 756 { 757 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 758 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); 759 const u8 *consts = tctx->consts; 760 761 if (ctx->len != SM4_BLOCK_SIZE) { 762 ctx->digest[ctx->len] ^= 0x80; 763 consts += SM4_BLOCK_SIZE; 764 } 765 766 kernel_neon_begin(); 767 sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1, 768 false, true); 769 kernel_neon_end(); 770 771 memcpy(out, ctx->digest, SM4_BLOCK_SIZE); 772 773 return 0; 774 } 775 776 static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out) 777 { 778 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 779 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); 780 781 if (ctx->len) { 782 kernel_neon_begin(); 783 sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest, 784 ctx->digest); 785 kernel_neon_end(); 786 } 787 788 memcpy(out, ctx->digest, SM4_BLOCK_SIZE); 789 790 return 0; 791 } 792 793 static struct shash_alg sm4_mac_algs[] = { 794 { 795 .base = { 796 .cra_name = "cmac(sm4)", 797 .cra_driver_name = "cmac-sm4-ce", 798 .cra_priority = 400, 799 .cra_blocksize = SM4_BLOCK_SIZE, 800 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) 801 + SM4_BLOCK_SIZE * 2, 802 .cra_module = THIS_MODULE, 803 }, 804 .digestsize = SM4_BLOCK_SIZE, 805 .init = sm4_mac_init, 806 .update = sm4_mac_update, 807 .final = sm4_cmac_final, 808 .setkey = sm4_cmac_setkey, 809 .descsize = sizeof(struct sm4_mac_desc_ctx), 810 }, { 811 .base = { 812 .cra_name = "xcbc(sm4)", 813 .cra_driver_name = "xcbc-sm4-ce", 814 .cra_priority = 400, 815 .cra_blocksize = SM4_BLOCK_SIZE, 816 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) 817 + SM4_BLOCK_SIZE * 2, 818 .cra_module = THIS_MODULE, 819 }, 820 .digestsize = SM4_BLOCK_SIZE, 821 .init = sm4_mac_init, 822 .update = sm4_mac_update, 823 .final = sm4_cmac_final, 824 .setkey = sm4_xcbc_setkey, 825 .descsize = sizeof(struct sm4_mac_desc_ctx), 826 }, { 827 .base = { 828 .cra_name = "cbcmac(sm4)", 829 .cra_driver_name = "cbcmac-sm4-ce", 830 .cra_priority = 400, 831 .cra_blocksize = 1, 832 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx), 833 .cra_module = THIS_MODULE, 834 }, 835 .digestsize = SM4_BLOCK_SIZE, 836 .init = sm4_mac_init, 837 .update = sm4_mac_update, 838 .final = sm4_cbcmac_final, 839 .setkey = sm4_cbcmac_setkey, 840 .descsize = sizeof(struct sm4_mac_desc_ctx), 841 } 842 }; 843 844 static int __init sm4_init(void) 845 { 846 int err; 847 848 err = crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 849 if (err) 850 return err; 851 852 err = crypto_register_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs)); 853 if (err) 854 goto out_err; 855 856 return 0; 857 858 out_err: 859 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 860 return err; 861 } 862 863 static void __exit sm4_exit(void) 864 { 865 crypto_unregister_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs)); 866 crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs)); 867 } 868 869 module_cpu_feature_match(SM4, sm4_init); 870 module_exit(sm4_exit); 871 872 MODULE_DESCRIPTION("SM4 ECB/CBC/CFB/CTR/XTS using ARMv8 Crypto Extensions"); 873 MODULE_ALIAS_CRYPTO("sm4-ce"); 874 MODULE_ALIAS_CRYPTO("sm4"); 875 MODULE_ALIAS_CRYPTO("ecb(sm4)"); 876 MODULE_ALIAS_CRYPTO("cbc(sm4)"); 877 MODULE_ALIAS_CRYPTO("cfb(sm4)"); 878 MODULE_ALIAS_CRYPTO("ctr(sm4)"); 879 MODULE_ALIAS_CRYPTO("cts(cbc(sm4))"); 880 MODULE_ALIAS_CRYPTO("xts(sm4)"); 881 MODULE_ALIAS_CRYPTO("cmac(sm4)"); 882 MODULE_ALIAS_CRYPTO("xcbc(sm4)"); 883 MODULE_ALIAS_CRYPTO("cbcmac(sm4)"); 884 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); 885 MODULE_LICENSE("GPL v2"); 886