15b33e0ecSTianjia Zhang /* SPDX-License-Identifier: GPL-2.0-or-later */
25b33e0ecSTianjia Zhang /*
35b33e0ecSTianjia Zhang * SM4 Cipher Algorithm, using ARMv8 Crypto Extensions
45b33e0ecSTianjia Zhang * as specified in
55b33e0ecSTianjia Zhang * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
65b33e0ecSTianjia Zhang *
75b33e0ecSTianjia Zhang * Copyright (C) 2022, Alibaba Group.
85b33e0ecSTianjia Zhang * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
95b33e0ecSTianjia Zhang */
105b33e0ecSTianjia Zhang
115b33e0ecSTianjia Zhang #include <linux/module.h>
125b33e0ecSTianjia Zhang #include <linux/crypto.h>
135b33e0ecSTianjia Zhang #include <linux/kernel.h>
145b33e0ecSTianjia Zhang #include <linux/cpufeature.h>
155b33e0ecSTianjia Zhang #include <asm/neon.h>
165b33e0ecSTianjia Zhang #include <asm/simd.h>
17*6b5360a5STianjia Zhang #include <crypto/b128ops.h>
185b33e0ecSTianjia Zhang #include <crypto/internal/simd.h>
195b33e0ecSTianjia Zhang #include <crypto/internal/skcipher.h>
20*6b5360a5STianjia Zhang #include <crypto/internal/hash.h>
21b1863fd0STianjia Zhang #include <crypto/scatterwalk.h>
2201f63311STianjia Zhang #include <crypto/xts.h>
235b33e0ecSTianjia Zhang #include <crypto/sm4.h>
245b33e0ecSTianjia Zhang
255b33e0ecSTianjia Zhang #define BYTES2BLKS(nbytes) ((nbytes) >> 4)
265b33e0ecSTianjia Zhang
275b33e0ecSTianjia Zhang asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec,
285b33e0ecSTianjia Zhang const u32 *fk, const u32 *ck);
295b33e0ecSTianjia Zhang asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src);
305b33e0ecSTianjia Zhang asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src,
315b33e0ecSTianjia Zhang unsigned int nblks);
325b33e0ecSTianjia Zhang asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src,
33ce41fefdSTianjia Zhang u8 *iv, unsigned int nblocks);
345b33e0ecSTianjia Zhang asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src,
35ce41fefdSTianjia Zhang u8 *iv, unsigned int nblocks);
36b1863fd0STianjia Zhang asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src,
37b1863fd0STianjia Zhang u8 *iv, unsigned int nbytes);
38b1863fd0STianjia Zhang asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src,
39b1863fd0STianjia Zhang u8 *iv, unsigned int nbytes);
405b33e0ecSTianjia Zhang asmlinkage void sm4_ce_cfb_enc(const u32 *rkey, u8 *dst, const u8 *src,
415b33e0ecSTianjia Zhang u8 *iv, unsigned int nblks);
425b33e0ecSTianjia Zhang asmlinkage void sm4_ce_cfb_dec(const u32 *rkey, u8 *dst, const u8 *src,
435b33e0ecSTianjia Zhang u8 *iv, unsigned int nblks);
445b33e0ecSTianjia Zhang asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src,
455b33e0ecSTianjia Zhang u8 *iv, unsigned int nblks);
4601f63311STianjia Zhang asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src,
4701f63311STianjia Zhang u8 *tweak, unsigned int nbytes,
4801f63311STianjia Zhang const u32 *rkey2_enc);
4901f63311STianjia Zhang asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src,
5001f63311STianjia Zhang u8 *tweak, unsigned int nbytes,
5101f63311STianjia Zhang const u32 *rkey2_enc);
52*6b5360a5STianjia Zhang asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest,
53*6b5360a5STianjia Zhang const u8 *src, unsigned int nblocks,
54*6b5360a5STianjia Zhang bool enc_before, bool enc_after);
555b33e0ecSTianjia Zhang
5645089dbeSTianjia Zhang EXPORT_SYMBOL(sm4_ce_expand_key);
5745089dbeSTianjia Zhang EXPORT_SYMBOL(sm4_ce_crypt_block);
5845089dbeSTianjia Zhang EXPORT_SYMBOL(sm4_ce_cbc_enc);
5945089dbeSTianjia Zhang EXPORT_SYMBOL(sm4_ce_cfb_enc);
6045089dbeSTianjia Zhang
6101f63311STianjia Zhang struct sm4_xts_ctx {
6201f63311STianjia Zhang struct sm4_ctx key1;
6301f63311STianjia Zhang struct sm4_ctx key2;
6401f63311STianjia Zhang };
6501f63311STianjia Zhang
66*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx {
67*6b5360a5STianjia Zhang struct sm4_ctx key;
68*6b5360a5STianjia Zhang u8 __aligned(8) consts[];
69*6b5360a5STianjia Zhang };
70*6b5360a5STianjia Zhang
71*6b5360a5STianjia Zhang struct sm4_mac_desc_ctx {
72*6b5360a5STianjia Zhang unsigned int len;
73*6b5360a5STianjia Zhang u8 digest[SM4_BLOCK_SIZE];
74*6b5360a5STianjia Zhang };
75*6b5360a5STianjia Zhang
sm4_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)765b33e0ecSTianjia Zhang static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
775b33e0ecSTianjia Zhang unsigned int key_len)
785b33e0ecSTianjia Zhang {
795b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
805b33e0ecSTianjia Zhang
815b33e0ecSTianjia Zhang if (key_len != SM4_KEY_SIZE)
825b33e0ecSTianjia Zhang return -EINVAL;
835b33e0ecSTianjia Zhang
84cb9ba02bSTianjia Zhang kernel_neon_begin();
855b33e0ecSTianjia Zhang sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec,
865b33e0ecSTianjia Zhang crypto_sm4_fk, crypto_sm4_ck);
87cb9ba02bSTianjia Zhang kernel_neon_end();
885b33e0ecSTianjia Zhang return 0;
895b33e0ecSTianjia Zhang }
905b33e0ecSTianjia Zhang
sm4_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int key_len)9101f63311STianjia Zhang static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
9201f63311STianjia Zhang unsigned int key_len)
9301f63311STianjia Zhang {
9401f63311STianjia Zhang struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
9501f63311STianjia Zhang int ret;
9601f63311STianjia Zhang
9701f63311STianjia Zhang if (key_len != SM4_KEY_SIZE * 2)
9801f63311STianjia Zhang return -EINVAL;
9901f63311STianjia Zhang
10001f63311STianjia Zhang ret = xts_verify_key(tfm, key, key_len);
10101f63311STianjia Zhang if (ret)
10201f63311STianjia Zhang return ret;
10301f63311STianjia Zhang
10401f63311STianjia Zhang kernel_neon_begin();
10501f63311STianjia Zhang sm4_ce_expand_key(key, ctx->key1.rkey_enc,
10601f63311STianjia Zhang ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
10701f63311STianjia Zhang sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc,
10801f63311STianjia Zhang ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
10901f63311STianjia Zhang kernel_neon_end();
11001f63311STianjia Zhang
11101f63311STianjia Zhang return 0;
11201f63311STianjia Zhang }
11301f63311STianjia Zhang
sm4_ecb_do_crypt(struct skcipher_request * req,const u32 * rkey)1145b33e0ecSTianjia Zhang static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
1155b33e0ecSTianjia Zhang {
1165b33e0ecSTianjia Zhang struct skcipher_walk walk;
1175b33e0ecSTianjia Zhang unsigned int nbytes;
1185b33e0ecSTianjia Zhang int err;
1195b33e0ecSTianjia Zhang
1205b33e0ecSTianjia Zhang err = skcipher_walk_virt(&walk, req, false);
1215b33e0ecSTianjia Zhang
1225b33e0ecSTianjia Zhang while ((nbytes = walk.nbytes) > 0) {
1235b33e0ecSTianjia Zhang const u8 *src = walk.src.virt.addr;
1245b33e0ecSTianjia Zhang u8 *dst = walk.dst.virt.addr;
1255b33e0ecSTianjia Zhang unsigned int nblks;
1265b33e0ecSTianjia Zhang
1275b33e0ecSTianjia Zhang kernel_neon_begin();
1285b33e0ecSTianjia Zhang
1295b33e0ecSTianjia Zhang nblks = BYTES2BLKS(nbytes);
1305b33e0ecSTianjia Zhang if (nblks) {
1315b33e0ecSTianjia Zhang sm4_ce_crypt(rkey, dst, src, nblks);
1325b33e0ecSTianjia Zhang nbytes -= nblks * SM4_BLOCK_SIZE;
1335b33e0ecSTianjia Zhang }
1345b33e0ecSTianjia Zhang
1355b33e0ecSTianjia Zhang kernel_neon_end();
1365b33e0ecSTianjia Zhang
1375b33e0ecSTianjia Zhang err = skcipher_walk_done(&walk, nbytes);
1385b33e0ecSTianjia Zhang }
1395b33e0ecSTianjia Zhang
1405b33e0ecSTianjia Zhang return err;
1415b33e0ecSTianjia Zhang }
1425b33e0ecSTianjia Zhang
sm4_ecb_encrypt(struct skcipher_request * req)1435b33e0ecSTianjia Zhang static int sm4_ecb_encrypt(struct skcipher_request *req)
1445b33e0ecSTianjia Zhang {
1455b33e0ecSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1465b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
1475b33e0ecSTianjia Zhang
1485b33e0ecSTianjia Zhang return sm4_ecb_do_crypt(req, ctx->rkey_enc);
1495b33e0ecSTianjia Zhang }
1505b33e0ecSTianjia Zhang
sm4_ecb_decrypt(struct skcipher_request * req)1515b33e0ecSTianjia Zhang static int sm4_ecb_decrypt(struct skcipher_request *req)
1525b33e0ecSTianjia Zhang {
1535b33e0ecSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1545b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
1555b33e0ecSTianjia Zhang
1565b33e0ecSTianjia Zhang return sm4_ecb_do_crypt(req, ctx->rkey_dec);
1575b33e0ecSTianjia Zhang }
1585b33e0ecSTianjia Zhang
sm4_cbc_crypt(struct skcipher_request * req,struct sm4_ctx * ctx,bool encrypt)159ce41fefdSTianjia Zhang static int sm4_cbc_crypt(struct skcipher_request *req,
160ce41fefdSTianjia Zhang struct sm4_ctx *ctx, bool encrypt)
1615b33e0ecSTianjia Zhang {
1625b33e0ecSTianjia Zhang struct skcipher_walk walk;
1635b33e0ecSTianjia Zhang unsigned int nbytes;
1645b33e0ecSTianjia Zhang int err;
1655b33e0ecSTianjia Zhang
1665b33e0ecSTianjia Zhang err = skcipher_walk_virt(&walk, req, false);
167ce41fefdSTianjia Zhang if (err)
168ce41fefdSTianjia Zhang return err;
1695b33e0ecSTianjia Zhang
1705b33e0ecSTianjia Zhang while ((nbytes = walk.nbytes) > 0) {
1715b33e0ecSTianjia Zhang const u8 *src = walk.src.virt.addr;
1725b33e0ecSTianjia Zhang u8 *dst = walk.dst.virt.addr;
173ce41fefdSTianjia Zhang unsigned int nblocks;
1745b33e0ecSTianjia Zhang
175ce41fefdSTianjia Zhang nblocks = nbytes / SM4_BLOCK_SIZE;
176ce41fefdSTianjia Zhang if (nblocks) {
1775b33e0ecSTianjia Zhang kernel_neon_begin();
1785b33e0ecSTianjia Zhang
179ce41fefdSTianjia Zhang if (encrypt)
180ce41fefdSTianjia Zhang sm4_ce_cbc_enc(ctx->rkey_enc, dst, src,
181ce41fefdSTianjia Zhang walk.iv, nblocks);
182ce41fefdSTianjia Zhang else
183ce41fefdSTianjia Zhang sm4_ce_cbc_dec(ctx->rkey_dec, dst, src,
184ce41fefdSTianjia Zhang walk.iv, nblocks);
1855b33e0ecSTianjia Zhang
1865b33e0ecSTianjia Zhang kernel_neon_end();
187ce41fefdSTianjia Zhang }
1885b33e0ecSTianjia Zhang
189ce41fefdSTianjia Zhang err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
1905b33e0ecSTianjia Zhang }
1915b33e0ecSTianjia Zhang
1925b33e0ecSTianjia Zhang return err;
1935b33e0ecSTianjia Zhang }
1945b33e0ecSTianjia Zhang
sm4_cbc_encrypt(struct skcipher_request * req)195ce41fefdSTianjia Zhang static int sm4_cbc_encrypt(struct skcipher_request *req)
196ce41fefdSTianjia Zhang {
197ce41fefdSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
198ce41fefdSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
199ce41fefdSTianjia Zhang
200ce41fefdSTianjia Zhang return sm4_cbc_crypt(req, ctx, true);
201ce41fefdSTianjia Zhang }
202ce41fefdSTianjia Zhang
sm4_cbc_decrypt(struct skcipher_request * req)2035b33e0ecSTianjia Zhang static int sm4_cbc_decrypt(struct skcipher_request *req)
2045b33e0ecSTianjia Zhang {
2055b33e0ecSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2065b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
2075b33e0ecSTianjia Zhang
208ce41fefdSTianjia Zhang return sm4_cbc_crypt(req, ctx, false);
2095b33e0ecSTianjia Zhang }
2105b33e0ecSTianjia Zhang
sm4_cbc_cts_crypt(struct skcipher_request * req,bool encrypt)211b1863fd0STianjia Zhang static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt)
212b1863fd0STianjia Zhang {
213b1863fd0STianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
214b1863fd0STianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
215b1863fd0STianjia Zhang struct scatterlist *src = req->src;
216b1863fd0STianjia Zhang struct scatterlist *dst = req->dst;
217b1863fd0STianjia Zhang struct scatterlist sg_src[2], sg_dst[2];
218b1863fd0STianjia Zhang struct skcipher_request subreq;
219b1863fd0STianjia Zhang struct skcipher_walk walk;
220b1863fd0STianjia Zhang int cbc_blocks;
221b1863fd0STianjia Zhang int err;
222b1863fd0STianjia Zhang
223b1863fd0STianjia Zhang if (req->cryptlen < SM4_BLOCK_SIZE)
224b1863fd0STianjia Zhang return -EINVAL;
225b1863fd0STianjia Zhang
226b1863fd0STianjia Zhang if (req->cryptlen == SM4_BLOCK_SIZE)
227b1863fd0STianjia Zhang return sm4_cbc_crypt(req, ctx, encrypt);
228b1863fd0STianjia Zhang
229b1863fd0STianjia Zhang skcipher_request_set_tfm(&subreq, tfm);
230b1863fd0STianjia Zhang skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
231b1863fd0STianjia Zhang NULL, NULL);
232b1863fd0STianjia Zhang
233b1863fd0STianjia Zhang /* handle the CBC cryption part */
234b1863fd0STianjia Zhang cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
235b1863fd0STianjia Zhang if (cbc_blocks) {
236b1863fd0STianjia Zhang skcipher_request_set_crypt(&subreq, src, dst,
237b1863fd0STianjia Zhang cbc_blocks * SM4_BLOCK_SIZE,
238b1863fd0STianjia Zhang req->iv);
239b1863fd0STianjia Zhang
240b1863fd0STianjia Zhang err = sm4_cbc_crypt(&subreq, ctx, encrypt);
241b1863fd0STianjia Zhang if (err)
242b1863fd0STianjia Zhang return err;
243b1863fd0STianjia Zhang
244b1863fd0STianjia Zhang dst = src = scatterwalk_ffwd(sg_src, src, subreq.cryptlen);
245b1863fd0STianjia Zhang if (req->dst != req->src)
246b1863fd0STianjia Zhang dst = scatterwalk_ffwd(sg_dst, req->dst,
247b1863fd0STianjia Zhang subreq.cryptlen);
248b1863fd0STianjia Zhang }
249b1863fd0STianjia Zhang
250b1863fd0STianjia Zhang /* handle ciphertext stealing */
251b1863fd0STianjia Zhang skcipher_request_set_crypt(&subreq, src, dst,
252b1863fd0STianjia Zhang req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE,
253b1863fd0STianjia Zhang req->iv);
254b1863fd0STianjia Zhang
255b1863fd0STianjia Zhang err = skcipher_walk_virt(&walk, &subreq, false);
256b1863fd0STianjia Zhang if (err)
257b1863fd0STianjia Zhang return err;
258b1863fd0STianjia Zhang
259b1863fd0STianjia Zhang kernel_neon_begin();
260b1863fd0STianjia Zhang
261b1863fd0STianjia Zhang if (encrypt)
262b1863fd0STianjia Zhang sm4_ce_cbc_cts_enc(ctx->rkey_enc, walk.dst.virt.addr,
263b1863fd0STianjia Zhang walk.src.virt.addr, walk.iv, walk.nbytes);
264b1863fd0STianjia Zhang else
265b1863fd0STianjia Zhang sm4_ce_cbc_cts_dec(ctx->rkey_dec, walk.dst.virt.addr,
266b1863fd0STianjia Zhang walk.src.virt.addr, walk.iv, walk.nbytes);
267b1863fd0STianjia Zhang
268b1863fd0STianjia Zhang kernel_neon_end();
269b1863fd0STianjia Zhang
270b1863fd0STianjia Zhang return skcipher_walk_done(&walk, 0);
271b1863fd0STianjia Zhang }
272b1863fd0STianjia Zhang
sm4_cbc_cts_encrypt(struct skcipher_request * req)273b1863fd0STianjia Zhang static int sm4_cbc_cts_encrypt(struct skcipher_request *req)
274b1863fd0STianjia Zhang {
275b1863fd0STianjia Zhang return sm4_cbc_cts_crypt(req, true);
276b1863fd0STianjia Zhang }
277b1863fd0STianjia Zhang
sm4_cbc_cts_decrypt(struct skcipher_request * req)278b1863fd0STianjia Zhang static int sm4_cbc_cts_decrypt(struct skcipher_request *req)
279b1863fd0STianjia Zhang {
280b1863fd0STianjia Zhang return sm4_cbc_cts_crypt(req, false);
281b1863fd0STianjia Zhang }
282b1863fd0STianjia Zhang
sm4_cfb_encrypt(struct skcipher_request * req)2835b33e0ecSTianjia Zhang static int sm4_cfb_encrypt(struct skcipher_request *req)
2845b33e0ecSTianjia Zhang {
2855b33e0ecSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2865b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
2875b33e0ecSTianjia Zhang struct skcipher_walk walk;
2885b33e0ecSTianjia Zhang unsigned int nbytes;
2895b33e0ecSTianjia Zhang int err;
2905b33e0ecSTianjia Zhang
2915b33e0ecSTianjia Zhang err = skcipher_walk_virt(&walk, req, false);
2925b33e0ecSTianjia Zhang
2935b33e0ecSTianjia Zhang while ((nbytes = walk.nbytes) > 0) {
2945b33e0ecSTianjia Zhang const u8 *src = walk.src.virt.addr;
2955b33e0ecSTianjia Zhang u8 *dst = walk.dst.virt.addr;
2965b33e0ecSTianjia Zhang unsigned int nblks;
2975b33e0ecSTianjia Zhang
2985b33e0ecSTianjia Zhang kernel_neon_begin();
2995b33e0ecSTianjia Zhang
3005b33e0ecSTianjia Zhang nblks = BYTES2BLKS(nbytes);
3015b33e0ecSTianjia Zhang if (nblks) {
3025b33e0ecSTianjia Zhang sm4_ce_cfb_enc(ctx->rkey_enc, dst, src, walk.iv, nblks);
3035b33e0ecSTianjia Zhang dst += nblks * SM4_BLOCK_SIZE;
3045b33e0ecSTianjia Zhang src += nblks * SM4_BLOCK_SIZE;
3055b33e0ecSTianjia Zhang nbytes -= nblks * SM4_BLOCK_SIZE;
3065b33e0ecSTianjia Zhang }
3075b33e0ecSTianjia Zhang
3085b33e0ecSTianjia Zhang /* tail */
3095b33e0ecSTianjia Zhang if (walk.nbytes == walk.total && nbytes > 0) {
3105b33e0ecSTianjia Zhang u8 keystream[SM4_BLOCK_SIZE];
3115b33e0ecSTianjia Zhang
3125b33e0ecSTianjia Zhang sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
3135b33e0ecSTianjia Zhang crypto_xor_cpy(dst, src, keystream, nbytes);
3145b33e0ecSTianjia Zhang nbytes = 0;
3155b33e0ecSTianjia Zhang }
3165b33e0ecSTianjia Zhang
3175b33e0ecSTianjia Zhang kernel_neon_end();
3185b33e0ecSTianjia Zhang
3195b33e0ecSTianjia Zhang err = skcipher_walk_done(&walk, nbytes);
3205b33e0ecSTianjia Zhang }
3215b33e0ecSTianjia Zhang
3225b33e0ecSTianjia Zhang return err;
3235b33e0ecSTianjia Zhang }
3245b33e0ecSTianjia Zhang
sm4_cfb_decrypt(struct skcipher_request * req)3255b33e0ecSTianjia Zhang static int sm4_cfb_decrypt(struct skcipher_request *req)
3265b33e0ecSTianjia Zhang {
3275b33e0ecSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3285b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
3295b33e0ecSTianjia Zhang struct skcipher_walk walk;
3305b33e0ecSTianjia Zhang unsigned int nbytes;
3315b33e0ecSTianjia Zhang int err;
3325b33e0ecSTianjia Zhang
3335b33e0ecSTianjia Zhang err = skcipher_walk_virt(&walk, req, false);
3345b33e0ecSTianjia Zhang
3355b33e0ecSTianjia Zhang while ((nbytes = walk.nbytes) > 0) {
3365b33e0ecSTianjia Zhang const u8 *src = walk.src.virt.addr;
3375b33e0ecSTianjia Zhang u8 *dst = walk.dst.virt.addr;
3385b33e0ecSTianjia Zhang unsigned int nblks;
3395b33e0ecSTianjia Zhang
3405b33e0ecSTianjia Zhang kernel_neon_begin();
3415b33e0ecSTianjia Zhang
3425b33e0ecSTianjia Zhang nblks = BYTES2BLKS(nbytes);
3435b33e0ecSTianjia Zhang if (nblks) {
3445b33e0ecSTianjia Zhang sm4_ce_cfb_dec(ctx->rkey_enc, dst, src, walk.iv, nblks);
3455b33e0ecSTianjia Zhang dst += nblks * SM4_BLOCK_SIZE;
3465b33e0ecSTianjia Zhang src += nblks * SM4_BLOCK_SIZE;
3475b33e0ecSTianjia Zhang nbytes -= nblks * SM4_BLOCK_SIZE;
3485b33e0ecSTianjia Zhang }
3495b33e0ecSTianjia Zhang
3505b33e0ecSTianjia Zhang /* tail */
3515b33e0ecSTianjia Zhang if (walk.nbytes == walk.total && nbytes > 0) {
3525b33e0ecSTianjia Zhang u8 keystream[SM4_BLOCK_SIZE];
3535b33e0ecSTianjia Zhang
3545b33e0ecSTianjia Zhang sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
3555b33e0ecSTianjia Zhang crypto_xor_cpy(dst, src, keystream, nbytes);
3565b33e0ecSTianjia Zhang nbytes = 0;
3575b33e0ecSTianjia Zhang }
3585b33e0ecSTianjia Zhang
3595b33e0ecSTianjia Zhang kernel_neon_end();
3605b33e0ecSTianjia Zhang
3615b33e0ecSTianjia Zhang err = skcipher_walk_done(&walk, nbytes);
3625b33e0ecSTianjia Zhang }
3635b33e0ecSTianjia Zhang
3645b33e0ecSTianjia Zhang return err;
3655b33e0ecSTianjia Zhang }
3665b33e0ecSTianjia Zhang
sm4_ctr_crypt(struct skcipher_request * req)3675b33e0ecSTianjia Zhang static int sm4_ctr_crypt(struct skcipher_request *req)
3685b33e0ecSTianjia Zhang {
3695b33e0ecSTianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3705b33e0ecSTianjia Zhang struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
3715b33e0ecSTianjia Zhang struct skcipher_walk walk;
3725b33e0ecSTianjia Zhang unsigned int nbytes;
3735b33e0ecSTianjia Zhang int err;
3745b33e0ecSTianjia Zhang
3755b33e0ecSTianjia Zhang err = skcipher_walk_virt(&walk, req, false);
3765b33e0ecSTianjia Zhang
3775b33e0ecSTianjia Zhang while ((nbytes = walk.nbytes) > 0) {
3785b33e0ecSTianjia Zhang const u8 *src = walk.src.virt.addr;
3795b33e0ecSTianjia Zhang u8 *dst = walk.dst.virt.addr;
3805b33e0ecSTianjia Zhang unsigned int nblks;
3815b33e0ecSTianjia Zhang
3825b33e0ecSTianjia Zhang kernel_neon_begin();
3835b33e0ecSTianjia Zhang
3845b33e0ecSTianjia Zhang nblks = BYTES2BLKS(nbytes);
3855b33e0ecSTianjia Zhang if (nblks) {
3865b33e0ecSTianjia Zhang sm4_ce_ctr_enc(ctx->rkey_enc, dst, src, walk.iv, nblks);
3875b33e0ecSTianjia Zhang dst += nblks * SM4_BLOCK_SIZE;
3885b33e0ecSTianjia Zhang src += nblks * SM4_BLOCK_SIZE;
3895b33e0ecSTianjia Zhang nbytes -= nblks * SM4_BLOCK_SIZE;
3905b33e0ecSTianjia Zhang }
3915b33e0ecSTianjia Zhang
3925b33e0ecSTianjia Zhang /* tail */
3935b33e0ecSTianjia Zhang if (walk.nbytes == walk.total && nbytes > 0) {
3945b33e0ecSTianjia Zhang u8 keystream[SM4_BLOCK_SIZE];
3955b33e0ecSTianjia Zhang
3965b33e0ecSTianjia Zhang sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
3975b33e0ecSTianjia Zhang crypto_inc(walk.iv, SM4_BLOCK_SIZE);
3985b33e0ecSTianjia Zhang crypto_xor_cpy(dst, src, keystream, nbytes);
3995b33e0ecSTianjia Zhang nbytes = 0;
4005b33e0ecSTianjia Zhang }
4015b33e0ecSTianjia Zhang
4025b33e0ecSTianjia Zhang kernel_neon_end();
4035b33e0ecSTianjia Zhang
4045b33e0ecSTianjia Zhang err = skcipher_walk_done(&walk, nbytes);
4055b33e0ecSTianjia Zhang }
4065b33e0ecSTianjia Zhang
4075b33e0ecSTianjia Zhang return err;
4085b33e0ecSTianjia Zhang }
4095b33e0ecSTianjia Zhang
sm4_xts_crypt(struct skcipher_request * req,bool encrypt)41001f63311STianjia Zhang static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt)
41101f63311STianjia Zhang {
41201f63311STianjia Zhang struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
41301f63311STianjia Zhang struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
41401f63311STianjia Zhang int tail = req->cryptlen % SM4_BLOCK_SIZE;
41501f63311STianjia Zhang const u32 *rkey2_enc = ctx->key2.rkey_enc;
41601f63311STianjia Zhang struct scatterlist sg_src[2], sg_dst[2];
41701f63311STianjia Zhang struct skcipher_request subreq;
41801f63311STianjia Zhang struct scatterlist *src, *dst;
41901f63311STianjia Zhang struct skcipher_walk walk;
42001f63311STianjia Zhang unsigned int nbytes;
42101f63311STianjia Zhang int err;
42201f63311STianjia Zhang
42301f63311STianjia Zhang if (req->cryptlen < SM4_BLOCK_SIZE)
42401f63311STianjia Zhang return -EINVAL;
42501f63311STianjia Zhang
42601f63311STianjia Zhang err = skcipher_walk_virt(&walk, req, false);
42701f63311STianjia Zhang if (err)
42801f63311STianjia Zhang return err;
42901f63311STianjia Zhang
43001f63311STianjia Zhang if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
43101f63311STianjia Zhang int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
43201f63311STianjia Zhang
43301f63311STianjia Zhang skcipher_walk_abort(&walk);
43401f63311STianjia Zhang
43501f63311STianjia Zhang skcipher_request_set_tfm(&subreq, tfm);
43601f63311STianjia Zhang skcipher_request_set_callback(&subreq,
43701f63311STianjia Zhang skcipher_request_flags(req),
43801f63311STianjia Zhang NULL, NULL);
43901f63311STianjia Zhang skcipher_request_set_crypt(&subreq, req->src, req->dst,
44001f63311STianjia Zhang nblocks * SM4_BLOCK_SIZE, req->iv);
44101f63311STianjia Zhang
44201f63311STianjia Zhang err = skcipher_walk_virt(&walk, &subreq, false);
44301f63311STianjia Zhang if (err)
44401f63311STianjia Zhang return err;
44501f63311STianjia Zhang } else {
44601f63311STianjia Zhang tail = 0;
44701f63311STianjia Zhang }
44801f63311STianjia Zhang
44901f63311STianjia Zhang while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) {
45001f63311STianjia Zhang if (nbytes < walk.total)
45101f63311STianjia Zhang nbytes &= ~(SM4_BLOCK_SIZE - 1);
45201f63311STianjia Zhang
45301f63311STianjia Zhang kernel_neon_begin();
45401f63311STianjia Zhang
45501f63311STianjia Zhang if (encrypt)
45601f63311STianjia Zhang sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
45701f63311STianjia Zhang walk.src.virt.addr, walk.iv, nbytes,
45801f63311STianjia Zhang rkey2_enc);
45901f63311STianjia Zhang else
46001f63311STianjia Zhang sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
46101f63311STianjia Zhang walk.src.virt.addr, walk.iv, nbytes,
46201f63311STianjia Zhang rkey2_enc);
46301f63311STianjia Zhang
46401f63311STianjia Zhang kernel_neon_end();
46501f63311STianjia Zhang
46601f63311STianjia Zhang rkey2_enc = NULL;
46701f63311STianjia Zhang
46801f63311STianjia Zhang err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
46901f63311STianjia Zhang if (err)
47001f63311STianjia Zhang return err;
47101f63311STianjia Zhang }
47201f63311STianjia Zhang
47301f63311STianjia Zhang if (likely(tail == 0))
47401f63311STianjia Zhang return 0;
47501f63311STianjia Zhang
47601f63311STianjia Zhang /* handle ciphertext stealing */
47701f63311STianjia Zhang
47801f63311STianjia Zhang dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
47901f63311STianjia Zhang if (req->dst != req->src)
48001f63311STianjia Zhang dst = scatterwalk_ffwd(sg_dst, req->dst, subreq.cryptlen);
48101f63311STianjia Zhang
48201f63311STianjia Zhang skcipher_request_set_crypt(&subreq, src, dst, SM4_BLOCK_SIZE + tail,
48301f63311STianjia Zhang req->iv);
48401f63311STianjia Zhang
48501f63311STianjia Zhang err = skcipher_walk_virt(&walk, &subreq, false);
48601f63311STianjia Zhang if (err)
48701f63311STianjia Zhang return err;
48801f63311STianjia Zhang
48901f63311STianjia Zhang kernel_neon_begin();
49001f63311STianjia Zhang
49101f63311STianjia Zhang if (encrypt)
49201f63311STianjia Zhang sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
49301f63311STianjia Zhang walk.src.virt.addr, walk.iv, walk.nbytes,
49401f63311STianjia Zhang rkey2_enc);
49501f63311STianjia Zhang else
49601f63311STianjia Zhang sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
49701f63311STianjia Zhang walk.src.virt.addr, walk.iv, walk.nbytes,
49801f63311STianjia Zhang rkey2_enc);
49901f63311STianjia Zhang
50001f63311STianjia Zhang kernel_neon_end();
50101f63311STianjia Zhang
50201f63311STianjia Zhang return skcipher_walk_done(&walk, 0);
50301f63311STianjia Zhang }
50401f63311STianjia Zhang
sm4_xts_encrypt(struct skcipher_request * req)50501f63311STianjia Zhang static int sm4_xts_encrypt(struct skcipher_request *req)
50601f63311STianjia Zhang {
50701f63311STianjia Zhang return sm4_xts_crypt(req, true);
50801f63311STianjia Zhang }
50901f63311STianjia Zhang
sm4_xts_decrypt(struct skcipher_request * req)51001f63311STianjia Zhang static int sm4_xts_decrypt(struct skcipher_request *req)
51101f63311STianjia Zhang {
51201f63311STianjia Zhang return sm4_xts_crypt(req, false);
51301f63311STianjia Zhang }
51401f63311STianjia Zhang
5155b33e0ecSTianjia Zhang static struct skcipher_alg sm4_algs[] = {
5165b33e0ecSTianjia Zhang {
5175b33e0ecSTianjia Zhang .base = {
5185b33e0ecSTianjia Zhang .cra_name = "ecb(sm4)",
5195b33e0ecSTianjia Zhang .cra_driver_name = "ecb-sm4-ce",
5205b33e0ecSTianjia Zhang .cra_priority = 400,
5215b33e0ecSTianjia Zhang .cra_blocksize = SM4_BLOCK_SIZE,
5225b33e0ecSTianjia Zhang .cra_ctxsize = sizeof(struct sm4_ctx),
5235b33e0ecSTianjia Zhang .cra_module = THIS_MODULE,
5245b33e0ecSTianjia Zhang },
5255b33e0ecSTianjia Zhang .min_keysize = SM4_KEY_SIZE,
5265b33e0ecSTianjia Zhang .max_keysize = SM4_KEY_SIZE,
5275b33e0ecSTianjia Zhang .setkey = sm4_setkey,
5285b33e0ecSTianjia Zhang .encrypt = sm4_ecb_encrypt,
5295b33e0ecSTianjia Zhang .decrypt = sm4_ecb_decrypt,
5305b33e0ecSTianjia Zhang }, {
5315b33e0ecSTianjia Zhang .base = {
5325b33e0ecSTianjia Zhang .cra_name = "cbc(sm4)",
5335b33e0ecSTianjia Zhang .cra_driver_name = "cbc-sm4-ce",
5345b33e0ecSTianjia Zhang .cra_priority = 400,
5355b33e0ecSTianjia Zhang .cra_blocksize = SM4_BLOCK_SIZE,
5365b33e0ecSTianjia Zhang .cra_ctxsize = sizeof(struct sm4_ctx),
5375b33e0ecSTianjia Zhang .cra_module = THIS_MODULE,
5385b33e0ecSTianjia Zhang },
5395b33e0ecSTianjia Zhang .min_keysize = SM4_KEY_SIZE,
5405b33e0ecSTianjia Zhang .max_keysize = SM4_KEY_SIZE,
5415b33e0ecSTianjia Zhang .ivsize = SM4_BLOCK_SIZE,
5425b33e0ecSTianjia Zhang .setkey = sm4_setkey,
5435b33e0ecSTianjia Zhang .encrypt = sm4_cbc_encrypt,
5445b33e0ecSTianjia Zhang .decrypt = sm4_cbc_decrypt,
5455b33e0ecSTianjia Zhang }, {
5465b33e0ecSTianjia Zhang .base = {
5475b33e0ecSTianjia Zhang .cra_name = "cfb(sm4)",
5485b33e0ecSTianjia Zhang .cra_driver_name = "cfb-sm4-ce",
5495b33e0ecSTianjia Zhang .cra_priority = 400,
5505b33e0ecSTianjia Zhang .cra_blocksize = 1,
5515b33e0ecSTianjia Zhang .cra_ctxsize = sizeof(struct sm4_ctx),
5525b33e0ecSTianjia Zhang .cra_module = THIS_MODULE,
5535b33e0ecSTianjia Zhang },
5545b33e0ecSTianjia Zhang .min_keysize = SM4_KEY_SIZE,
5555b33e0ecSTianjia Zhang .max_keysize = SM4_KEY_SIZE,
5565b33e0ecSTianjia Zhang .ivsize = SM4_BLOCK_SIZE,
5575b33e0ecSTianjia Zhang .chunksize = SM4_BLOCK_SIZE,
5585b33e0ecSTianjia Zhang .setkey = sm4_setkey,
5595b33e0ecSTianjia Zhang .encrypt = sm4_cfb_encrypt,
5605b33e0ecSTianjia Zhang .decrypt = sm4_cfb_decrypt,
5615b33e0ecSTianjia Zhang }, {
5625b33e0ecSTianjia Zhang .base = {
5635b33e0ecSTianjia Zhang .cra_name = "ctr(sm4)",
5645b33e0ecSTianjia Zhang .cra_driver_name = "ctr-sm4-ce",
5655b33e0ecSTianjia Zhang .cra_priority = 400,
5665b33e0ecSTianjia Zhang .cra_blocksize = 1,
5675b33e0ecSTianjia Zhang .cra_ctxsize = sizeof(struct sm4_ctx),
5685b33e0ecSTianjia Zhang .cra_module = THIS_MODULE,
5695b33e0ecSTianjia Zhang },
5705b33e0ecSTianjia Zhang .min_keysize = SM4_KEY_SIZE,
5715b33e0ecSTianjia Zhang .max_keysize = SM4_KEY_SIZE,
5725b33e0ecSTianjia Zhang .ivsize = SM4_BLOCK_SIZE,
5735b33e0ecSTianjia Zhang .chunksize = SM4_BLOCK_SIZE,
5745b33e0ecSTianjia Zhang .setkey = sm4_setkey,
5755b33e0ecSTianjia Zhang .encrypt = sm4_ctr_crypt,
5765b33e0ecSTianjia Zhang .decrypt = sm4_ctr_crypt,
577b1863fd0STianjia Zhang }, {
578b1863fd0STianjia Zhang .base = {
579b1863fd0STianjia Zhang .cra_name = "cts(cbc(sm4))",
580b1863fd0STianjia Zhang .cra_driver_name = "cts-cbc-sm4-ce",
581b1863fd0STianjia Zhang .cra_priority = 400,
582b1863fd0STianjia Zhang .cra_blocksize = SM4_BLOCK_SIZE,
583b1863fd0STianjia Zhang .cra_ctxsize = sizeof(struct sm4_ctx),
584b1863fd0STianjia Zhang .cra_module = THIS_MODULE,
585b1863fd0STianjia Zhang },
586b1863fd0STianjia Zhang .min_keysize = SM4_KEY_SIZE,
587b1863fd0STianjia Zhang .max_keysize = SM4_KEY_SIZE,
588b1863fd0STianjia Zhang .ivsize = SM4_BLOCK_SIZE,
589b1863fd0STianjia Zhang .walksize = SM4_BLOCK_SIZE * 2,
590b1863fd0STianjia Zhang .setkey = sm4_setkey,
591b1863fd0STianjia Zhang .encrypt = sm4_cbc_cts_encrypt,
592b1863fd0STianjia Zhang .decrypt = sm4_cbc_cts_decrypt,
59301f63311STianjia Zhang }, {
59401f63311STianjia Zhang .base = {
59501f63311STianjia Zhang .cra_name = "xts(sm4)",
59601f63311STianjia Zhang .cra_driver_name = "xts-sm4-ce",
59701f63311STianjia Zhang .cra_priority = 400,
59801f63311STianjia Zhang .cra_blocksize = SM4_BLOCK_SIZE,
59901f63311STianjia Zhang .cra_ctxsize = sizeof(struct sm4_xts_ctx),
60001f63311STianjia Zhang .cra_module = THIS_MODULE,
60101f63311STianjia Zhang },
60201f63311STianjia Zhang .min_keysize = SM4_KEY_SIZE * 2,
60301f63311STianjia Zhang .max_keysize = SM4_KEY_SIZE * 2,
60401f63311STianjia Zhang .ivsize = SM4_BLOCK_SIZE,
60501f63311STianjia Zhang .walksize = SM4_BLOCK_SIZE * 2,
60601f63311STianjia Zhang .setkey = sm4_xts_setkey,
60701f63311STianjia Zhang .encrypt = sm4_xts_encrypt,
60801f63311STianjia Zhang .decrypt = sm4_xts_decrypt,
6095b33e0ecSTianjia Zhang }
6105b33e0ecSTianjia Zhang };
6115b33e0ecSTianjia Zhang
sm4_cbcmac_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int key_len)612*6b5360a5STianjia Zhang static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key,
613*6b5360a5STianjia Zhang unsigned int key_len)
614*6b5360a5STianjia Zhang {
615*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
616*6b5360a5STianjia Zhang
617*6b5360a5STianjia Zhang if (key_len != SM4_KEY_SIZE)
618*6b5360a5STianjia Zhang return -EINVAL;
619*6b5360a5STianjia Zhang
620*6b5360a5STianjia Zhang kernel_neon_begin();
621*6b5360a5STianjia Zhang sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
622*6b5360a5STianjia Zhang crypto_sm4_fk, crypto_sm4_ck);
623*6b5360a5STianjia Zhang kernel_neon_end();
624*6b5360a5STianjia Zhang
625*6b5360a5STianjia Zhang return 0;
626*6b5360a5STianjia Zhang }
627*6b5360a5STianjia Zhang
sm4_cmac_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int key_len)628*6b5360a5STianjia Zhang static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key,
629*6b5360a5STianjia Zhang unsigned int key_len)
630*6b5360a5STianjia Zhang {
631*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
632*6b5360a5STianjia Zhang be128 *consts = (be128 *)ctx->consts;
633*6b5360a5STianjia Zhang u64 a, b;
634*6b5360a5STianjia Zhang
635*6b5360a5STianjia Zhang if (key_len != SM4_KEY_SIZE)
636*6b5360a5STianjia Zhang return -EINVAL;
637*6b5360a5STianjia Zhang
638*6b5360a5STianjia Zhang memset(consts, 0, SM4_BLOCK_SIZE);
639*6b5360a5STianjia Zhang
640*6b5360a5STianjia Zhang kernel_neon_begin();
641*6b5360a5STianjia Zhang
642*6b5360a5STianjia Zhang sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
643*6b5360a5STianjia Zhang crypto_sm4_fk, crypto_sm4_ck);
644*6b5360a5STianjia Zhang
645*6b5360a5STianjia Zhang /* encrypt the zero block */
646*6b5360a5STianjia Zhang sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts);
647*6b5360a5STianjia Zhang
648*6b5360a5STianjia Zhang kernel_neon_end();
649*6b5360a5STianjia Zhang
650*6b5360a5STianjia Zhang /* gf(2^128) multiply zero-ciphertext with u and u^2 */
651*6b5360a5STianjia Zhang a = be64_to_cpu(consts[0].a);
652*6b5360a5STianjia Zhang b = be64_to_cpu(consts[0].b);
653*6b5360a5STianjia Zhang consts[0].a = cpu_to_be64((a << 1) | (b >> 63));
654*6b5360a5STianjia Zhang consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
655*6b5360a5STianjia Zhang
656*6b5360a5STianjia Zhang a = be64_to_cpu(consts[0].a);
657*6b5360a5STianjia Zhang b = be64_to_cpu(consts[0].b);
658*6b5360a5STianjia Zhang consts[1].a = cpu_to_be64((a << 1) | (b >> 63));
659*6b5360a5STianjia Zhang consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
660*6b5360a5STianjia Zhang
661*6b5360a5STianjia Zhang return 0;
662*6b5360a5STianjia Zhang }
663*6b5360a5STianjia Zhang
sm4_xcbc_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int key_len)664*6b5360a5STianjia Zhang static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key,
665*6b5360a5STianjia Zhang unsigned int key_len)
666*6b5360a5STianjia Zhang {
667*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
668*6b5360a5STianjia Zhang u8 __aligned(8) key2[SM4_BLOCK_SIZE];
669*6b5360a5STianjia Zhang static u8 const ks[3][SM4_BLOCK_SIZE] = {
670*6b5360a5STianjia Zhang { [0 ... SM4_BLOCK_SIZE - 1] = 0x1},
671*6b5360a5STianjia Zhang { [0 ... SM4_BLOCK_SIZE - 1] = 0x2},
672*6b5360a5STianjia Zhang { [0 ... SM4_BLOCK_SIZE - 1] = 0x3},
673*6b5360a5STianjia Zhang };
674*6b5360a5STianjia Zhang
675*6b5360a5STianjia Zhang if (key_len != SM4_KEY_SIZE)
676*6b5360a5STianjia Zhang return -EINVAL;
677*6b5360a5STianjia Zhang
678*6b5360a5STianjia Zhang kernel_neon_begin();
679*6b5360a5STianjia Zhang
680*6b5360a5STianjia Zhang sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
681*6b5360a5STianjia Zhang crypto_sm4_fk, crypto_sm4_ck);
682*6b5360a5STianjia Zhang
683*6b5360a5STianjia Zhang sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]);
684*6b5360a5STianjia Zhang sm4_ce_crypt(ctx->key.rkey_enc, ctx->consts, ks[1], 2);
685*6b5360a5STianjia Zhang
686*6b5360a5STianjia Zhang sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec,
687*6b5360a5STianjia Zhang crypto_sm4_fk, crypto_sm4_ck);
688*6b5360a5STianjia Zhang
689*6b5360a5STianjia Zhang kernel_neon_end();
690*6b5360a5STianjia Zhang
691*6b5360a5STianjia Zhang return 0;
692*6b5360a5STianjia Zhang }
693*6b5360a5STianjia Zhang
sm4_mac_init(struct shash_desc * desc)694*6b5360a5STianjia Zhang static int sm4_mac_init(struct shash_desc *desc)
695*6b5360a5STianjia Zhang {
696*6b5360a5STianjia Zhang struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
697*6b5360a5STianjia Zhang
698*6b5360a5STianjia Zhang memset(ctx->digest, 0, SM4_BLOCK_SIZE);
699*6b5360a5STianjia Zhang ctx->len = 0;
700*6b5360a5STianjia Zhang
701*6b5360a5STianjia Zhang return 0;
702*6b5360a5STianjia Zhang }
703*6b5360a5STianjia Zhang
sm4_mac_update(struct shash_desc * desc,const u8 * p,unsigned int len)704*6b5360a5STianjia Zhang static int sm4_mac_update(struct shash_desc *desc, const u8 *p,
705*6b5360a5STianjia Zhang unsigned int len)
706*6b5360a5STianjia Zhang {
707*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
708*6b5360a5STianjia Zhang struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
709*6b5360a5STianjia Zhang unsigned int l, nblocks;
710*6b5360a5STianjia Zhang
711*6b5360a5STianjia Zhang if (len == 0)
712*6b5360a5STianjia Zhang return 0;
713*6b5360a5STianjia Zhang
714*6b5360a5STianjia Zhang if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) {
715*6b5360a5STianjia Zhang l = min(len, SM4_BLOCK_SIZE - ctx->len);
716*6b5360a5STianjia Zhang
717*6b5360a5STianjia Zhang crypto_xor(ctx->digest + ctx->len, p, l);
718*6b5360a5STianjia Zhang ctx->len += l;
719*6b5360a5STianjia Zhang len -= l;
720*6b5360a5STianjia Zhang p += l;
721*6b5360a5STianjia Zhang }
722*6b5360a5STianjia Zhang
723*6b5360a5STianjia Zhang if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) {
724*6b5360a5STianjia Zhang kernel_neon_begin();
725*6b5360a5STianjia Zhang
726*6b5360a5STianjia Zhang if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) {
727*6b5360a5STianjia Zhang sm4_ce_crypt_block(tctx->key.rkey_enc,
728*6b5360a5STianjia Zhang ctx->digest, ctx->digest);
729*6b5360a5STianjia Zhang ctx->len = 0;
730*6b5360a5STianjia Zhang } else {
731*6b5360a5STianjia Zhang nblocks = len / SM4_BLOCK_SIZE;
732*6b5360a5STianjia Zhang len %= SM4_BLOCK_SIZE;
733*6b5360a5STianjia Zhang
734*6b5360a5STianjia Zhang sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p,
735*6b5360a5STianjia Zhang nblocks, (ctx->len == SM4_BLOCK_SIZE),
736*6b5360a5STianjia Zhang (len != 0));
737*6b5360a5STianjia Zhang
738*6b5360a5STianjia Zhang p += nblocks * SM4_BLOCK_SIZE;
739*6b5360a5STianjia Zhang
740*6b5360a5STianjia Zhang if (len == 0)
741*6b5360a5STianjia Zhang ctx->len = SM4_BLOCK_SIZE;
742*6b5360a5STianjia Zhang }
743*6b5360a5STianjia Zhang
744*6b5360a5STianjia Zhang kernel_neon_end();
745*6b5360a5STianjia Zhang
746*6b5360a5STianjia Zhang if (len) {
747*6b5360a5STianjia Zhang crypto_xor(ctx->digest, p, len);
748*6b5360a5STianjia Zhang ctx->len = len;
749*6b5360a5STianjia Zhang }
750*6b5360a5STianjia Zhang }
751*6b5360a5STianjia Zhang
752*6b5360a5STianjia Zhang return 0;
753*6b5360a5STianjia Zhang }
754*6b5360a5STianjia Zhang
sm4_cmac_final(struct shash_desc * desc,u8 * out)755*6b5360a5STianjia Zhang static int sm4_cmac_final(struct shash_desc *desc, u8 *out)
756*6b5360a5STianjia Zhang {
757*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
758*6b5360a5STianjia Zhang struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
759*6b5360a5STianjia Zhang const u8 *consts = tctx->consts;
760*6b5360a5STianjia Zhang
761*6b5360a5STianjia Zhang if (ctx->len != SM4_BLOCK_SIZE) {
762*6b5360a5STianjia Zhang ctx->digest[ctx->len] ^= 0x80;
763*6b5360a5STianjia Zhang consts += SM4_BLOCK_SIZE;
764*6b5360a5STianjia Zhang }
765*6b5360a5STianjia Zhang
766*6b5360a5STianjia Zhang kernel_neon_begin();
767*6b5360a5STianjia Zhang sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1,
768*6b5360a5STianjia Zhang false, true);
769*6b5360a5STianjia Zhang kernel_neon_end();
770*6b5360a5STianjia Zhang
771*6b5360a5STianjia Zhang memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
772*6b5360a5STianjia Zhang
773*6b5360a5STianjia Zhang return 0;
774*6b5360a5STianjia Zhang }
775*6b5360a5STianjia Zhang
sm4_cbcmac_final(struct shash_desc * desc,u8 * out)776*6b5360a5STianjia Zhang static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out)
777*6b5360a5STianjia Zhang {
778*6b5360a5STianjia Zhang struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
779*6b5360a5STianjia Zhang struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
780*6b5360a5STianjia Zhang
781*6b5360a5STianjia Zhang if (ctx->len) {
782*6b5360a5STianjia Zhang kernel_neon_begin();
783*6b5360a5STianjia Zhang sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest,
784*6b5360a5STianjia Zhang ctx->digest);
785*6b5360a5STianjia Zhang kernel_neon_end();
786*6b5360a5STianjia Zhang }
787*6b5360a5STianjia Zhang
788*6b5360a5STianjia Zhang memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
789*6b5360a5STianjia Zhang
790*6b5360a5STianjia Zhang return 0;
791*6b5360a5STianjia Zhang }
792*6b5360a5STianjia Zhang
793*6b5360a5STianjia Zhang static struct shash_alg sm4_mac_algs[] = {
794*6b5360a5STianjia Zhang {
795*6b5360a5STianjia Zhang .base = {
796*6b5360a5STianjia Zhang .cra_name = "cmac(sm4)",
797*6b5360a5STianjia Zhang .cra_driver_name = "cmac-sm4-ce",
798*6b5360a5STianjia Zhang .cra_priority = 400,
799*6b5360a5STianjia Zhang .cra_blocksize = SM4_BLOCK_SIZE,
800*6b5360a5STianjia Zhang .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
801*6b5360a5STianjia Zhang + SM4_BLOCK_SIZE * 2,
802*6b5360a5STianjia Zhang .cra_module = THIS_MODULE,
803*6b5360a5STianjia Zhang },
804*6b5360a5STianjia Zhang .digestsize = SM4_BLOCK_SIZE,
805*6b5360a5STianjia Zhang .init = sm4_mac_init,
806*6b5360a5STianjia Zhang .update = sm4_mac_update,
807*6b5360a5STianjia Zhang .final = sm4_cmac_final,
808*6b5360a5STianjia Zhang .setkey = sm4_cmac_setkey,
809*6b5360a5STianjia Zhang .descsize = sizeof(struct sm4_mac_desc_ctx),
810*6b5360a5STianjia Zhang }, {
811*6b5360a5STianjia Zhang .base = {
812*6b5360a5STianjia Zhang .cra_name = "xcbc(sm4)",
813*6b5360a5STianjia Zhang .cra_driver_name = "xcbc-sm4-ce",
814*6b5360a5STianjia Zhang .cra_priority = 400,
815*6b5360a5STianjia Zhang .cra_blocksize = SM4_BLOCK_SIZE,
816*6b5360a5STianjia Zhang .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
817*6b5360a5STianjia Zhang + SM4_BLOCK_SIZE * 2,
818*6b5360a5STianjia Zhang .cra_module = THIS_MODULE,
819*6b5360a5STianjia Zhang },
820*6b5360a5STianjia Zhang .digestsize = SM4_BLOCK_SIZE,
821*6b5360a5STianjia Zhang .init = sm4_mac_init,
822*6b5360a5STianjia Zhang .update = sm4_mac_update,
823*6b5360a5STianjia Zhang .final = sm4_cmac_final,
824*6b5360a5STianjia Zhang .setkey = sm4_xcbc_setkey,
825*6b5360a5STianjia Zhang .descsize = sizeof(struct sm4_mac_desc_ctx),
826*6b5360a5STianjia Zhang }, {
827*6b5360a5STianjia Zhang .base = {
828*6b5360a5STianjia Zhang .cra_name = "cbcmac(sm4)",
829*6b5360a5STianjia Zhang .cra_driver_name = "cbcmac-sm4-ce",
830*6b5360a5STianjia Zhang .cra_priority = 400,
831*6b5360a5STianjia Zhang .cra_blocksize = 1,
832*6b5360a5STianjia Zhang .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx),
833*6b5360a5STianjia Zhang .cra_module = THIS_MODULE,
834*6b5360a5STianjia Zhang },
835*6b5360a5STianjia Zhang .digestsize = SM4_BLOCK_SIZE,
836*6b5360a5STianjia Zhang .init = sm4_mac_init,
837*6b5360a5STianjia Zhang .update = sm4_mac_update,
838*6b5360a5STianjia Zhang .final = sm4_cbcmac_final,
839*6b5360a5STianjia Zhang .setkey = sm4_cbcmac_setkey,
840*6b5360a5STianjia Zhang .descsize = sizeof(struct sm4_mac_desc_ctx),
841*6b5360a5STianjia Zhang }
842*6b5360a5STianjia Zhang };
843*6b5360a5STianjia Zhang
sm4_init(void)8445b33e0ecSTianjia Zhang static int __init sm4_init(void)
8455b33e0ecSTianjia Zhang {
846*6b5360a5STianjia Zhang int err;
847*6b5360a5STianjia Zhang
848*6b5360a5STianjia Zhang err = crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
849*6b5360a5STianjia Zhang if (err)
850*6b5360a5STianjia Zhang return err;
851*6b5360a5STianjia Zhang
852*6b5360a5STianjia Zhang err = crypto_register_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
853*6b5360a5STianjia Zhang if (err)
854*6b5360a5STianjia Zhang goto out_err;
855*6b5360a5STianjia Zhang
856*6b5360a5STianjia Zhang return 0;
857*6b5360a5STianjia Zhang
858*6b5360a5STianjia Zhang out_err:
859*6b5360a5STianjia Zhang crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
860*6b5360a5STianjia Zhang return err;
8615b33e0ecSTianjia Zhang }
8625b33e0ecSTianjia Zhang
sm4_exit(void)8635b33e0ecSTianjia Zhang static void __exit sm4_exit(void)
8645b33e0ecSTianjia Zhang {
865*6b5360a5STianjia Zhang crypto_unregister_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
8665b33e0ecSTianjia Zhang crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
8675b33e0ecSTianjia Zhang }
8685b33e0ecSTianjia Zhang
8695b33e0ecSTianjia Zhang module_cpu_feature_match(SM4, sm4_init);
8705b33e0ecSTianjia Zhang module_exit(sm4_exit);
8715b33e0ecSTianjia Zhang
87201f63311STianjia Zhang MODULE_DESCRIPTION("SM4 ECB/CBC/CFB/CTR/XTS using ARMv8 Crypto Extensions");
8735b33e0ecSTianjia Zhang MODULE_ALIAS_CRYPTO("sm4-ce");
8745b33e0ecSTianjia Zhang MODULE_ALIAS_CRYPTO("sm4");
8755b33e0ecSTianjia Zhang MODULE_ALIAS_CRYPTO("ecb(sm4)");
8765b33e0ecSTianjia Zhang MODULE_ALIAS_CRYPTO("cbc(sm4)");
8775b33e0ecSTianjia Zhang MODULE_ALIAS_CRYPTO("cfb(sm4)");
8785b33e0ecSTianjia Zhang MODULE_ALIAS_CRYPTO("ctr(sm4)");
879b1863fd0STianjia Zhang MODULE_ALIAS_CRYPTO("cts(cbc(sm4))");
88001f63311STianjia Zhang MODULE_ALIAS_CRYPTO("xts(sm4)");
881*6b5360a5STianjia Zhang MODULE_ALIAS_CRYPTO("cmac(sm4)");
882*6b5360a5STianjia Zhang MODULE_ALIAS_CRYPTO("xcbc(sm4)");
883*6b5360a5STianjia Zhang MODULE_ALIAS_CRYPTO("cbcmac(sm4)");
8845b33e0ecSTianjia Zhang MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
8855b33e0ecSTianjia Zhang MODULE_LICENSE("GPL v2");
886