1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * MIPS accelerated ChaCha and XChaCha stream ciphers,
4 * including ChaCha20 (RFC7539)
5 *
6 * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
7 */
8
9 #include <asm/byteorder.h>
10 #include <crypto/algapi.h>
11 #include <crypto/internal/chacha.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15
16 asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src,
17 unsigned int bytes, int nrounds);
18 EXPORT_SYMBOL(chacha_crypt_arch);
19
20 asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds);
21 EXPORT_SYMBOL(hchacha_block_arch);
22
chacha_init_arch(u32 * state,const u32 * key,const u8 * iv)23 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
24 {
25 chacha_init_generic(state, key, iv);
26 }
27 EXPORT_SYMBOL(chacha_init_arch);
28
chacha_mips_stream_xor(struct skcipher_request * req,const struct chacha_ctx * ctx,const u8 * iv)29 static int chacha_mips_stream_xor(struct skcipher_request *req,
30 const struct chacha_ctx *ctx, const u8 *iv)
31 {
32 struct skcipher_walk walk;
33 u32 state[16];
34 int err;
35
36 err = skcipher_walk_virt(&walk, req, false);
37
38 chacha_init_generic(state, ctx->key, iv);
39
40 while (walk.nbytes > 0) {
41 unsigned int nbytes = walk.nbytes;
42
43 if (nbytes < walk.total)
44 nbytes = round_down(nbytes, walk.stride);
45
46 chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr,
47 nbytes, ctx->nrounds);
48 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
49 }
50
51 return err;
52 }
53
chacha_mips(struct skcipher_request * req)54 static int chacha_mips(struct skcipher_request *req)
55 {
56 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
57 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
58
59 return chacha_mips_stream_xor(req, ctx, req->iv);
60 }
61
xchacha_mips(struct skcipher_request * req)62 static int xchacha_mips(struct skcipher_request *req)
63 {
64 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
65 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
66 struct chacha_ctx subctx;
67 u32 state[16];
68 u8 real_iv[16];
69
70 chacha_init_generic(state, ctx->key, req->iv);
71
72 hchacha_block(state, subctx.key, ctx->nrounds);
73 subctx.nrounds = ctx->nrounds;
74
75 memcpy(&real_iv[0], req->iv + 24, 8);
76 memcpy(&real_iv[8], req->iv + 16, 8);
77 return chacha_mips_stream_xor(req, &subctx, real_iv);
78 }
79
80 static struct skcipher_alg algs[] = {
81 {
82 .base.cra_name = "chacha20",
83 .base.cra_driver_name = "chacha20-mips",
84 .base.cra_priority = 200,
85 .base.cra_blocksize = 1,
86 .base.cra_ctxsize = sizeof(struct chacha_ctx),
87 .base.cra_module = THIS_MODULE,
88
89 .min_keysize = CHACHA_KEY_SIZE,
90 .max_keysize = CHACHA_KEY_SIZE,
91 .ivsize = CHACHA_IV_SIZE,
92 .chunksize = CHACHA_BLOCK_SIZE,
93 .setkey = chacha20_setkey,
94 .encrypt = chacha_mips,
95 .decrypt = chacha_mips,
96 }, {
97 .base.cra_name = "xchacha20",
98 .base.cra_driver_name = "xchacha20-mips",
99 .base.cra_priority = 200,
100 .base.cra_blocksize = 1,
101 .base.cra_ctxsize = sizeof(struct chacha_ctx),
102 .base.cra_module = THIS_MODULE,
103
104 .min_keysize = CHACHA_KEY_SIZE,
105 .max_keysize = CHACHA_KEY_SIZE,
106 .ivsize = XCHACHA_IV_SIZE,
107 .chunksize = CHACHA_BLOCK_SIZE,
108 .setkey = chacha20_setkey,
109 .encrypt = xchacha_mips,
110 .decrypt = xchacha_mips,
111 }, {
112 .base.cra_name = "xchacha12",
113 .base.cra_driver_name = "xchacha12-mips",
114 .base.cra_priority = 200,
115 .base.cra_blocksize = 1,
116 .base.cra_ctxsize = sizeof(struct chacha_ctx),
117 .base.cra_module = THIS_MODULE,
118
119 .min_keysize = CHACHA_KEY_SIZE,
120 .max_keysize = CHACHA_KEY_SIZE,
121 .ivsize = XCHACHA_IV_SIZE,
122 .chunksize = CHACHA_BLOCK_SIZE,
123 .setkey = chacha12_setkey,
124 .encrypt = xchacha_mips,
125 .decrypt = xchacha_mips,
126 }
127 };
128
chacha_simd_mod_init(void)129 static int __init chacha_simd_mod_init(void)
130 {
131 return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ?
132 crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0;
133 }
134
chacha_simd_mod_fini(void)135 static void __exit chacha_simd_mod_fini(void)
136 {
137 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER))
138 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
139 }
140
141 module_init(chacha_simd_mod_init);
142 module_exit(chacha_simd_mod_fini);
143
144 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)");
145 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
146 MODULE_LICENSE("GPL v2");
147 MODULE_ALIAS_CRYPTO("chacha20");
148 MODULE_ALIAS_CRYPTO("chacha20-mips");
149 MODULE_ALIAS_CRYPTO("xchacha20");
150 MODULE_ALIAS_CRYPTO("xchacha20-mips");
151 MODULE_ALIAS_CRYPTO("xchacha12");
152 MODULE_ALIAS_CRYPTO("xchacha12-mips");
153