1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * PowerPC P10 (ppc64le) accelerated ChaCha and XChaCha stream ciphers,
4  * including ChaCha20 (RFC7539)
5  *
6  * Copyright 2023- IBM Corp. All rights reserved.
7  */
8 
9 #include <crypto/algapi.h>
10 #include <crypto/internal/chacha.h>
11 #include <crypto/internal/simd.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/cpufeature.h>
16 #include <linux/sizes.h>
17 #include <asm/simd.h>
18 #include <asm/switch_to.h>
19 
20 asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src,
21 				unsigned int len, int nrounds);
22 
23 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
24 
25 static void vsx_begin(void)
26 {
27 	preempt_disable();
28 	enable_kernel_vsx();
29 }
30 
31 static void vsx_end(void)
32 {
33 	disable_kernel_vsx();
34 	preempt_enable();
35 }
36 
37 static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src,
38 			     unsigned int bytes, int nrounds)
39 {
40 	unsigned int l = bytes & ~0x0FF;
41 
42 	if (l > 0) {
43 		chacha_p10le_8x(state, dst, src, l, nrounds);
44 		bytes -= l;
45 		src += l;
46 		dst += l;
47 		state[12] += l / CHACHA_BLOCK_SIZE;
48 	}
49 
50 	if (bytes > 0)
51 		chacha_crypt_generic(state, dst, src, bytes, nrounds);
52 }
53 
54 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
55 {
56 	hchacha_block_generic(state, stream, nrounds);
57 }
58 EXPORT_SYMBOL(hchacha_block_arch);
59 
60 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
61 {
62 	chacha_init_generic(state, key, iv);
63 }
64 EXPORT_SYMBOL(chacha_init_arch);
65 
66 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
67 		       int nrounds)
68 {
69 	if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
70 	    !crypto_simd_usable())
71 		return chacha_crypt_generic(state, dst, src, bytes, nrounds);
72 
73 	do {
74 		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
75 
76 		vsx_begin();
77 		chacha_p10_do_8x(state, dst, src, todo, nrounds);
78 		vsx_end();
79 
80 		bytes -= todo;
81 		src += todo;
82 		dst += todo;
83 	} while (bytes);
84 }
85 EXPORT_SYMBOL(chacha_crypt_arch);
86 
87 static int chacha_p10_stream_xor(struct skcipher_request *req,
88 				 const struct chacha_ctx *ctx, const u8 *iv)
89 {
90 	struct skcipher_walk walk;
91 	u32 state[16];
92 	int err;
93 
94 	err = skcipher_walk_virt(&walk, req, false);
95 	if (err)
96 		return err;
97 
98 	chacha_init_generic(state, ctx->key, iv);
99 
100 	while (walk.nbytes > 0) {
101 		unsigned int nbytes = walk.nbytes;
102 
103 		if (nbytes < walk.total)
104 			nbytes = rounddown(nbytes, walk.stride);
105 
106 		if (!crypto_simd_usable()) {
107 			chacha_crypt_generic(state, walk.dst.virt.addr,
108 					     walk.src.virt.addr, nbytes,
109 					     ctx->nrounds);
110 		} else {
111 			vsx_begin();
112 			chacha_p10_do_8x(state, walk.dst.virt.addr,
113 				      walk.src.virt.addr, nbytes, ctx->nrounds);
114 			vsx_end();
115 		}
116 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
117 		if (err)
118 			break;
119 	}
120 
121 	return err;
122 }
123 
124 static int chacha_p10(struct skcipher_request *req)
125 {
126 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
127 	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
128 
129 	return chacha_p10_stream_xor(req, ctx, req->iv);
130 }
131 
132 static int xchacha_p10(struct skcipher_request *req)
133 {
134 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
135 	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
136 	struct chacha_ctx subctx;
137 	u32 state[16];
138 	u8 real_iv[16];
139 
140 	chacha_init_generic(state, ctx->key, req->iv);
141 	hchacha_block_arch(state, subctx.key, ctx->nrounds);
142 	subctx.nrounds = ctx->nrounds;
143 
144 	memcpy(&real_iv[0], req->iv + 24, 8);
145 	memcpy(&real_iv[8], req->iv + 16, 8);
146 	return chacha_p10_stream_xor(req, &subctx, real_iv);
147 }
148 
149 static struct skcipher_alg algs[] = {
150 	{
151 		.base.cra_name		= "chacha20",
152 		.base.cra_driver_name	= "chacha20-p10",
153 		.base.cra_priority	= 300,
154 		.base.cra_blocksize	= 1,
155 		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
156 		.base.cra_module	= THIS_MODULE,
157 
158 		.min_keysize		= CHACHA_KEY_SIZE,
159 		.max_keysize		= CHACHA_KEY_SIZE,
160 		.ivsize			= CHACHA_IV_SIZE,
161 		.chunksize		= CHACHA_BLOCK_SIZE,
162 		.setkey			= chacha20_setkey,
163 		.encrypt		= chacha_p10,
164 		.decrypt		= chacha_p10,
165 	}, {
166 		.base.cra_name		= "xchacha20",
167 		.base.cra_driver_name	= "xchacha20-p10",
168 		.base.cra_priority	= 300,
169 		.base.cra_blocksize	= 1,
170 		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
171 		.base.cra_module	= THIS_MODULE,
172 
173 		.min_keysize		= CHACHA_KEY_SIZE,
174 		.max_keysize		= CHACHA_KEY_SIZE,
175 		.ivsize			= XCHACHA_IV_SIZE,
176 		.chunksize		= CHACHA_BLOCK_SIZE,
177 		.setkey			= chacha20_setkey,
178 		.encrypt		= xchacha_p10,
179 		.decrypt		= xchacha_p10,
180 	}, {
181 		.base.cra_name		= "xchacha12",
182 		.base.cra_driver_name	= "xchacha12-p10",
183 		.base.cra_priority	= 300,
184 		.base.cra_blocksize	= 1,
185 		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
186 		.base.cra_module	= THIS_MODULE,
187 
188 		.min_keysize		= CHACHA_KEY_SIZE,
189 		.max_keysize		= CHACHA_KEY_SIZE,
190 		.ivsize			= XCHACHA_IV_SIZE,
191 		.chunksize		= CHACHA_BLOCK_SIZE,
192 		.setkey			= chacha12_setkey,
193 		.encrypt		= xchacha_p10,
194 		.decrypt		= xchacha_p10,
195 	}
196 };
197 
198 static int __init chacha_p10_init(void)
199 {
200 	static_branch_enable(&have_p10);
201 
202 	return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
203 }
204 
205 static void __exit chacha_p10_exit(void)
206 {
207 	crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
208 }
209 
210 module_cpu_feature_match(PPC_MODULE_FEATURE_P10, chacha_p10_init);
211 module_exit(chacha_p10_exit);
212 
213 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)");
214 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
215 MODULE_LICENSE("GPL v2");
216 MODULE_ALIAS_CRYPTO("chacha20");
217 MODULE_ALIAS_CRYPTO("chacha20-p10");
218 MODULE_ALIAS_CRYPTO("xchacha20");
219 MODULE_ALIAS_CRYPTO("xchacha20-p10");
220 MODULE_ALIAS_CRYPTO("xchacha12");
221 MODULE_ALIAS_CRYPTO("xchacha12-p10");
222