xref: /openbmc/linux/crypto/pcrypt.c (revision 852a53a0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * pcrypt - Parallel crypto wrapper.
4  *
5  * Copyright (C) 2009 secunet Security Networks AG
6  * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
7  */
8 
9 #include <crypto/algapi.h>
10 #include <crypto/internal/aead.h>
11 #include <linux/atomic.h>
12 #include <linux/err.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/slab.h>
16 #include <linux/kobject.h>
17 #include <linux/cpu.h>
18 #include <crypto/pcrypt.h>
19 
20 static struct padata_instance *pencrypt;
21 static struct padata_instance *pdecrypt;
22 static struct kset           *pcrypt_kset;
23 
24 struct pcrypt_instance_ctx {
25 	struct crypto_aead_spawn spawn;
26 	struct padata_shell *psenc;
27 	struct padata_shell *psdec;
28 	atomic_t tfm_count;
29 };
30 
31 struct pcrypt_aead_ctx {
32 	struct crypto_aead *child;
33 	unsigned int cb_cpu;
34 };
35 
36 static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx(
37 	struct crypto_aead *tfm)
38 {
39 	return aead_instance_ctx(aead_alg_instance(tfm));
40 }
41 
42 static int pcrypt_aead_setkey(struct crypto_aead *parent,
43 			      const u8 *key, unsigned int keylen)
44 {
45 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
46 
47 	return crypto_aead_setkey(ctx->child, key, keylen);
48 }
49 
50 static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
51 				   unsigned int authsize)
52 {
53 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
54 
55 	return crypto_aead_setauthsize(ctx->child, authsize);
56 }
57 
58 static void pcrypt_aead_serial(struct padata_priv *padata)
59 {
60 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
61 	struct aead_request *req = pcrypt_request_ctx(preq);
62 
63 	aead_request_complete(req->base.data, padata->info);
64 }
65 
66 static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
67 {
68 	struct aead_request *req = areq->data;
69 	struct pcrypt_request *preq = aead_request_ctx(req);
70 	struct padata_priv *padata = pcrypt_request_padata(preq);
71 
72 	padata->info = err;
73 
74 	padata_do_serial(padata);
75 }
76 
77 static void pcrypt_aead_enc(struct padata_priv *padata)
78 {
79 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
80 	struct aead_request *req = pcrypt_request_ctx(preq);
81 
82 	padata->info = crypto_aead_encrypt(req);
83 
84 	if (padata->info == -EINPROGRESS)
85 		return;
86 
87 	padata_do_serial(padata);
88 }
89 
90 static int pcrypt_aead_encrypt(struct aead_request *req)
91 {
92 	int err;
93 	struct pcrypt_request *preq = aead_request_ctx(req);
94 	struct aead_request *creq = pcrypt_request_ctx(preq);
95 	struct padata_priv *padata = pcrypt_request_padata(preq);
96 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
97 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
98 	u32 flags = aead_request_flags(req);
99 	struct pcrypt_instance_ctx *ictx;
100 
101 	ictx = pcrypt_tfm_ictx(aead);
102 
103 	memset(padata, 0, sizeof(struct padata_priv));
104 
105 	padata->parallel = pcrypt_aead_enc;
106 	padata->serial = pcrypt_aead_serial;
107 
108 	aead_request_set_tfm(creq, ctx->child);
109 	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
110 				  pcrypt_aead_done, req);
111 	aead_request_set_crypt(creq, req->src, req->dst,
112 			       req->cryptlen, req->iv);
113 	aead_request_set_ad(creq, req->assoclen);
114 
115 	err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu);
116 	if (!err)
117 		return -EINPROGRESS;
118 
119 	return err;
120 }
121 
122 static void pcrypt_aead_dec(struct padata_priv *padata)
123 {
124 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
125 	struct aead_request *req = pcrypt_request_ctx(preq);
126 
127 	padata->info = crypto_aead_decrypt(req);
128 
129 	if (padata->info == -EINPROGRESS)
130 		return;
131 
132 	padata_do_serial(padata);
133 }
134 
135 static int pcrypt_aead_decrypt(struct aead_request *req)
136 {
137 	int err;
138 	struct pcrypt_request *preq = aead_request_ctx(req);
139 	struct aead_request *creq = pcrypt_request_ctx(preq);
140 	struct padata_priv *padata = pcrypt_request_padata(preq);
141 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
142 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
143 	u32 flags = aead_request_flags(req);
144 	struct pcrypt_instance_ctx *ictx;
145 
146 	ictx = pcrypt_tfm_ictx(aead);
147 
148 	memset(padata, 0, sizeof(struct padata_priv));
149 
150 	padata->parallel = pcrypt_aead_dec;
151 	padata->serial = pcrypt_aead_serial;
152 
153 	aead_request_set_tfm(creq, ctx->child);
154 	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
155 				  pcrypt_aead_done, req);
156 	aead_request_set_crypt(creq, req->src, req->dst,
157 			       req->cryptlen, req->iv);
158 	aead_request_set_ad(creq, req->assoclen);
159 
160 	err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu);
161 	if (!err)
162 		return -EINPROGRESS;
163 
164 	return err;
165 }
166 
167 static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
168 {
169 	int cpu, cpu_index;
170 	struct aead_instance *inst = aead_alg_instance(tfm);
171 	struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
172 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
173 	struct crypto_aead *cipher;
174 
175 	cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
176 		    cpumask_weight(cpu_online_mask);
177 
178 	ctx->cb_cpu = cpumask_first(cpu_online_mask);
179 	for (cpu = 0; cpu < cpu_index; cpu++)
180 		ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
181 
182 	cipher = crypto_spawn_aead(&ictx->spawn);
183 
184 	if (IS_ERR(cipher))
185 		return PTR_ERR(cipher);
186 
187 	ctx->child = cipher;
188 	crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) +
189 				     sizeof(struct aead_request) +
190 				     crypto_aead_reqsize(cipher));
191 
192 	return 0;
193 }
194 
195 static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm)
196 {
197 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
198 
199 	crypto_free_aead(ctx->child);
200 }
201 
202 static void pcrypt_free(struct aead_instance *inst)
203 {
204 	struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst);
205 
206 	crypto_drop_aead(&ctx->spawn);
207 	padata_free_shell(ctx->psdec);
208 	padata_free_shell(ctx->psenc);
209 	kfree(inst);
210 }
211 
212 static int pcrypt_init_instance(struct crypto_instance *inst,
213 				struct crypto_alg *alg)
214 {
215 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
216 		     "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
217 		return -ENAMETOOLONG;
218 
219 	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
220 
221 	inst->alg.cra_priority = alg->cra_priority + 100;
222 	inst->alg.cra_blocksize = alg->cra_blocksize;
223 	inst->alg.cra_alignmask = alg->cra_alignmask;
224 
225 	return 0;
226 }
227 
228 static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb,
229 			      struct crypto_attr_type *algt)
230 {
231 	struct pcrypt_instance_ctx *ctx;
232 	struct aead_instance *inst;
233 	struct aead_alg *alg;
234 	u32 mask = crypto_algt_inherited_mask(algt);
235 	int err;
236 
237 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
238 	if (!inst)
239 		return -ENOMEM;
240 
241 	err = -ENOMEM;
242 
243 	ctx = aead_instance_ctx(inst);
244 	ctx->psenc = padata_alloc_shell(pencrypt);
245 	if (!ctx->psenc)
246 		goto err_free_inst;
247 
248 	ctx->psdec = padata_alloc_shell(pdecrypt);
249 	if (!ctx->psdec)
250 		goto err_free_inst;
251 
252 	err = crypto_grab_aead(&ctx->spawn, aead_crypto_instance(inst),
253 			       crypto_attr_alg_name(tb[1]), 0, mask);
254 	if (err)
255 		goto err_free_inst;
256 
257 	alg = crypto_spawn_aead_alg(&ctx->spawn);
258 	err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base);
259 	if (err)
260 		goto err_free_inst;
261 
262 	inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC;
263 
264 	inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
265 	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
266 
267 	inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
268 
269 	inst->alg.init = pcrypt_aead_init_tfm;
270 	inst->alg.exit = pcrypt_aead_exit_tfm;
271 
272 	inst->alg.setkey = pcrypt_aead_setkey;
273 	inst->alg.setauthsize = pcrypt_aead_setauthsize;
274 	inst->alg.encrypt = pcrypt_aead_encrypt;
275 	inst->alg.decrypt = pcrypt_aead_decrypt;
276 
277 	inst->free = pcrypt_free;
278 
279 	err = aead_register_instance(tmpl, inst);
280 	if (err) {
281 err_free_inst:
282 		pcrypt_free(inst);
283 	}
284 	return err;
285 }
286 
287 static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb)
288 {
289 	struct crypto_attr_type *algt;
290 
291 	algt = crypto_get_attr_type(tb);
292 	if (IS_ERR(algt))
293 		return PTR_ERR(algt);
294 
295 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
296 	case CRYPTO_ALG_TYPE_AEAD:
297 		return pcrypt_create_aead(tmpl, tb, algt);
298 	}
299 
300 	return -EINVAL;
301 }
302 
303 static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
304 {
305 	int ret;
306 
307 	pinst->kobj.kset = pcrypt_kset;
308 	ret = kobject_add(&pinst->kobj, NULL, "%s", name);
309 	if (!ret)
310 		kobject_uevent(&pinst->kobj, KOBJ_ADD);
311 
312 	return ret;
313 }
314 
315 static int pcrypt_init_padata(struct padata_instance **pinst, const char *name)
316 {
317 	int ret = -ENOMEM;
318 
319 	*pinst = padata_alloc(name);
320 	if (!*pinst)
321 		return ret;
322 
323 	ret = pcrypt_sysfs_add(*pinst, name);
324 	if (ret)
325 		padata_free(*pinst);
326 
327 	return ret;
328 }
329 
330 static struct crypto_template pcrypt_tmpl = {
331 	.name = "pcrypt",
332 	.create = pcrypt_create,
333 	.module = THIS_MODULE,
334 };
335 
336 static int __init pcrypt_init(void)
337 {
338 	int err = -ENOMEM;
339 
340 	pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
341 	if (!pcrypt_kset)
342 		goto err;
343 
344 	err = pcrypt_init_padata(&pencrypt, "pencrypt");
345 	if (err)
346 		goto err_unreg_kset;
347 
348 	err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
349 	if (err)
350 		goto err_deinit_pencrypt;
351 
352 	return crypto_register_template(&pcrypt_tmpl);
353 
354 err_deinit_pencrypt:
355 	padata_free(pencrypt);
356 err_unreg_kset:
357 	kset_unregister(pcrypt_kset);
358 err:
359 	return err;
360 }
361 
362 static void __exit pcrypt_exit(void)
363 {
364 	crypto_unregister_template(&pcrypt_tmpl);
365 
366 	padata_free(pencrypt);
367 	padata_free(pdecrypt);
368 
369 	kset_unregister(pcrypt_kset);
370 }
371 
372 subsys_initcall(pcrypt_init);
373 module_exit(pcrypt_exit);
374 
375 MODULE_LICENSE("GPL");
376 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
377 MODULE_DESCRIPTION("Parallel crypto wrapper");
378 MODULE_ALIAS_CRYPTO("pcrypt");
379