1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/printk.h>
4 #include <linux/crypto.h>
5 #include <linux/rtnetlink.h>
6 
7 #include <crypto/aead.h>
8 #include <crypto/authenc.h>
9 #include <crypto/des.h>
10 #include <crypto/internal/aead.h>
11 #include <crypto/scatterwalk.h>
12 #include <crypto/gcm.h>
13 
14 #include "nitrox_dev.h"
15 #include "nitrox_common.h"
16 #include "nitrox_req.h"
17 
18 #define GCM_AES_SALT_SIZE	4
19 
20 union gph_p3 {
21 	struct {
22 #ifdef __BIG_ENDIAN_BITFIELD
23 		u16 iv_offset : 8;
24 		u16 auth_offset	: 8;
25 #else
26 		u16 auth_offset	: 8;
27 		u16 iv_offset : 8;
28 #endif
29 	};
30 	u16 param;
31 };
32 
nitrox_aes_gcm_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)33 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
34 				 unsigned int keylen)
35 {
36 	int aes_keylen;
37 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
38 	struct flexi_crypto_context *fctx;
39 	union fc_ctx_flags flags;
40 
41 	aes_keylen = flexi_aes_keylen(keylen);
42 	if (aes_keylen < 0)
43 		return -EINVAL;
44 
45 	/* fill crypto context */
46 	fctx = nctx->u.fctx;
47 	flags.fu = be64_to_cpu(fctx->flags.f);
48 	flags.w0.aes_keylen = aes_keylen;
49 	fctx->flags.f = cpu_to_be64(flags.fu);
50 
51 	/* copy enc key to context */
52 	memset(&fctx->crypto, 0, sizeof(fctx->crypto));
53 	memcpy(fctx->crypto.u.key, key, keylen);
54 
55 	return 0;
56 }
57 
nitrox_aead_setauthsize(struct crypto_aead * aead,unsigned int authsize)58 static int nitrox_aead_setauthsize(struct crypto_aead *aead,
59 				   unsigned int authsize)
60 {
61 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
62 	struct flexi_crypto_context *fctx = nctx->u.fctx;
63 	union fc_ctx_flags flags;
64 
65 	flags.fu = be64_to_cpu(fctx->flags.f);
66 	flags.w0.mac_len = authsize;
67 	fctx->flags.f = cpu_to_be64(flags.fu);
68 
69 	aead->authsize = authsize;
70 
71 	return 0;
72 }
73 
nitrox_aes_gcm_setauthsize(struct crypto_aead * aead,unsigned int authsize)74 static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
75 				      unsigned int authsize)
76 {
77 	switch (authsize) {
78 	case 4:
79 	case 8:
80 	case 12:
81 	case 13:
82 	case 14:
83 	case 15:
84 	case 16:
85 		break;
86 	default:
87 		return -EINVAL;
88 	}
89 
90 	return nitrox_aead_setauthsize(aead, authsize);
91 }
92 
alloc_src_sglist(struct nitrox_kcrypt_request * nkreq,struct scatterlist * src,char * iv,int ivsize,int buflen)93 static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
94 			    struct scatterlist *src, char *iv, int ivsize,
95 			    int buflen)
96 {
97 	int nents = sg_nents_for_len(src, buflen);
98 	int ret;
99 
100 	if (nents < 0)
101 		return nents;
102 
103 	/* IV entry */
104 	nents += 1;
105 	/* Allocate buffer to hold IV and input scatterlist array */
106 	ret = alloc_src_req_buf(nkreq, nents, ivsize);
107 	if (ret)
108 		return ret;
109 
110 	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
111 	nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
112 
113 	return 0;
114 }
115 
alloc_dst_sglist(struct nitrox_kcrypt_request * nkreq,struct scatterlist * dst,int ivsize,int buflen)116 static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
117 			    struct scatterlist *dst, int ivsize, int buflen)
118 {
119 	int nents = sg_nents_for_len(dst, buflen);
120 	int ret;
121 
122 	if (nents < 0)
123 		return nents;
124 
125 	/* IV, ORH, COMPLETION entries */
126 	nents += 3;
127 	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
128 	 * array
129 	 */
130 	ret = alloc_dst_req_buf(nkreq, nents);
131 	if (ret)
132 		return ret;
133 
134 	nitrox_creq_set_orh(nkreq);
135 	nitrox_creq_set_comp(nkreq);
136 	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
137 
138 	return 0;
139 }
140 
free_src_sglist(struct nitrox_kcrypt_request * nkreq)141 static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
142 {
143 	kfree(nkreq->src);
144 }
145 
free_dst_sglist(struct nitrox_kcrypt_request * nkreq)146 static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
147 {
148 	kfree(nkreq->dst);
149 }
150 
nitrox_set_creq(struct nitrox_aead_rctx * rctx)151 static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
152 {
153 	struct se_crypto_request *creq = &rctx->nkreq.creq;
154 	union gph_p3 param3;
155 	int ret;
156 
157 	creq->flags = rctx->flags;
158 	creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
159 							       GFP_ATOMIC;
160 
161 	creq->ctrl.value = 0;
162 	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
163 	creq->ctrl.s.arg = rctx->ctrl_arg;
164 
165 	creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
166 	creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
167 	creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
168 	param3.iv_offset = 0;
169 	param3.auth_offset = rctx->ivsize;
170 	creq->gph.param3 = cpu_to_be16(param3.param);
171 
172 	creq->ctx_handle = rctx->ctx_handle;
173 	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
174 
175 	ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
176 			       rctx->srclen);
177 	if (ret)
178 		return ret;
179 
180 	ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
181 			       rctx->dstlen);
182 	if (ret) {
183 		free_src_sglist(&rctx->nkreq);
184 		return ret;
185 	}
186 
187 	return 0;
188 }
189 
nitrox_aead_callback(void * arg,int err)190 static void nitrox_aead_callback(void *arg, int err)
191 {
192 	struct aead_request *areq = arg;
193 	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
194 
195 	free_src_sglist(&rctx->nkreq);
196 	free_dst_sglist(&rctx->nkreq);
197 	if (err) {
198 		pr_err_ratelimited("request failed status 0x%0x\n", err);
199 		err = -EINVAL;
200 	}
201 
202 	aead_request_complete(areq, err);
203 }
204 
nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)205 static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
206 {
207 	if (assoclen <= 512)
208 		return true;
209 
210 	return false;
211 }
212 
nitrox_aes_gcm_enc(struct aead_request * areq)213 static int nitrox_aes_gcm_enc(struct aead_request *areq)
214 {
215 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
216 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
217 	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
218 	struct se_crypto_request *creq = &rctx->nkreq.creq;
219 	struct flexi_crypto_context *fctx = nctx->u.fctx;
220 	int ret;
221 
222 	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
223 		return -EINVAL;
224 
225 	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
226 
227 	rctx->cryptlen = areq->cryptlen;
228 	rctx->assoclen = areq->assoclen;
229 	rctx->srclen = areq->assoclen + areq->cryptlen;
230 	rctx->dstlen = rctx->srclen + aead->authsize;
231 	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
232 	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
233 	rctx->flags = areq->base.flags;
234 	rctx->ctx_handle = nctx->u.ctx_handle;
235 	rctx->src = areq->src;
236 	rctx->dst = areq->dst;
237 	rctx->ctrl_arg = ENCRYPT;
238 	ret = nitrox_set_creq(rctx);
239 	if (ret)
240 		return ret;
241 
242 	/* send the crypto request */
243 	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
244 					 areq);
245 }
246 
nitrox_aes_gcm_dec(struct aead_request * areq)247 static int nitrox_aes_gcm_dec(struct aead_request *areq)
248 {
249 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
250 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
251 	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
252 	struct se_crypto_request *creq = &rctx->nkreq.creq;
253 	struct flexi_crypto_context *fctx = nctx->u.fctx;
254 	int ret;
255 
256 	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
257 		return -EINVAL;
258 
259 	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
260 
261 	rctx->cryptlen = areq->cryptlen - aead->authsize;
262 	rctx->assoclen = areq->assoclen;
263 	rctx->srclen = areq->cryptlen + areq->assoclen;
264 	rctx->dstlen = rctx->srclen - aead->authsize;
265 	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
266 	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
267 	rctx->flags = areq->base.flags;
268 	rctx->ctx_handle = nctx->u.ctx_handle;
269 	rctx->src = areq->src;
270 	rctx->dst = areq->dst;
271 	rctx->ctrl_arg = DECRYPT;
272 	ret = nitrox_set_creq(rctx);
273 	if (ret)
274 		return ret;
275 
276 	/* send the crypto request */
277 	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
278 					 areq);
279 }
280 
nitrox_aead_init(struct crypto_aead * aead)281 static int nitrox_aead_init(struct crypto_aead *aead)
282 {
283 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
284 	struct crypto_ctx_hdr *chdr;
285 
286 	/* get the first device */
287 	nctx->ndev = nitrox_get_first_device();
288 	if (!nctx->ndev)
289 		return -ENODEV;
290 
291 	/* allocate nitrox crypto context */
292 	chdr = crypto_alloc_context(nctx->ndev);
293 	if (!chdr) {
294 		nitrox_put_device(nctx->ndev);
295 		return -ENOMEM;
296 	}
297 	nctx->chdr = chdr;
298 	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
299 					 sizeof(struct ctx_hdr));
300 	nctx->u.fctx->flags.f = 0;
301 
302 	return 0;
303 }
304 
nitrox_gcm_common_init(struct crypto_aead * aead)305 static int nitrox_gcm_common_init(struct crypto_aead *aead)
306 {
307 	int ret;
308 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
309 	union fc_ctx_flags *flags;
310 
311 	ret = nitrox_aead_init(aead);
312 	if (ret)
313 		return ret;
314 
315 	flags = &nctx->u.fctx->flags;
316 	flags->w0.cipher_type = CIPHER_AES_GCM;
317 	flags->w0.hash_type = AUTH_NULL;
318 	flags->w0.iv_source = IV_FROM_DPTR;
319 	/* ask microcode to calculate ipad/opad */
320 	flags->w0.auth_input_type = 1;
321 	flags->f = cpu_to_be64(flags->fu);
322 
323 	return 0;
324 }
325 
nitrox_aes_gcm_init(struct crypto_aead * aead)326 static int nitrox_aes_gcm_init(struct crypto_aead *aead)
327 {
328 	int ret;
329 
330 	ret = nitrox_gcm_common_init(aead);
331 	if (ret)
332 		return ret;
333 
334 	crypto_aead_set_reqsize(aead,
335 				sizeof(struct aead_request) +
336 					sizeof(struct nitrox_aead_rctx));
337 
338 	return 0;
339 }
340 
nitrox_aead_exit(struct crypto_aead * aead)341 static void nitrox_aead_exit(struct crypto_aead *aead)
342 {
343 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
344 
345 	/* free the nitrox crypto context */
346 	if (nctx->u.ctx_handle) {
347 		struct flexi_crypto_context *fctx = nctx->u.fctx;
348 
349 		memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
350 		memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
351 		crypto_free_context((void *)nctx->chdr);
352 	}
353 	nitrox_put_device(nctx->ndev);
354 
355 	nctx->u.ctx_handle = 0;
356 	nctx->ndev = NULL;
357 }
358 
nitrox_rfc4106_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)359 static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
360 				 unsigned int keylen)
361 {
362 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
363 	struct flexi_crypto_context *fctx = nctx->u.fctx;
364 	int ret;
365 
366 	if (keylen < GCM_AES_SALT_SIZE)
367 		return -EINVAL;
368 
369 	keylen -= GCM_AES_SALT_SIZE;
370 	ret = nitrox_aes_gcm_setkey(aead, key, keylen);
371 	if (ret)
372 		return ret;
373 
374 	memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
375 	return 0;
376 }
377 
nitrox_rfc4106_setauthsize(struct crypto_aead * aead,unsigned int authsize)378 static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
379 				      unsigned int authsize)
380 {
381 	switch (authsize) {
382 	case 8:
383 	case 12:
384 	case 16:
385 		break;
386 	default:
387 		return -EINVAL;
388 	}
389 
390 	return nitrox_aead_setauthsize(aead, authsize);
391 }
392 
nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request * areq)393 static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
394 {
395 	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
396 	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
397 	unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
398 	struct scatterlist *sg;
399 
400 	if (areq->assoclen != 16 && areq->assoclen != 20)
401 		return -EINVAL;
402 
403 	scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
404 	sg_init_table(rctx->src, 3);
405 	sg_set_buf(rctx->src, rctx->assoc, assoclen);
406 	sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
407 	if (sg != rctx->src + 1)
408 		sg_chain(rctx->src, 2, sg);
409 
410 	if (areq->src != areq->dst) {
411 		sg_init_table(rctx->dst, 3);
412 		sg_set_buf(rctx->dst, rctx->assoc, assoclen);
413 		sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
414 		if (sg != rctx->dst + 1)
415 			sg_chain(rctx->dst, 2, sg);
416 	}
417 
418 	aead_rctx->src = rctx->src;
419 	aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
420 
421 	return 0;
422 }
423 
nitrox_rfc4106_callback(void * arg,int err)424 static void nitrox_rfc4106_callback(void *arg, int err)
425 {
426 	struct aead_request *areq = arg;
427 	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
428 	struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
429 
430 	free_src_sglist(nkreq);
431 	free_dst_sglist(nkreq);
432 	if (err) {
433 		pr_err_ratelimited("request failed status 0x%0x\n", err);
434 		err = -EINVAL;
435 	}
436 
437 	aead_request_complete(areq, err);
438 }
439 
nitrox_rfc4106_enc(struct aead_request * areq)440 static int nitrox_rfc4106_enc(struct aead_request *areq)
441 {
442 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
443 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
444 	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
445 	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
446 	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
447 	int ret;
448 
449 	aead_rctx->cryptlen = areq->cryptlen;
450 	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
451 	aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
452 	aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
453 	aead_rctx->iv = areq->iv;
454 	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
455 	aead_rctx->flags = areq->base.flags;
456 	aead_rctx->ctx_handle = nctx->u.ctx_handle;
457 	aead_rctx->ctrl_arg = ENCRYPT;
458 
459 	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
460 	if (ret)
461 		return ret;
462 
463 	ret = nitrox_set_creq(aead_rctx);
464 	if (ret)
465 		return ret;
466 
467 	/* send the crypto request */
468 	return nitrox_process_se_request(nctx->ndev, creq,
469 					 nitrox_rfc4106_callback, areq);
470 }
471 
nitrox_rfc4106_dec(struct aead_request * areq)472 static int nitrox_rfc4106_dec(struct aead_request *areq)
473 {
474 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
475 	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead);
476 	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
477 	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
478 	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
479 	int ret;
480 
481 	aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
482 	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
483 	aead_rctx->srclen =
484 		areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
485 	aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
486 	aead_rctx->iv = areq->iv;
487 	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
488 	aead_rctx->flags = areq->base.flags;
489 	aead_rctx->ctx_handle = nctx->u.ctx_handle;
490 	aead_rctx->ctrl_arg = DECRYPT;
491 
492 	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
493 	if (ret)
494 		return ret;
495 
496 	ret = nitrox_set_creq(aead_rctx);
497 	if (ret)
498 		return ret;
499 
500 	/* send the crypto request */
501 	return nitrox_process_se_request(nctx->ndev, creq,
502 					 nitrox_rfc4106_callback, areq);
503 }
504 
nitrox_rfc4106_init(struct crypto_aead * aead)505 static int nitrox_rfc4106_init(struct crypto_aead *aead)
506 {
507 	int ret;
508 
509 	ret = nitrox_gcm_common_init(aead);
510 	if (ret)
511 		return ret;
512 
513 	crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) +
514 					  sizeof(struct nitrox_rfc4106_rctx));
515 
516 	return 0;
517 }
518 
519 static struct aead_alg nitrox_aeads[] = { {
520 	.base = {
521 		.cra_name = "gcm(aes)",
522 		.cra_driver_name = "n5_aes_gcm",
523 		.cra_priority = PRIO,
524 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
525 		.cra_blocksize = 1,
526 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
527 		.cra_alignmask = 0,
528 		.cra_module = THIS_MODULE,
529 	},
530 	.setkey = nitrox_aes_gcm_setkey,
531 	.setauthsize = nitrox_aes_gcm_setauthsize,
532 	.encrypt = nitrox_aes_gcm_enc,
533 	.decrypt = nitrox_aes_gcm_dec,
534 	.init = nitrox_aes_gcm_init,
535 	.exit = nitrox_aead_exit,
536 	.ivsize = GCM_AES_IV_SIZE,
537 	.maxauthsize = AES_BLOCK_SIZE,
538 }, {
539 	.base = {
540 		.cra_name = "rfc4106(gcm(aes))",
541 		.cra_driver_name = "n5_rfc4106",
542 		.cra_priority = PRIO,
543 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
544 		.cra_blocksize = 1,
545 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
546 		.cra_alignmask = 0,
547 		.cra_module = THIS_MODULE,
548 	},
549 	.setkey = nitrox_rfc4106_setkey,
550 	.setauthsize = nitrox_rfc4106_setauthsize,
551 	.encrypt = nitrox_rfc4106_enc,
552 	.decrypt = nitrox_rfc4106_dec,
553 	.init = nitrox_rfc4106_init,
554 	.exit = nitrox_aead_exit,
555 	.ivsize = GCM_RFC4106_IV_SIZE,
556 	.maxauthsize = AES_BLOCK_SIZE,
557 } };
558 
nitrox_register_aeads(void)559 int nitrox_register_aeads(void)
560 {
561 	return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
562 }
563 
nitrox_unregister_aeads(void)564 void nitrox_unregister_aeads(void)
565 {
566 	crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
567 }
568