1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/crypto.h>
3 #include <linux/kernel.h>
4 #include <linux/module.h>
5 #include <linux/printk.h>
6 
7 #include <crypto/aes.h>
8 #include <crypto/skcipher.h>
9 #include <crypto/ctr.h>
10 #include <crypto/des.h>
11 #include <crypto/xts.h>
12 
13 #include "nitrox_dev.h"
14 #include "nitrox_common.h"
15 #include "nitrox_req.h"
16 
17 #define PRIO 4001
18 
19 struct nitrox_cipher {
20 	const char *name;
21 	enum flexi_cipher value;
22 };
23 
24 /**
25  * supported cipher list
26  */
27 static const struct nitrox_cipher flexi_cipher_table[] = {
28 	{ "null",		CIPHER_NULL },
29 	{ "cbc(des3_ede)",	CIPHER_3DES_CBC },
30 	{ "ecb(des3_ede)",	CIPHER_3DES_ECB },
31 	{ "cbc(aes)",		CIPHER_AES_CBC },
32 	{ "ecb(aes)",		CIPHER_AES_ECB },
33 	{ "cfb(aes)",		CIPHER_AES_CFB },
34 	{ "rfc3686(ctr(aes))",	CIPHER_AES_CTR },
35 	{ "xts(aes)",		CIPHER_AES_XTS },
36 	{ "cts(cbc(aes))",	CIPHER_AES_CBC_CTS },
37 	{ NULL,			CIPHER_INVALID }
38 };
39 
40 static enum flexi_cipher flexi_cipher_type(const char *name)
41 {
42 	const struct nitrox_cipher *cipher = flexi_cipher_table;
43 
44 	while (cipher->name) {
45 		if (!strcmp(cipher->name, name))
46 			break;
47 		cipher++;
48 	}
49 	return cipher->value;
50 }
51 
52 static int flexi_aes_keylen(int keylen)
53 {
54 	int aes_keylen;
55 
56 	switch (keylen) {
57 	case AES_KEYSIZE_128:
58 		aes_keylen = 1;
59 		break;
60 	case AES_KEYSIZE_192:
61 		aes_keylen = 2;
62 		break;
63 	case AES_KEYSIZE_256:
64 		aes_keylen = 3;
65 		break;
66 	default:
67 		aes_keylen = -EINVAL;
68 		break;
69 	}
70 	return aes_keylen;
71 }
72 
73 static int nitrox_skcipher_init(struct crypto_skcipher *tfm)
74 {
75 	struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
76 	struct crypto_ctx_hdr *chdr;
77 
78 	/* get the first device */
79 	nctx->ndev = nitrox_get_first_device();
80 	if (!nctx->ndev)
81 		return -ENODEV;
82 
83 	/* allocate nitrox crypto context */
84 	chdr = crypto_alloc_context(nctx->ndev);
85 	if (!chdr) {
86 		nitrox_put_device(nctx->ndev);
87 		return -ENOMEM;
88 	}
89 	nctx->chdr = chdr;
90 	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
91 					 sizeof(struct ctx_hdr));
92 	crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) +
93 				    sizeof(struct nitrox_kcrypt_request));
94 	return 0;
95 }
96 
97 static void nitrox_skcipher_exit(struct crypto_skcipher *tfm)
98 {
99 	struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
100 
101 	/* free the nitrox crypto context */
102 	if (nctx->u.ctx_handle) {
103 		struct flexi_crypto_context *fctx = nctx->u.fctx;
104 
105 		memset(&fctx->crypto, 0, sizeof(struct crypto_keys));
106 		memset(&fctx->auth, 0, sizeof(struct auth_keys));
107 		crypto_free_context((void *)nctx->chdr);
108 	}
109 	nitrox_put_device(nctx->ndev);
110 
111 	nctx->u.ctx_handle = 0;
112 	nctx->ndev = NULL;
113 }
114 
115 static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher,
116 					 int aes_keylen, const u8 *key,
117 					 unsigned int keylen)
118 {
119 	struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
120 	struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
121 	struct flexi_crypto_context *fctx;
122 	enum flexi_cipher cipher_type;
123 	const char *name;
124 
125 	name = crypto_tfm_alg_name(tfm);
126 	cipher_type = flexi_cipher_type(name);
127 	if (unlikely(cipher_type == CIPHER_INVALID)) {
128 		pr_err("unsupported cipher: %s\n", name);
129 		return -EINVAL;
130 	}
131 
132 	/* fill crypto context */
133 	fctx = nctx->u.fctx;
134 	fctx->flags = 0;
135 	fctx->w0.cipher_type = cipher_type;
136 	fctx->w0.aes_keylen = aes_keylen;
137 	fctx->w0.iv_source = IV_FROM_DPTR;
138 	fctx->flags = cpu_to_be64(*(u64 *)&fctx->w0);
139 	/* copy the key to context */
140 	memcpy(fctx->crypto.u.key, key, keylen);
141 
142 	return 0;
143 }
144 
145 static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
146 			     unsigned int keylen)
147 {
148 	int aes_keylen;
149 
150 	aes_keylen = flexi_aes_keylen(keylen);
151 	if (aes_keylen < 0) {
152 		crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
153 		return -EINVAL;
154 	}
155 	return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
156 }
157 
158 static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize)
159 {
160 	struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
161 	int nents = sg_nents(skreq->src) + 1;
162 	struct se_crypto_request *creq = &nkreq->creq;
163 	char *iv;
164 	struct scatterlist *sg;
165 
166 	/* Allocate buffer to hold IV and input scatterlist array */
167 	nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
168 	if (!nkreq->src)
169 		return -ENOMEM;
170 
171 	/* copy iv */
172 	iv = nkreq->src;
173 	memcpy(iv, skreq->iv, ivsize);
174 
175 	sg = (struct scatterlist *)(iv + ivsize);
176 	creq->src = sg;
177 	sg_init_table(sg, nents);
178 
179 	/* Input format:
180 	 * +----+----------------+
181 	 * | IV | SRC sg entries |
182 	 * +----+----------------+
183 	 */
184 
185 	/* IV */
186 	sg = create_single_sg(sg, iv, ivsize);
187 	/* SRC entries */
188 	create_multi_sg(sg, skreq->src);
189 
190 	return 0;
191 }
192 
193 static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize)
194 {
195 	struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
196 	int nents = sg_nents(skreq->dst) + 3;
197 	int extralen = ORH_HLEN + COMP_HLEN;
198 	struct se_crypto_request *creq = &nkreq->creq;
199 	struct scatterlist *sg;
200 	char *iv = nkreq->src;
201 
202 	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
203 	 * array
204 	 */
205 	nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
206 	if (!nkreq->dst)
207 		return -ENOMEM;
208 
209 	creq->orh = (u64 *)(nkreq->dst);
210 	set_orh_value(creq->orh);
211 
212 	creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
213 	set_comp_value(creq->comp);
214 
215 	sg = (struct scatterlist *)(nkreq->dst + ORH_HLEN + COMP_HLEN);
216 	creq->dst = sg;
217 	sg_init_table(sg, nents);
218 
219 	/* Output format:
220 	 * +-----+----+----------------+-----------------+
221 	 * | ORH | IV | DST sg entries | COMPLETION Bytes|
222 	 * +-----+----+----------------+-----------------+
223 	 */
224 
225 	/* ORH */
226 	sg = create_single_sg(sg, creq->orh, ORH_HLEN);
227 	/* IV */
228 	sg = create_single_sg(sg, iv, ivsize);
229 	/* DST entries */
230 	sg = create_multi_sg(sg, skreq->dst);
231 	/* COMPLETION Bytes */
232 	create_single_sg(sg, creq->comp, COMP_HLEN);
233 
234 	return 0;
235 }
236 
237 static void free_src_sglist(struct skcipher_request *skreq)
238 {
239 	struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
240 
241 	kfree(nkreq->src);
242 }
243 
244 static void free_dst_sglist(struct skcipher_request *skreq)
245 {
246 	struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
247 
248 	kfree(nkreq->dst);
249 }
250 
251 static void nitrox_skcipher_callback(struct skcipher_request *skreq,
252 				     int err)
253 {
254 	free_src_sglist(skreq);
255 	free_dst_sglist(skreq);
256 	if (err) {
257 		pr_err_ratelimited("request failed status 0x%0x\n", err);
258 		err = -EINVAL;
259 	}
260 
261 	skcipher_request_complete(skreq, err);
262 }
263 
264 static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc)
265 {
266 	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
267 	struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
268 	struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
269 	int ivsize = crypto_skcipher_ivsize(cipher);
270 	struct se_crypto_request *creq;
271 	int ret;
272 
273 	creq = &nkreq->creq;
274 	creq->flags = skreq->base.flags;
275 	creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
276 		     GFP_KERNEL : GFP_ATOMIC;
277 
278 	/* fill the request */
279 	creq->ctrl.value = 0;
280 	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
281 	creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT);
282 	/* param0: length of the data to be encrypted */
283 	creq->gph.param0 = cpu_to_be16(skreq->cryptlen);
284 	creq->gph.param1 = 0;
285 	/* param2: encryption data offset */
286 	creq->gph.param2 = cpu_to_be16(ivsize);
287 	creq->gph.param3 = 0;
288 
289 	creq->ctx_handle = nctx->u.ctx_handle;
290 	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
291 
292 	ret = alloc_src_sglist(skreq, ivsize);
293 	if (ret)
294 		return ret;
295 
296 	ret = alloc_dst_sglist(skreq, ivsize);
297 	if (ret) {
298 		free_src_sglist(skreq);
299 		return ret;
300 	}
301 
302 	nkreq->nctx = nctx;
303 	nkreq->skreq = skreq;
304 
305 	/* send the crypto request */
306 	return nitrox_process_se_request(nctx->ndev, creq,
307 					 nitrox_skcipher_callback, skreq);
308 }
309 
310 static int nitrox_aes_encrypt(struct skcipher_request *skreq)
311 {
312 	return nitrox_skcipher_crypt(skreq, true);
313 }
314 
315 static int nitrox_aes_decrypt(struct skcipher_request *skreq)
316 {
317 	return nitrox_skcipher_crypt(skreq, false);
318 }
319 
320 static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
321 			      const u8 *key, unsigned int keylen)
322 {
323 	if (keylen != DES3_EDE_KEY_SIZE) {
324 		crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
325 		return -EINVAL;
326 	}
327 
328 	return nitrox_skcipher_setkey(cipher, 0, key, keylen);
329 }
330 
331 static int nitrox_3des_encrypt(struct skcipher_request *skreq)
332 {
333 	return nitrox_skcipher_crypt(skreq, true);
334 }
335 
336 static int nitrox_3des_decrypt(struct skcipher_request *skreq)
337 {
338 	return nitrox_skcipher_crypt(skreq, false);
339 }
340 
341 static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher,
342 				 const u8 *key, unsigned int keylen)
343 {
344 	struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
345 	struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
346 	struct flexi_crypto_context *fctx;
347 	int aes_keylen, ret;
348 
349 	ret = xts_check_key(tfm, key, keylen);
350 	if (ret)
351 		return ret;
352 
353 	keylen /= 2;
354 
355 	aes_keylen = flexi_aes_keylen(keylen);
356 	if (aes_keylen < 0) {
357 		crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
358 		return -EINVAL;
359 	}
360 
361 	fctx = nctx->u.fctx;
362 	/* copy KEY2 */
363 	memcpy(fctx->auth.u.key2, (key + keylen), keylen);
364 
365 	return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
366 }
367 
368 static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher,
369 					 const u8 *key, unsigned int keylen)
370 {
371 	struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
372 	struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
373 	struct flexi_crypto_context *fctx;
374 	int aes_keylen;
375 
376 	if (keylen < CTR_RFC3686_NONCE_SIZE)
377 		return -EINVAL;
378 
379 	fctx = nctx->u.fctx;
380 
381 	memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE),
382 	       CTR_RFC3686_NONCE_SIZE);
383 
384 	keylen -= CTR_RFC3686_NONCE_SIZE;
385 
386 	aes_keylen = flexi_aes_keylen(keylen);
387 	if (aes_keylen < 0) {
388 		crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
389 		return -EINVAL;
390 	}
391 	return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
392 }
393 
394 static struct skcipher_alg nitrox_skciphers[] = { {
395 	.base = {
396 		.cra_name = "cbc(aes)",
397 		.cra_driver_name = "n5_cbc(aes)",
398 		.cra_priority = PRIO,
399 		.cra_flags = CRYPTO_ALG_ASYNC,
400 		.cra_blocksize = AES_BLOCK_SIZE,
401 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
402 		.cra_alignmask = 0,
403 		.cra_module = THIS_MODULE,
404 	},
405 	.min_keysize = AES_MIN_KEY_SIZE,
406 	.max_keysize = AES_MAX_KEY_SIZE,
407 	.ivsize = AES_BLOCK_SIZE,
408 	.setkey = nitrox_aes_setkey,
409 	.encrypt = nitrox_aes_encrypt,
410 	.decrypt = nitrox_aes_decrypt,
411 	.init = nitrox_skcipher_init,
412 	.exit = nitrox_skcipher_exit,
413 }, {
414 	.base = {
415 		.cra_name = "ecb(aes)",
416 		.cra_driver_name = "n5_ecb(aes)",
417 		.cra_priority = PRIO,
418 		.cra_flags = CRYPTO_ALG_ASYNC,
419 		.cra_blocksize = AES_BLOCK_SIZE,
420 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
421 		.cra_alignmask = 0,
422 		.cra_module = THIS_MODULE,
423 	},
424 	.min_keysize = AES_MIN_KEY_SIZE,
425 	.max_keysize = AES_MAX_KEY_SIZE,
426 	.ivsize = AES_BLOCK_SIZE,
427 	.setkey = nitrox_aes_setkey,
428 	.encrypt = nitrox_aes_encrypt,
429 	.decrypt = nitrox_aes_decrypt,
430 	.init = nitrox_skcipher_init,
431 	.exit = nitrox_skcipher_exit,
432 }, {
433 	.base = {
434 		.cra_name = "cfb(aes)",
435 		.cra_driver_name = "n5_cfb(aes)",
436 		.cra_priority = PRIO,
437 		.cra_flags = CRYPTO_ALG_ASYNC,
438 		.cra_blocksize = AES_BLOCK_SIZE,
439 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
440 		.cra_alignmask = 0,
441 		.cra_module = THIS_MODULE,
442 	},
443 	.min_keysize = AES_MIN_KEY_SIZE,
444 	.max_keysize = AES_MAX_KEY_SIZE,
445 	.ivsize = AES_BLOCK_SIZE,
446 	.setkey = nitrox_aes_setkey,
447 	.encrypt = nitrox_aes_encrypt,
448 	.decrypt = nitrox_aes_decrypt,
449 	.init = nitrox_skcipher_init,
450 	.exit = nitrox_skcipher_exit,
451 }, {
452 	.base = {
453 		.cra_name = "xts(aes)",
454 		.cra_driver_name = "n5_xts(aes)",
455 		.cra_priority = PRIO,
456 		.cra_flags = CRYPTO_ALG_ASYNC,
457 		.cra_blocksize = AES_BLOCK_SIZE,
458 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
459 		.cra_alignmask = 0,
460 		.cra_module = THIS_MODULE,
461 	},
462 	.min_keysize = 2 * AES_MIN_KEY_SIZE,
463 	.max_keysize = 2 * AES_MAX_KEY_SIZE,
464 	.ivsize = AES_BLOCK_SIZE,
465 	.setkey = nitrox_aes_xts_setkey,
466 	.encrypt = nitrox_aes_encrypt,
467 	.decrypt = nitrox_aes_decrypt,
468 	.init = nitrox_skcipher_init,
469 	.exit = nitrox_skcipher_exit,
470 }, {
471 	.base = {
472 		.cra_name = "rfc3686(ctr(aes))",
473 		.cra_driver_name = "n5_rfc3686(ctr(aes))",
474 		.cra_priority = PRIO,
475 		.cra_flags = CRYPTO_ALG_ASYNC,
476 		.cra_blocksize = 1,
477 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
478 		.cra_alignmask = 0,
479 		.cra_module = THIS_MODULE,
480 	},
481 	.min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
482 	.max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
483 	.ivsize = CTR_RFC3686_IV_SIZE,
484 	.init = nitrox_skcipher_init,
485 	.exit = nitrox_skcipher_exit,
486 	.setkey = nitrox_aes_ctr_rfc3686_setkey,
487 	.encrypt = nitrox_aes_encrypt,
488 	.decrypt = nitrox_aes_decrypt,
489 }, {
490 	.base = {
491 		.cra_name = "cts(cbc(aes))",
492 		.cra_driver_name = "n5_cts(cbc(aes))",
493 		.cra_priority = PRIO,
494 		.cra_flags = CRYPTO_ALG_ASYNC,
495 		.cra_blocksize = AES_BLOCK_SIZE,
496 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
497 		.cra_alignmask = 0,
498 		.cra_type = &crypto_ablkcipher_type,
499 		.cra_module = THIS_MODULE,
500 	},
501 	.min_keysize = AES_MIN_KEY_SIZE,
502 	.max_keysize = AES_MAX_KEY_SIZE,
503 	.ivsize = AES_BLOCK_SIZE,
504 	.setkey = nitrox_aes_setkey,
505 	.encrypt = nitrox_aes_encrypt,
506 	.decrypt = nitrox_aes_decrypt,
507 	.init = nitrox_skcipher_init,
508 	.exit = nitrox_skcipher_exit,
509 }, {
510 	.base = {
511 		.cra_name = "cbc(des3_ede)",
512 		.cra_driver_name = "n5_cbc(des3_ede)",
513 		.cra_priority = PRIO,
514 		.cra_flags = CRYPTO_ALG_ASYNC,
515 		.cra_blocksize = DES3_EDE_BLOCK_SIZE,
516 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
517 		.cra_alignmask = 0,
518 		.cra_module = THIS_MODULE,
519 	},
520 	.min_keysize = DES3_EDE_KEY_SIZE,
521 	.max_keysize = DES3_EDE_KEY_SIZE,
522 	.ivsize = DES3_EDE_BLOCK_SIZE,
523 	.setkey = nitrox_3des_setkey,
524 	.encrypt = nitrox_3des_encrypt,
525 	.decrypt = nitrox_3des_decrypt,
526 	.init = nitrox_skcipher_init,
527 	.exit = nitrox_skcipher_exit,
528 }, {
529 	.base = {
530 		.cra_name = "ecb(des3_ede)",
531 		.cra_driver_name = "n5_ecb(des3_ede)",
532 		.cra_priority = PRIO,
533 		.cra_flags = CRYPTO_ALG_ASYNC,
534 		.cra_blocksize = DES3_EDE_BLOCK_SIZE,
535 		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
536 		.cra_alignmask = 0,
537 		.cra_module = THIS_MODULE,
538 	},
539 	.min_keysize = DES3_EDE_KEY_SIZE,
540 	.max_keysize = DES3_EDE_KEY_SIZE,
541 	.ivsize = DES3_EDE_BLOCK_SIZE,
542 	.setkey = nitrox_3des_setkey,
543 	.encrypt = nitrox_3des_encrypt,
544 	.decrypt = nitrox_3des_decrypt,
545 	.init = nitrox_skcipher_init,
546 	.exit = nitrox_skcipher_exit,
547 }
548 
549 };
550 
551 int nitrox_crypto_register(void)
552 {
553 	return crypto_register_skciphers(nitrox_skciphers,
554 					 ARRAY_SIZE(nitrox_skciphers));
555 }
556 
557 void nitrox_crypto_unregister(void)
558 {
559 	crypto_unregister_skciphers(nitrox_skciphers,
560 				    ARRAY_SIZE(nitrox_skciphers));
561 }
562