1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /*
4  * Copyright (C) 2016 Cavium, Inc.
5  */
6 
7 #include <crypto/aes.h>
8 #include <crypto/algapi.h>
9 #include <crypto/authenc.h>
10 #include <crypto/des.h>
11 #include <crypto/xts.h>
12 #include <linux/crypto.h>
13 #include <linux/err.h>
14 #include <linux/list.h>
15 #include <linux/scatterlist.h>
16 
17 #include "cptvf.h"
18 #include "cptvf_algs.h"
19 
20 struct cpt_device_handle {
21 	void *cdev[MAX_DEVICES];
22 	u32 dev_count;
23 };
24 
25 static struct cpt_device_handle dev_handle;
26 
27 static void cvm_callback(u32 status, void *arg)
28 {
29 	struct crypto_async_request *req = (struct crypto_async_request *)arg;
30 
31 	req->complete(req, !status);
32 }
33 
34 static inline void update_input_iv(struct cpt_request_info *req_info,
35 				   u8 *iv, u32 enc_iv_len,
36 				   u32 *argcnt)
37 {
38 	/* Setting the iv information */
39 	req_info->in[*argcnt].vptr = (void *)iv;
40 	req_info->in[*argcnt].size = enc_iv_len;
41 	req_info->req.dlen += enc_iv_len;
42 
43 	++(*argcnt);
44 }
45 
46 static inline void update_output_iv(struct cpt_request_info *req_info,
47 				    u8 *iv, u32 enc_iv_len,
48 				    u32 *argcnt)
49 {
50 	/* Setting the iv information */
51 	req_info->out[*argcnt].vptr = (void *)iv;
52 	req_info->out[*argcnt].size = enc_iv_len;
53 	req_info->rlen += enc_iv_len;
54 
55 	++(*argcnt);
56 }
57 
58 static inline void update_input_data(struct cpt_request_info *req_info,
59 				     struct scatterlist *inp_sg,
60 				     u32 nbytes, u32 *argcnt)
61 {
62 	req_info->req.dlen += nbytes;
63 
64 	while (nbytes) {
65 		u32 len = min(nbytes, inp_sg->length);
66 		u8 *ptr = sg_virt(inp_sg);
67 
68 		req_info->in[*argcnt].vptr = (void *)ptr;
69 		req_info->in[*argcnt].size = len;
70 		nbytes -= len;
71 
72 		++(*argcnt);
73 		++inp_sg;
74 	}
75 }
76 
77 static inline void update_output_data(struct cpt_request_info *req_info,
78 				      struct scatterlist *outp_sg,
79 				      u32 nbytes, u32 *argcnt)
80 {
81 	req_info->rlen += nbytes;
82 
83 	while (nbytes) {
84 		u32 len = min(nbytes, outp_sg->length);
85 		u8 *ptr = sg_virt(outp_sg);
86 
87 		req_info->out[*argcnt].vptr = (void *)ptr;
88 		req_info->out[*argcnt].size = len;
89 		nbytes -= len;
90 		++(*argcnt);
91 		++outp_sg;
92 	}
93 }
94 
95 static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
96 				 u32 *argcnt)
97 {
98 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
99 	struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
100 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
101 	struct fc_context *fctx = &rctx->fctx;
102 	u64 *offset_control = &rctx->control_word;
103 	u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
104 	struct cpt_request_info *req_info = &rctx->cpt_req;
105 	u64 *ctrl_flags = NULL;
106 
107 	req_info->ctrl.s.grp = 0;
108 	req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
109 	req_info->ctrl.s.se_req = SE_CORE_REQ;
110 
111 	req_info->req.opcode.s.major = MAJOR_OP_FC |
112 					DMA_MODE_FLAG(DMA_GATHER_SCATTER);
113 	if (enc)
114 		req_info->req.opcode.s.minor = 2;
115 	else
116 		req_info->req.opcode.s.minor = 3;
117 
118 	req_info->req.param1 = req->nbytes; /* Encryption Data length */
119 	req_info->req.param2 = 0; /*Auth data length */
120 
121 	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
122 	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
123 	fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
124 
125 	if (ctx->cipher_type == AES_XTS)
126 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
127 	else
128 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
129 	ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
130 	*ctrl_flags = cpu_to_be64(*ctrl_flags);
131 
132 	*offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
133 	/* Storing  Packet Data Information in offset
134 	 * Control Word First 8 bytes
135 	 */
136 	req_info->in[*argcnt].vptr = (u8 *)offset_control;
137 	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
138 	req_info->req.dlen += CONTROL_WORD_LEN;
139 	++(*argcnt);
140 
141 	req_info->in[*argcnt].vptr = (u8 *)fctx;
142 	req_info->in[*argcnt].size = sizeof(struct fc_context);
143 	req_info->req.dlen += sizeof(struct fc_context);
144 
145 	++(*argcnt);
146 
147 	return 0;
148 }
149 
150 static inline u32 create_input_list(struct ablkcipher_request  *req, u32 enc,
151 				    u32 enc_iv_len)
152 {
153 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
154 	struct cpt_request_info *req_info = &rctx->cpt_req;
155 	u32 argcnt =  0;
156 
157 	create_ctx_hdr(req, enc, &argcnt);
158 	update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
159 	update_input_data(req_info, req->src, req->nbytes, &argcnt);
160 	req_info->incnt = argcnt;
161 
162 	return 0;
163 }
164 
165 static inline void store_cb_info(struct ablkcipher_request *req,
166 				 struct cpt_request_info *req_info)
167 {
168 	req_info->callback = (void *)cvm_callback;
169 	req_info->callback_arg = (void *)&req->base;
170 }
171 
172 static inline void create_output_list(struct ablkcipher_request *req,
173 				      u32 enc_iv_len)
174 {
175 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
176 	struct cpt_request_info *req_info = &rctx->cpt_req;
177 	u32 argcnt = 0;
178 
179 	/* OUTPUT Buffer Processing
180 	 * AES encryption/decryption output would be
181 	 * received in the following format
182 	 *
183 	 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
184 	 * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
185 	 */
186 	/* Reading IV information */
187 	update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
188 	update_output_data(req_info, req->dst, req->nbytes, &argcnt);
189 	req_info->outcnt = argcnt;
190 }
191 
192 static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc)
193 {
194 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
195 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
196 	u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
197 	struct fc_context *fctx = &rctx->fctx;
198 	struct cpt_request_info *req_info = &rctx->cpt_req;
199 	void *cdev = NULL;
200 	int status;
201 
202 	memset(req_info, 0, sizeof(struct cpt_request_info));
203 	memset(fctx, 0, sizeof(struct fc_context));
204 	create_input_list(req, enc, enc_iv_len);
205 	create_output_list(req, enc_iv_len);
206 	store_cb_info(req, req_info);
207 	cdev = dev_handle.cdev[smp_processor_id()];
208 	status = cptvf_do_request(cdev, req_info);
209 	/* We perform an asynchronous send and once
210 	 * the request is completed the driver would
211 	 * intimate through  registered call back functions
212 	 */
213 
214 	if (status)
215 		return status;
216 	else
217 		return -EINPROGRESS;
218 }
219 
220 static int cvm_encrypt(struct ablkcipher_request *req)
221 {
222 	return cvm_enc_dec(req, true);
223 }
224 
225 static int cvm_decrypt(struct ablkcipher_request *req)
226 {
227 	return cvm_enc_dec(req, false);
228 }
229 
230 static int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
231 		   u32 keylen)
232 {
233 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
234 	struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
235 	int err;
236 	const u8 *key1 = key;
237 	const u8 *key2 = key + (keylen / 2);
238 
239 	err = xts_check_key(tfm, key, keylen);
240 	if (err)
241 		return err;
242 	ctx->key_len = keylen;
243 	memcpy(ctx->enc_key, key1, keylen / 2);
244 	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
245 	ctx->cipher_type = AES_XTS;
246 	switch (ctx->key_len) {
247 	case 32:
248 		ctx->key_type = AES_128_BIT;
249 		break;
250 	case 64:
251 		ctx->key_type = AES_256_BIT;
252 		break;
253 	default:
254 		return -EINVAL;
255 	}
256 
257 	return 0;
258 }
259 
260 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
261 {
262 	if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
263 		ctx->key_len = keylen;
264 		switch (ctx->key_len) {
265 		case 16:
266 			ctx->key_type = AES_128_BIT;
267 			break;
268 		case 24:
269 			ctx->key_type = AES_192_BIT;
270 			break;
271 		case 32:
272 			ctx->key_type = AES_256_BIT;
273 			break;
274 		default:
275 			return -EINVAL;
276 		}
277 
278 		if (ctx->cipher_type == DES3_CBC)
279 			ctx->key_type = 0;
280 
281 		return 0;
282 	}
283 
284 	return -EINVAL;
285 }
286 
287 static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
288 		      u32 keylen, u8 cipher_type)
289 {
290 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
291 	struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
292 
293 	ctx->cipher_type = cipher_type;
294 	if (!cvm_validate_keylen(ctx, keylen)) {
295 		memcpy(ctx->enc_key, key, keylen);
296 		return 0;
297 	} else {
298 		crypto_ablkcipher_set_flags(cipher,
299 					    CRYPTO_TFM_RES_BAD_KEY_LEN);
300 		return -EINVAL;
301 	}
302 }
303 
304 static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
305 			      u32 keylen)
306 {
307 	return cvm_setkey(cipher, key, keylen, AES_CBC);
308 }
309 
310 static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
311 			      u32 keylen)
312 {
313 	return cvm_setkey(cipher, key, keylen, AES_ECB);
314 }
315 
316 static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
317 			      u32 keylen)
318 {
319 	return cvm_setkey(cipher, key, keylen, AES_CFB);
320 }
321 
322 static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
323 			       u32 keylen)
324 {
325 	u32 flags = crypto_ablkcipher_get_flags(cipher);
326 	int err;
327 
328 	err = __des3_verify_key(&flags, key);
329 	if (unlikely(err)) {
330 		crypto_ablkcipher_set_flags(cipher, flags);
331 		return err;
332 	}
333 
334 	return cvm_setkey(cipher, key, keylen, DES3_CBC);
335 }
336 
337 static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
338 			       u32 keylen)
339 {
340 	u32 flags = crypto_ablkcipher_get_flags(cipher);
341 	int err;
342 
343 	err = __des3_verify_key(&flags, key);
344 	if (unlikely(err)) {
345 		crypto_ablkcipher_set_flags(cipher, flags);
346 		return err;
347 	}
348 
349 	return cvm_setkey(cipher, key, keylen, DES3_ECB);
350 }
351 
352 static int cvm_enc_dec_init(struct crypto_tfm *tfm)
353 {
354 	tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx);
355 	return 0;
356 }
357 
358 static struct crypto_alg algs[] = { {
359 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
360 	.cra_blocksize = AES_BLOCK_SIZE,
361 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
362 	.cra_alignmask = 7,
363 	.cra_priority = 4001,
364 	.cra_name = "xts(aes)",
365 	.cra_driver_name = "cavium-xts-aes",
366 	.cra_type = &crypto_ablkcipher_type,
367 	.cra_u = {
368 		.ablkcipher = {
369 			.ivsize = AES_BLOCK_SIZE,
370 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
371 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
372 			.setkey = cvm_xts_setkey,
373 			.encrypt = cvm_encrypt,
374 			.decrypt = cvm_decrypt,
375 		},
376 	},
377 	.cra_init = cvm_enc_dec_init,
378 	.cra_module = THIS_MODULE,
379 }, {
380 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
381 	.cra_blocksize = AES_BLOCK_SIZE,
382 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
383 	.cra_alignmask = 7,
384 	.cra_priority = 4001,
385 	.cra_name = "cbc(aes)",
386 	.cra_driver_name = "cavium-cbc-aes",
387 	.cra_type = &crypto_ablkcipher_type,
388 	.cra_u = {
389 		.ablkcipher = {
390 			.ivsize = AES_BLOCK_SIZE,
391 			.min_keysize = AES_MIN_KEY_SIZE,
392 			.max_keysize = AES_MAX_KEY_SIZE,
393 			.setkey = cvm_cbc_aes_setkey,
394 			.encrypt = cvm_encrypt,
395 			.decrypt = cvm_decrypt,
396 		},
397 	},
398 	.cra_init = cvm_enc_dec_init,
399 	.cra_module = THIS_MODULE,
400 }, {
401 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
402 	.cra_blocksize = AES_BLOCK_SIZE,
403 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
404 	.cra_alignmask = 7,
405 	.cra_priority = 4001,
406 	.cra_name = "ecb(aes)",
407 	.cra_driver_name = "cavium-ecb-aes",
408 	.cra_type = &crypto_ablkcipher_type,
409 	.cra_u = {
410 		.ablkcipher = {
411 			.ivsize = AES_BLOCK_SIZE,
412 			.min_keysize = AES_MIN_KEY_SIZE,
413 			.max_keysize = AES_MAX_KEY_SIZE,
414 			.setkey = cvm_ecb_aes_setkey,
415 			.encrypt = cvm_encrypt,
416 			.decrypt = cvm_decrypt,
417 		},
418 	},
419 	.cra_init = cvm_enc_dec_init,
420 	.cra_module = THIS_MODULE,
421 }, {
422 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
423 	.cra_blocksize = AES_BLOCK_SIZE,
424 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
425 	.cra_alignmask = 7,
426 	.cra_priority = 4001,
427 	.cra_name = "cfb(aes)",
428 	.cra_driver_name = "cavium-cfb-aes",
429 	.cra_type = &crypto_ablkcipher_type,
430 	.cra_u = {
431 		.ablkcipher = {
432 			.ivsize = AES_BLOCK_SIZE,
433 			.min_keysize = AES_MIN_KEY_SIZE,
434 			.max_keysize = AES_MAX_KEY_SIZE,
435 			.setkey = cvm_cfb_aes_setkey,
436 			.encrypt = cvm_encrypt,
437 			.decrypt = cvm_decrypt,
438 		},
439 	},
440 	.cra_init = cvm_enc_dec_init,
441 	.cra_module = THIS_MODULE,
442 }, {
443 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
444 	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
445 	.cra_ctxsize = sizeof(struct cvm_des3_ctx),
446 	.cra_alignmask = 7,
447 	.cra_priority = 4001,
448 	.cra_name = "cbc(des3_ede)",
449 	.cra_driver_name = "cavium-cbc-des3_ede",
450 	.cra_type = &crypto_ablkcipher_type,
451 	.cra_u = {
452 		.ablkcipher = {
453 			.min_keysize = DES3_EDE_KEY_SIZE,
454 			.max_keysize = DES3_EDE_KEY_SIZE,
455 			.ivsize = DES_BLOCK_SIZE,
456 			.setkey = cvm_cbc_des3_setkey,
457 			.encrypt = cvm_encrypt,
458 			.decrypt = cvm_decrypt,
459 		},
460 	},
461 	.cra_init = cvm_enc_dec_init,
462 	.cra_module = THIS_MODULE,
463 }, {
464 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
465 	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
466 	.cra_ctxsize = sizeof(struct cvm_des3_ctx),
467 	.cra_alignmask = 7,
468 	.cra_priority = 4001,
469 	.cra_name = "ecb(des3_ede)",
470 	.cra_driver_name = "cavium-ecb-des3_ede",
471 	.cra_type = &crypto_ablkcipher_type,
472 	.cra_u = {
473 		.ablkcipher = {
474 			.min_keysize = DES3_EDE_KEY_SIZE,
475 			.max_keysize = DES3_EDE_KEY_SIZE,
476 			.ivsize = DES_BLOCK_SIZE,
477 			.setkey = cvm_ecb_des3_setkey,
478 			.encrypt = cvm_encrypt,
479 			.decrypt = cvm_decrypt,
480 		},
481 	},
482 	.cra_init = cvm_enc_dec_init,
483 	.cra_module = THIS_MODULE,
484 } };
485 
486 static inline int cav_register_algs(void)
487 {
488 	int err = 0;
489 
490 	err = crypto_register_algs(algs, ARRAY_SIZE(algs));
491 	if (err)
492 		return err;
493 
494 	return 0;
495 }
496 
497 static inline void cav_unregister_algs(void)
498 {
499 	crypto_unregister_algs(algs, ARRAY_SIZE(algs));
500 }
501 
502 int cvm_crypto_init(struct cpt_vf *cptvf)
503 {
504 	struct pci_dev *pdev = cptvf->pdev;
505 	u32 dev_count;
506 
507 	dev_count = dev_handle.dev_count;
508 	dev_handle.cdev[dev_count] = cptvf;
509 	dev_handle.dev_count++;
510 
511 	if (dev_count == 3) {
512 		if (cav_register_algs()) {
513 			dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
514 			return -EINVAL;
515 		}
516 	}
517 
518 	return 0;
519 }
520 
521 void cvm_crypto_exit(void)
522 {
523 	u32 dev_count;
524 
525 	dev_count = --dev_handle.dev_count;
526 	if (!dev_count)
527 		cav_unregister_algs();
528 }
529