1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2016-2017 HiSilicon Limited. */
3 #include <linux/crypto.h>
4 #include <linux/dma-mapping.h>
5 #include <linux/dmapool.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/slab.h>
9 
10 #include <crypto/aes.h>
11 #include <crypto/algapi.h>
12 #include <crypto/internal/des.h>
13 #include <crypto/skcipher.h>
14 #include <crypto/xts.h>
15 #include <crypto/internal/skcipher.h>
16 
17 #include "sec_drv.h"
18 
19 #define SEC_MAX_CIPHER_KEY		64
20 #define SEC_REQ_LIMIT SZ_32M
21 
22 struct sec_c_alg_cfg {
23 	unsigned c_alg		: 3;
24 	unsigned c_mode		: 3;
25 	unsigned key_len	: 2;
26 	unsigned c_width	: 2;
27 };
28 
29 static const struct sec_c_alg_cfg sec_c_alg_cfgs[] =  {
30 	[SEC_C_DES_ECB_64] = {
31 		.c_alg = SEC_C_ALG_DES,
32 		.c_mode = SEC_C_MODE_ECB,
33 		.key_len = SEC_KEY_LEN_DES,
34 	},
35 	[SEC_C_DES_CBC_64] = {
36 		.c_alg = SEC_C_ALG_DES,
37 		.c_mode = SEC_C_MODE_CBC,
38 		.key_len = SEC_KEY_LEN_DES,
39 	},
40 	[SEC_C_3DES_ECB_192_3KEY] = {
41 		.c_alg = SEC_C_ALG_3DES,
42 		.c_mode = SEC_C_MODE_ECB,
43 		.key_len = SEC_KEY_LEN_3DES_3_KEY,
44 	},
45 	[SEC_C_3DES_ECB_192_2KEY] = {
46 		.c_alg = SEC_C_ALG_3DES,
47 		.c_mode = SEC_C_MODE_ECB,
48 		.key_len = SEC_KEY_LEN_3DES_2_KEY,
49 	},
50 	[SEC_C_3DES_CBC_192_3KEY] = {
51 		.c_alg = SEC_C_ALG_3DES,
52 		.c_mode = SEC_C_MODE_CBC,
53 		.key_len = SEC_KEY_LEN_3DES_3_KEY,
54 	},
55 	[SEC_C_3DES_CBC_192_2KEY] = {
56 		.c_alg = SEC_C_ALG_3DES,
57 		.c_mode = SEC_C_MODE_CBC,
58 		.key_len = SEC_KEY_LEN_3DES_2_KEY,
59 	},
60 	[SEC_C_AES_ECB_128] = {
61 		.c_alg = SEC_C_ALG_AES,
62 		.c_mode = SEC_C_MODE_ECB,
63 		.key_len = SEC_KEY_LEN_AES_128,
64 	},
65 	[SEC_C_AES_ECB_192] = {
66 		.c_alg = SEC_C_ALG_AES,
67 		.c_mode = SEC_C_MODE_ECB,
68 		.key_len = SEC_KEY_LEN_AES_192,
69 	},
70 	[SEC_C_AES_ECB_256] = {
71 		.c_alg = SEC_C_ALG_AES,
72 		.c_mode = SEC_C_MODE_ECB,
73 		.key_len = SEC_KEY_LEN_AES_256,
74 	},
75 	[SEC_C_AES_CBC_128] = {
76 		.c_alg = SEC_C_ALG_AES,
77 		.c_mode = SEC_C_MODE_CBC,
78 		.key_len = SEC_KEY_LEN_AES_128,
79 	},
80 	[SEC_C_AES_CBC_192] = {
81 		.c_alg = SEC_C_ALG_AES,
82 		.c_mode = SEC_C_MODE_CBC,
83 		.key_len = SEC_KEY_LEN_AES_192,
84 	},
85 	[SEC_C_AES_CBC_256] = {
86 		.c_alg = SEC_C_ALG_AES,
87 		.c_mode = SEC_C_MODE_CBC,
88 		.key_len = SEC_KEY_LEN_AES_256,
89 	},
90 	[SEC_C_AES_CTR_128] = {
91 		.c_alg = SEC_C_ALG_AES,
92 		.c_mode = SEC_C_MODE_CTR,
93 		.key_len = SEC_KEY_LEN_AES_128,
94 	},
95 	[SEC_C_AES_CTR_192] = {
96 		.c_alg = SEC_C_ALG_AES,
97 		.c_mode = SEC_C_MODE_CTR,
98 		.key_len = SEC_KEY_LEN_AES_192,
99 	},
100 	[SEC_C_AES_CTR_256] = {
101 		.c_alg = SEC_C_ALG_AES,
102 		.c_mode = SEC_C_MODE_CTR,
103 		.key_len = SEC_KEY_LEN_AES_256,
104 	},
105 	[SEC_C_AES_XTS_128] = {
106 		.c_alg = SEC_C_ALG_AES,
107 		.c_mode = SEC_C_MODE_XTS,
108 		.key_len = SEC_KEY_LEN_AES_128,
109 	},
110 	[SEC_C_AES_XTS_256] = {
111 		.c_alg = SEC_C_ALG_AES,
112 		.c_mode = SEC_C_MODE_XTS,
113 		.key_len = SEC_KEY_LEN_AES_256,
114 	},
115 	[SEC_C_NULL] = {
116 	},
117 };
118 
119 /*
120  * Mutex used to ensure safe operation of reference count of
121  * alg providers
122  */
123 static DEFINE_MUTEX(algs_lock);
124 static unsigned int active_devs;
125 
126 static void sec_alg_skcipher_init_template(struct sec_alg_tfm_ctx *ctx,
127 					   struct sec_bd_info *req,
128 					   enum sec_cipher_alg alg)
129 {
130 	const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg];
131 
132 	memset(req, 0, sizeof(*req));
133 	req->w0 |= cfg->c_mode << SEC_BD_W0_C_MODE_S;
134 	req->w1 |= cfg->c_alg << SEC_BD_W1_C_ALG_S;
135 	req->w3 |= cfg->key_len << SEC_BD_W3_C_KEY_LEN_S;
136 	req->w0 |= cfg->c_width << SEC_BD_W0_C_WIDTH_S;
137 
138 	req->cipher_key_addr_lo = lower_32_bits(ctx->pkey);
139 	req->cipher_key_addr_hi = upper_32_bits(ctx->pkey);
140 }
141 
142 static void sec_alg_skcipher_init_context(struct crypto_skcipher *atfm,
143 					  const u8 *key,
144 					  unsigned int keylen,
145 					  enum sec_cipher_alg alg)
146 {
147 	struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
148 	struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
149 
150 	ctx->cipher_alg = alg;
151 	memcpy(ctx->key, key, keylen);
152 	sec_alg_skcipher_init_template(ctx, &ctx->req_template,
153 				       ctx->cipher_alg);
154 }
155 
156 static void sec_free_hw_sgl(struct sec_hw_sgl *hw_sgl,
157 			    dma_addr_t psec_sgl, struct sec_dev_info *info)
158 {
159 	struct sec_hw_sgl *sgl_current, *sgl_next;
160 	dma_addr_t sgl_next_dma;
161 
162 	sgl_current = hw_sgl;
163 	while (sgl_current) {
164 		sgl_next = sgl_current->next;
165 		sgl_next_dma = sgl_current->next_sgl;
166 
167 		dma_pool_free(info->hw_sgl_pool, sgl_current, psec_sgl);
168 
169 		sgl_current = sgl_next;
170 		psec_sgl = sgl_next_dma;
171 	}
172 }
173 
174 static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl,
175 				     dma_addr_t *psec_sgl,
176 				     struct scatterlist *sgl,
177 				     int count,
178 				     struct sec_dev_info *info,
179 				     gfp_t gfp)
180 {
181 	struct sec_hw_sgl *sgl_current = NULL;
182 	struct sec_hw_sgl *sgl_next;
183 	dma_addr_t sgl_next_dma;
184 	struct scatterlist *sg;
185 	int ret, sge_index, i;
186 
187 	if (!count)
188 		return -EINVAL;
189 
190 	for_each_sg(sgl, sg, count, i) {
191 		sge_index = i % SEC_MAX_SGE_NUM;
192 		if (sge_index == 0) {
193 			sgl_next = dma_pool_zalloc(info->hw_sgl_pool,
194 						   gfp, &sgl_next_dma);
195 			if (!sgl_next) {
196 				ret = -ENOMEM;
197 				goto err_free_hw_sgls;
198 			}
199 
200 			if (!sgl_current) { /* First one */
201 				*psec_sgl = sgl_next_dma;
202 				*sec_sgl = sgl_next;
203 			} else { /* Chained */
204 				sgl_current->entry_sum_in_sgl = SEC_MAX_SGE_NUM;
205 				sgl_current->next_sgl = sgl_next_dma;
206 				sgl_current->next = sgl_next;
207 			}
208 			sgl_current = sgl_next;
209 		}
210 		sgl_current->sge_entries[sge_index].buf = sg_dma_address(sg);
211 		sgl_current->sge_entries[sge_index].len = sg_dma_len(sg);
212 		sgl_current->data_bytes_in_sgl += sg_dma_len(sg);
213 	}
214 	sgl_current->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM;
215 	sgl_current->next_sgl = 0;
216 	(*sec_sgl)->entry_sum_in_chain = count;
217 
218 	return 0;
219 
220 err_free_hw_sgls:
221 	sec_free_hw_sgl(*sec_sgl, *psec_sgl, info);
222 	*psec_sgl = 0;
223 
224 	return ret;
225 }
226 
227 static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm,
228 				   const u8 *key, unsigned int keylen,
229 				   enum sec_cipher_alg alg)
230 {
231 	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
232 	struct device *dev = ctx->queue->dev_info->dev;
233 
234 	mutex_lock(&ctx->lock);
235 	if (ctx->key) {
236 		/* rekeying */
237 		memset(ctx->key, 0, SEC_MAX_CIPHER_KEY);
238 	} else {
239 		/* new key */
240 		ctx->key = dma_alloc_coherent(dev, SEC_MAX_CIPHER_KEY,
241 					      &ctx->pkey, GFP_KERNEL);
242 		if (!ctx->key) {
243 			mutex_unlock(&ctx->lock);
244 			return -ENOMEM;
245 		}
246 	}
247 	mutex_unlock(&ctx->lock);
248 	sec_alg_skcipher_init_context(tfm, key, keylen, alg);
249 
250 	return 0;
251 }
252 
253 static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm,
254 					   const u8 *key, unsigned int keylen)
255 {
256 	enum sec_cipher_alg alg;
257 
258 	switch (keylen) {
259 	case AES_KEYSIZE_128:
260 		alg = SEC_C_AES_ECB_128;
261 		break;
262 	case AES_KEYSIZE_192:
263 		alg = SEC_C_AES_ECB_192;
264 		break;
265 	case AES_KEYSIZE_256:
266 		alg = SEC_C_AES_ECB_256;
267 		break;
268 	default:
269 		return -EINVAL;
270 	}
271 
272 	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
273 }
274 
275 static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm,
276 					   const u8 *key, unsigned int keylen)
277 {
278 	enum sec_cipher_alg alg;
279 
280 	switch (keylen) {
281 	case AES_KEYSIZE_128:
282 		alg = SEC_C_AES_CBC_128;
283 		break;
284 	case AES_KEYSIZE_192:
285 		alg = SEC_C_AES_CBC_192;
286 		break;
287 	case AES_KEYSIZE_256:
288 		alg = SEC_C_AES_CBC_256;
289 		break;
290 	default:
291 		return -EINVAL;
292 	}
293 
294 	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
295 }
296 
297 static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm,
298 					   const u8 *key, unsigned int keylen)
299 {
300 	enum sec_cipher_alg alg;
301 
302 	switch (keylen) {
303 	case AES_KEYSIZE_128:
304 		alg = SEC_C_AES_CTR_128;
305 		break;
306 	case AES_KEYSIZE_192:
307 		alg = SEC_C_AES_CTR_192;
308 		break;
309 	case AES_KEYSIZE_256:
310 		alg = SEC_C_AES_CTR_256;
311 		break;
312 	default:
313 		return -EINVAL;
314 	}
315 
316 	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
317 }
318 
319 static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
320 					   const u8 *key, unsigned int keylen)
321 {
322 	enum sec_cipher_alg alg;
323 	int ret;
324 
325 	ret = xts_verify_key(tfm, key, keylen);
326 	if (ret)
327 		return ret;
328 
329 	switch (keylen) {
330 	case AES_KEYSIZE_128 * 2:
331 		alg = SEC_C_AES_XTS_128;
332 		break;
333 	case AES_KEYSIZE_256 * 2:
334 		alg = SEC_C_AES_XTS_256;
335 		break;
336 	default:
337 		return -EINVAL;
338 	}
339 
340 	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
341 }
342 
343 static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
344 					   const u8 *key, unsigned int keylen)
345 {
346 	return verify_skcipher_des_key(tfm, key) ?:
347 	       sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
348 }
349 
350 static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
351 					   const u8 *key, unsigned int keylen)
352 {
353 	return verify_skcipher_des_key(tfm, key) ?:
354 	       sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
355 }
356 
357 static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
358 					    const u8 *key, unsigned int keylen)
359 {
360 	return verify_skcipher_des3_key(tfm, key) ?:
361 	       sec_alg_skcipher_setkey(tfm, key, keylen,
362 				       SEC_C_3DES_ECB_192_3KEY);
363 }
364 
365 static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
366 					    const u8 *key, unsigned int keylen)
367 {
368 	return verify_skcipher_des3_key(tfm, key) ?:
369 	       sec_alg_skcipher_setkey(tfm, key, keylen,
370 				       SEC_C_3DES_CBC_192_3KEY);
371 }
372 
373 static void sec_alg_free_el(struct sec_request_el *el,
374 			    struct sec_dev_info *info)
375 {
376 	sec_free_hw_sgl(el->out, el->dma_out, info);
377 	sec_free_hw_sgl(el->in, el->dma_in, info);
378 	kfree(el->sgl_in);
379 	kfree(el->sgl_out);
380 	kfree(el);
381 }
382 
383 /* queuelock must be held */
384 static int sec_send_request(struct sec_request *sec_req, struct sec_queue *queue)
385 {
386 	struct sec_request_el *el, *temp;
387 	int ret = 0;
388 
389 	mutex_lock(&sec_req->lock);
390 	list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
391 		/*
392 		 * Add to hardware queue only under following circumstances
393 		 * 1) Software and hardware queue empty so no chain dependencies
394 		 * 2) No dependencies as new IV - (check software queue empty
395 		 *    to maintain order)
396 		 * 3) No dependencies because the mode does no chaining.
397 		 *
398 		 * In other cases first insert onto the software queue which
399 		 * is then emptied as requests complete
400 		 */
401 		if (!queue->havesoftqueue ||
402 		    (kfifo_is_empty(&queue->softqueue) &&
403 		     sec_queue_empty(queue))) {
404 			ret = sec_queue_send(queue, &el->req, sec_req);
405 			if (ret == -EAGAIN) {
406 				/* Wait unti we can send then try again */
407 				/* DEAD if here - should not happen */
408 				ret = -EBUSY;
409 				goto err_unlock;
410 			}
411 		} else {
412 			kfifo_put(&queue->softqueue, el);
413 		}
414 	}
415 err_unlock:
416 	mutex_unlock(&sec_req->lock);
417 
418 	return ret;
419 }
420 
421 static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp,
422 				      struct crypto_async_request *req_base)
423 {
424 	struct skcipher_request *skreq = container_of(req_base,
425 						      struct skcipher_request,
426 						      base);
427 	struct sec_request *sec_req = skcipher_request_ctx(skreq);
428 	struct sec_request *backlog_req;
429 	struct sec_request_el *sec_req_el, *nextrequest;
430 	struct sec_alg_tfm_ctx *ctx = sec_req->tfm_ctx;
431 	struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
432 	struct device *dev = ctx->queue->dev_info->dev;
433 	int icv_or_skey_en, ret;
434 	bool done;
435 
436 	sec_req_el = list_first_entry(&sec_req->elements, struct sec_request_el,
437 				      head);
438 	icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >>
439 		SEC_BD_W0_ICV_OR_SKEY_EN_S;
440 	if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) {
441 		dev_err(dev, "Got an invalid answer %lu %d\n",
442 			sec_resp->w1 & SEC_BD_W1_BD_INVALID,
443 			icv_or_skey_en);
444 		sec_req->err = -EINVAL;
445 		/*
446 		 * We need to muddle on to avoid getting stuck with elements
447 		 * on the queue. Error will be reported so requester so
448 		 * it should be able to handle appropriately.
449 		 */
450 	}
451 
452 	spin_lock_bh(&ctx->queue->queuelock);
453 	/* Put the IV in place for chained cases */
454 	switch (ctx->cipher_alg) {
455 	case SEC_C_AES_CBC_128:
456 	case SEC_C_AES_CBC_192:
457 	case SEC_C_AES_CBC_256:
458 		if (sec_req_el->req.w0 & SEC_BD_W0_DE)
459 			sg_pcopy_to_buffer(sec_req_el->sgl_out,
460 					   sg_nents(sec_req_el->sgl_out),
461 					   skreq->iv,
462 					   crypto_skcipher_ivsize(atfm),
463 					   sec_req_el->el_length -
464 					   crypto_skcipher_ivsize(atfm));
465 		else
466 			sg_pcopy_to_buffer(sec_req_el->sgl_in,
467 					   sg_nents(sec_req_el->sgl_in),
468 					   skreq->iv,
469 					   crypto_skcipher_ivsize(atfm),
470 					   sec_req_el->el_length -
471 					   crypto_skcipher_ivsize(atfm));
472 		/* No need to sync to the device as coherent DMA */
473 		break;
474 	case SEC_C_AES_CTR_128:
475 	case SEC_C_AES_CTR_192:
476 	case SEC_C_AES_CTR_256:
477 		crypto_inc(skreq->iv, 16);
478 		break;
479 	default:
480 		/* Do not update */
481 		break;
482 	}
483 
484 	if (ctx->queue->havesoftqueue &&
485 	    !kfifo_is_empty(&ctx->queue->softqueue) &&
486 	    sec_queue_empty(ctx->queue)) {
487 		ret = kfifo_get(&ctx->queue->softqueue, &nextrequest);
488 		if (ret <= 0)
489 			dev_err(dev,
490 				"Error getting next element from kfifo %d\n",
491 				ret);
492 		else
493 			/* We know there is space so this cannot fail */
494 			sec_queue_send(ctx->queue, &nextrequest->req,
495 				       nextrequest->sec_req);
496 	} else if (!list_empty(&ctx->backlog)) {
497 		/* Need to verify there is room first */
498 		backlog_req = list_first_entry(&ctx->backlog,
499 					       typeof(*backlog_req),
500 					       backlog_head);
501 		if (sec_queue_can_enqueue(ctx->queue,
502 		    backlog_req->num_elements) ||
503 		    (ctx->queue->havesoftqueue &&
504 		     kfifo_avail(&ctx->queue->softqueue) >
505 		     backlog_req->num_elements)) {
506 			sec_send_request(backlog_req, ctx->queue);
507 			crypto_request_complete(backlog_req->req_base,
508 						-EINPROGRESS);
509 			list_del(&backlog_req->backlog_head);
510 		}
511 	}
512 	spin_unlock_bh(&ctx->queue->queuelock);
513 
514 	mutex_lock(&sec_req->lock);
515 	list_del(&sec_req_el->head);
516 	mutex_unlock(&sec_req->lock);
517 	sec_alg_free_el(sec_req_el, ctx->queue->dev_info);
518 
519 	/*
520 	 * Request is done.
521 	 * The dance is needed as the lock is freed in the completion
522 	 */
523 	mutex_lock(&sec_req->lock);
524 	done = list_empty(&sec_req->elements);
525 	mutex_unlock(&sec_req->lock);
526 	if (done) {
527 		if (crypto_skcipher_ivsize(atfm)) {
528 			dma_unmap_single(dev, sec_req->dma_iv,
529 					 crypto_skcipher_ivsize(atfm),
530 					 DMA_TO_DEVICE);
531 		}
532 		dma_unmap_sg(dev, skreq->src, sec_req->len_in,
533 			     DMA_BIDIRECTIONAL);
534 		if (skreq->src != skreq->dst)
535 			dma_unmap_sg(dev, skreq->dst, sec_req->len_out,
536 				     DMA_BIDIRECTIONAL);
537 		skcipher_request_complete(skreq, sec_req->err);
538 	}
539 }
540 
541 void sec_alg_callback(struct sec_bd_info *resp, void *shadow)
542 {
543 	struct sec_request *sec_req = shadow;
544 
545 	sec_req->cb(resp, sec_req->req_base);
546 }
547 
548 static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes,
549 					      int *steps, gfp_t gfp)
550 {
551 	size_t *sizes;
552 	int i;
553 
554 	/* Split into suitable sized blocks */
555 	*steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT;
556 	sizes = kcalloc(*steps, sizeof(*sizes), gfp);
557 	if (!sizes)
558 		return -ENOMEM;
559 
560 	for (i = 0; i < *steps - 1; i++)
561 		sizes[i] = SEC_REQ_LIMIT;
562 	sizes[*steps - 1] = length - SEC_REQ_LIMIT * (*steps - 1);
563 	*split_sizes = sizes;
564 
565 	return 0;
566 }
567 
568 static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes,
569 				int steps, struct scatterlist ***splits,
570 				int **splits_nents,
571 				int sgl_len_in,
572 				struct device *dev, gfp_t gfp)
573 {
574 	int ret, count;
575 
576 	count = dma_map_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
577 	if (!count)
578 		return -EINVAL;
579 
580 	*splits = kcalloc(steps, sizeof(struct scatterlist *), gfp);
581 	if (!*splits) {
582 		ret = -ENOMEM;
583 		goto err_unmap_sg;
584 	}
585 	*splits_nents = kcalloc(steps, sizeof(int), gfp);
586 	if (!*splits_nents) {
587 		ret = -ENOMEM;
588 		goto err_free_splits;
589 	}
590 
591 	/* output the scatter list before and after this */
592 	ret = sg_split(sgl, count, 0, steps, split_sizes,
593 		       *splits, *splits_nents, gfp);
594 	if (ret) {
595 		ret = -ENOMEM;
596 		goto err_free_splits_nents;
597 	}
598 
599 	return 0;
600 
601 err_free_splits_nents:
602 	kfree(*splits_nents);
603 err_free_splits:
604 	kfree(*splits);
605 err_unmap_sg:
606 	dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
607 
608 	return ret;
609 }
610 
611 /*
612  * Reverses the sec_map_and_split_sg call for messages not yet added to
613  * the queues.
614  */
615 static void sec_unmap_sg_on_err(struct scatterlist *sgl, int steps,
616 				struct scatterlist **splits, int *splits_nents,
617 				int sgl_len_in, struct device *dev)
618 {
619 	int i;
620 
621 	for (i = 0; i < steps; i++)
622 		kfree(splits[i]);
623 	kfree(splits_nents);
624 	kfree(splits);
625 
626 	dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
627 }
628 
629 static struct sec_request_el
630 *sec_alg_alloc_and_fill_el(struct sec_bd_info *template, int encrypt,
631 			   int el_size, bool different_dest,
632 			   struct scatterlist *sgl_in, int n_ents_in,
633 			   struct scatterlist *sgl_out, int n_ents_out,
634 			   struct sec_dev_info *info, gfp_t gfp)
635 {
636 	struct sec_request_el *el;
637 	struct sec_bd_info *req;
638 	int ret;
639 
640 	el = kzalloc(sizeof(*el), gfp);
641 	if (!el)
642 		return ERR_PTR(-ENOMEM);
643 	el->el_length = el_size;
644 	req = &el->req;
645 	memcpy(req, template, sizeof(*req));
646 
647 	req->w0 &= ~SEC_BD_W0_CIPHER_M;
648 	if (encrypt)
649 		req->w0 |= SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S;
650 	else
651 		req->w0 |= SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S;
652 
653 	req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_19_16_M;
654 	req->w0 |= ((el_size >> 16) << SEC_BD_W0_C_GRAN_SIZE_19_16_S) &
655 		SEC_BD_W0_C_GRAN_SIZE_19_16_M;
656 
657 	req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_21_20_M;
658 	req->w0 |= ((el_size >> 20) << SEC_BD_W0_C_GRAN_SIZE_21_20_S) &
659 		SEC_BD_W0_C_GRAN_SIZE_21_20_M;
660 
661 	/* Writing whole u32 so no need to take care of masking */
662 	req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) |
663 		((el_size << SEC_BD_W2_C_GRAN_SIZE_15_0_S) &
664 		 SEC_BD_W2_C_GRAN_SIZE_15_0_M);
665 
666 	req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M;
667 	req->w1 |= SEC_BD_W1_ADDR_TYPE;
668 
669 	el->sgl_in = sgl_in;
670 
671 	ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in,
672 					n_ents_in, info, gfp);
673 	if (ret)
674 		goto err_free_el;
675 
676 	req->data_addr_lo = lower_32_bits(el->dma_in);
677 	req->data_addr_hi = upper_32_bits(el->dma_in);
678 
679 	if (different_dest) {
680 		el->sgl_out = sgl_out;
681 		ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out,
682 						el->sgl_out,
683 						n_ents_out, info, gfp);
684 		if (ret)
685 			goto err_free_hw_sgl_in;
686 
687 		req->w0 |= SEC_BD_W0_DE;
688 		req->cipher_destin_addr_lo = lower_32_bits(el->dma_out);
689 		req->cipher_destin_addr_hi = upper_32_bits(el->dma_out);
690 
691 	} else {
692 		req->w0 &= ~SEC_BD_W0_DE;
693 		req->cipher_destin_addr_lo = lower_32_bits(el->dma_in);
694 		req->cipher_destin_addr_hi = upper_32_bits(el->dma_in);
695 	}
696 
697 	return el;
698 
699 err_free_hw_sgl_in:
700 	sec_free_hw_sgl(el->in, el->dma_in, info);
701 err_free_el:
702 	kfree(el);
703 
704 	return ERR_PTR(ret);
705 }
706 
707 static int sec_alg_skcipher_crypto(struct skcipher_request *skreq,
708 				   bool encrypt)
709 {
710 	struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
711 	struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
712 	struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
713 	struct sec_queue *queue = ctx->queue;
714 	struct sec_request *sec_req = skcipher_request_ctx(skreq);
715 	struct sec_dev_info *info = queue->dev_info;
716 	int i, ret, steps;
717 	size_t *split_sizes;
718 	struct scatterlist **splits_in;
719 	struct scatterlist **splits_out = NULL;
720 	int *splits_in_nents;
721 	int *splits_out_nents = NULL;
722 	struct sec_request_el *el, *temp;
723 	bool split = skreq->src != skreq->dst;
724 	gfp_t gfp = skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
725 
726 	mutex_init(&sec_req->lock);
727 	sec_req->req_base = &skreq->base;
728 	sec_req->err = 0;
729 	/* SGL mapping out here to allow us to break it up as necessary */
730 	sec_req->len_in = sg_nents(skreq->src);
731 
732 	ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes,
733 						 &steps, gfp);
734 	if (ret)
735 		return ret;
736 	sec_req->num_elements = steps;
737 	ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in,
738 				   &splits_in_nents, sec_req->len_in,
739 				   info->dev, gfp);
740 	if (ret)
741 		goto err_free_split_sizes;
742 
743 	if (split) {
744 		sec_req->len_out = sg_nents(skreq->dst);
745 		ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps,
746 					   &splits_out, &splits_out_nents,
747 					   sec_req->len_out, info->dev, gfp);
748 		if (ret)
749 			goto err_unmap_in_sg;
750 	}
751 	/* Shared info stored in seq_req - applies to all BDs */
752 	sec_req->tfm_ctx = ctx;
753 	sec_req->cb = sec_skcipher_alg_callback;
754 	INIT_LIST_HEAD(&sec_req->elements);
755 
756 	/*
757 	 * Future optimization.
758 	 * In the chaining case we can't use a dma pool bounce buffer
759 	 * but in the case where we know there is no chaining we can
760 	 */
761 	if (crypto_skcipher_ivsize(atfm)) {
762 		sec_req->dma_iv = dma_map_single(info->dev, skreq->iv,
763 						 crypto_skcipher_ivsize(atfm),
764 						 DMA_TO_DEVICE);
765 		if (dma_mapping_error(info->dev, sec_req->dma_iv)) {
766 			ret = -ENOMEM;
767 			goto err_unmap_out_sg;
768 		}
769 	}
770 
771 	/* Set them all up then queue - cleaner error handling. */
772 	for (i = 0; i < steps; i++) {
773 		el = sec_alg_alloc_and_fill_el(&ctx->req_template,
774 					       encrypt ? 1 : 0,
775 					       split_sizes[i],
776 					       skreq->src != skreq->dst,
777 					       splits_in[i], splits_in_nents[i],
778 					       split ? splits_out[i] : NULL,
779 					       split ? splits_out_nents[i] : 0,
780 					       info, gfp);
781 		if (IS_ERR(el)) {
782 			ret = PTR_ERR(el);
783 			goto err_free_elements;
784 		}
785 		el->req.cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv);
786 		el->req.cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv);
787 		el->sec_req = sec_req;
788 		list_add_tail(&el->head, &sec_req->elements);
789 	}
790 
791 	/*
792 	 * Only attempt to queue if the whole lot can fit in the queue -
793 	 * we can't successfully cleanup after a partial queing so this
794 	 * must succeed or fail atomically.
795 	 *
796 	 * Big hammer test of both software and hardware queues - could be
797 	 * more refined but this is unlikely to happen so no need.
798 	 */
799 
800 	/* Grab a big lock for a long time to avoid concurrency issues */
801 	spin_lock_bh(&queue->queuelock);
802 
803 	/*
804 	 * Can go on to queue if we have space in either:
805 	 * 1) The hardware queue and no software queue
806 	 * 2) The software queue
807 	 * AND there is nothing in the backlog.  If there is backlog we
808 	 * have to only queue to the backlog queue and return busy.
809 	 */
810 	if ((!sec_queue_can_enqueue(queue, steps) &&
811 	     (!queue->havesoftqueue ||
812 	      kfifo_avail(&queue->softqueue) > steps)) ||
813 	    !list_empty(&ctx->backlog)) {
814 		ret = -EBUSY;
815 		if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
816 			list_add_tail(&sec_req->backlog_head, &ctx->backlog);
817 			spin_unlock_bh(&queue->queuelock);
818 			goto out;
819 		}
820 
821 		spin_unlock_bh(&queue->queuelock);
822 		goto err_free_elements;
823 	}
824 	ret = sec_send_request(sec_req, queue);
825 	spin_unlock_bh(&queue->queuelock);
826 	if (ret)
827 		goto err_free_elements;
828 
829 	ret = -EINPROGRESS;
830 out:
831 	/* Cleanup - all elements in pointer arrays have been copied */
832 	kfree(splits_in_nents);
833 	kfree(splits_in);
834 	kfree(splits_out_nents);
835 	kfree(splits_out);
836 	kfree(split_sizes);
837 	return ret;
838 
839 err_free_elements:
840 	list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
841 		list_del(&el->head);
842 		sec_alg_free_el(el, info);
843 	}
844 	if (crypto_skcipher_ivsize(atfm))
845 		dma_unmap_single(info->dev, sec_req->dma_iv,
846 				 crypto_skcipher_ivsize(atfm),
847 				 DMA_BIDIRECTIONAL);
848 err_unmap_out_sg:
849 	if (split)
850 		sec_unmap_sg_on_err(skreq->dst, steps, splits_out,
851 				    splits_out_nents, sec_req->len_out,
852 				    info->dev);
853 err_unmap_in_sg:
854 	sec_unmap_sg_on_err(skreq->src, steps, splits_in, splits_in_nents,
855 			    sec_req->len_in, info->dev);
856 err_free_split_sizes:
857 	kfree(split_sizes);
858 
859 	return ret;
860 }
861 
862 static int sec_alg_skcipher_encrypt(struct skcipher_request *req)
863 {
864 	return sec_alg_skcipher_crypto(req, true);
865 }
866 
867 static int sec_alg_skcipher_decrypt(struct skcipher_request *req)
868 {
869 	return sec_alg_skcipher_crypto(req, false);
870 }
871 
872 static int sec_alg_skcipher_init(struct crypto_skcipher *tfm)
873 {
874 	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
875 
876 	mutex_init(&ctx->lock);
877 	INIT_LIST_HEAD(&ctx->backlog);
878 	crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_request));
879 
880 	ctx->queue = sec_queue_alloc_start_safe();
881 	if (IS_ERR(ctx->queue))
882 		return PTR_ERR(ctx->queue);
883 
884 	spin_lock_init(&ctx->queue->queuelock);
885 	ctx->queue->havesoftqueue = false;
886 
887 	return 0;
888 }
889 
890 static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm)
891 {
892 	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
893 	struct device *dev = ctx->queue->dev_info->dev;
894 
895 	if (ctx->key) {
896 		memzero_explicit(ctx->key, SEC_MAX_CIPHER_KEY);
897 		dma_free_coherent(dev, SEC_MAX_CIPHER_KEY, ctx->key,
898 				  ctx->pkey);
899 	}
900 	sec_queue_stop_release(ctx->queue);
901 }
902 
903 static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm)
904 {
905 	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
906 	int ret;
907 
908 	ret = sec_alg_skcipher_init(tfm);
909 	if (ret)
910 		return ret;
911 
912 	INIT_KFIFO(ctx->queue->softqueue);
913 	ret = kfifo_alloc(&ctx->queue->softqueue, 512, GFP_KERNEL);
914 	if (ret) {
915 		sec_alg_skcipher_exit(tfm);
916 		return ret;
917 	}
918 	ctx->queue->havesoftqueue = true;
919 
920 	return 0;
921 }
922 
923 static void sec_alg_skcipher_exit_with_queue(struct crypto_skcipher *tfm)
924 {
925 	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
926 
927 	kfifo_free(&ctx->queue->softqueue);
928 	sec_alg_skcipher_exit(tfm);
929 }
930 
931 static struct skcipher_alg sec_algs[] = {
932 	{
933 		.base = {
934 			.cra_name = "ecb(aes)",
935 			.cra_driver_name = "hisi_sec_aes_ecb",
936 			.cra_priority = 4001,
937 			.cra_flags = CRYPTO_ALG_ASYNC |
938 				     CRYPTO_ALG_ALLOCATES_MEMORY,
939 			.cra_blocksize = AES_BLOCK_SIZE,
940 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
941 			.cra_alignmask = 0,
942 			.cra_module = THIS_MODULE,
943 		},
944 		.init = sec_alg_skcipher_init,
945 		.exit = sec_alg_skcipher_exit,
946 		.setkey = sec_alg_skcipher_setkey_aes_ecb,
947 		.decrypt = sec_alg_skcipher_decrypt,
948 		.encrypt = sec_alg_skcipher_encrypt,
949 		.min_keysize = AES_MIN_KEY_SIZE,
950 		.max_keysize = AES_MAX_KEY_SIZE,
951 		.ivsize = 0,
952 	}, {
953 		.base = {
954 			.cra_name = "cbc(aes)",
955 			.cra_driver_name = "hisi_sec_aes_cbc",
956 			.cra_priority = 4001,
957 			.cra_flags = CRYPTO_ALG_ASYNC |
958 				     CRYPTO_ALG_ALLOCATES_MEMORY,
959 			.cra_blocksize = AES_BLOCK_SIZE,
960 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
961 			.cra_alignmask = 0,
962 			.cra_module = THIS_MODULE,
963 		},
964 		.init = sec_alg_skcipher_init_with_queue,
965 		.exit = sec_alg_skcipher_exit_with_queue,
966 		.setkey = sec_alg_skcipher_setkey_aes_cbc,
967 		.decrypt = sec_alg_skcipher_decrypt,
968 		.encrypt = sec_alg_skcipher_encrypt,
969 		.min_keysize = AES_MIN_KEY_SIZE,
970 		.max_keysize = AES_MAX_KEY_SIZE,
971 		.ivsize = AES_BLOCK_SIZE,
972 	}, {
973 		.base = {
974 			.cra_name = "ctr(aes)",
975 			.cra_driver_name = "hisi_sec_aes_ctr",
976 			.cra_priority = 4001,
977 			.cra_flags = CRYPTO_ALG_ASYNC |
978 				     CRYPTO_ALG_ALLOCATES_MEMORY,
979 			.cra_blocksize = AES_BLOCK_SIZE,
980 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
981 			.cra_alignmask = 0,
982 			.cra_module = THIS_MODULE,
983 		},
984 		.init = sec_alg_skcipher_init_with_queue,
985 		.exit = sec_alg_skcipher_exit_with_queue,
986 		.setkey = sec_alg_skcipher_setkey_aes_ctr,
987 		.decrypt = sec_alg_skcipher_decrypt,
988 		.encrypt = sec_alg_skcipher_encrypt,
989 		.min_keysize = AES_MIN_KEY_SIZE,
990 		.max_keysize = AES_MAX_KEY_SIZE,
991 		.ivsize = AES_BLOCK_SIZE,
992 	}, {
993 		.base = {
994 			.cra_name = "xts(aes)",
995 			.cra_driver_name = "hisi_sec_aes_xts",
996 			.cra_priority = 4001,
997 			.cra_flags = CRYPTO_ALG_ASYNC |
998 				     CRYPTO_ALG_ALLOCATES_MEMORY,
999 			.cra_blocksize = AES_BLOCK_SIZE,
1000 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1001 			.cra_alignmask = 0,
1002 			.cra_module = THIS_MODULE,
1003 		},
1004 		.init = sec_alg_skcipher_init,
1005 		.exit = sec_alg_skcipher_exit,
1006 		.setkey = sec_alg_skcipher_setkey_aes_xts,
1007 		.decrypt = sec_alg_skcipher_decrypt,
1008 		.encrypt = sec_alg_skcipher_encrypt,
1009 		.min_keysize = 2 * AES_MIN_KEY_SIZE,
1010 		.max_keysize = 2 * AES_MAX_KEY_SIZE,
1011 		.ivsize = AES_BLOCK_SIZE,
1012 	}, {
1013 	/* Unable to find any test vectors so untested */
1014 		.base = {
1015 			.cra_name = "ecb(des)",
1016 			.cra_driver_name = "hisi_sec_des_ecb",
1017 			.cra_priority = 4001,
1018 			.cra_flags = CRYPTO_ALG_ASYNC |
1019 				     CRYPTO_ALG_ALLOCATES_MEMORY,
1020 			.cra_blocksize = DES_BLOCK_SIZE,
1021 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1022 			.cra_alignmask = 0,
1023 			.cra_module = THIS_MODULE,
1024 		},
1025 		.init = sec_alg_skcipher_init,
1026 		.exit = sec_alg_skcipher_exit,
1027 		.setkey = sec_alg_skcipher_setkey_des_ecb,
1028 		.decrypt = sec_alg_skcipher_decrypt,
1029 		.encrypt = sec_alg_skcipher_encrypt,
1030 		.min_keysize = DES_KEY_SIZE,
1031 		.max_keysize = DES_KEY_SIZE,
1032 		.ivsize = 0,
1033 	}, {
1034 		.base = {
1035 			.cra_name = "cbc(des)",
1036 			.cra_driver_name = "hisi_sec_des_cbc",
1037 			.cra_priority = 4001,
1038 			.cra_flags = CRYPTO_ALG_ASYNC |
1039 				     CRYPTO_ALG_ALLOCATES_MEMORY,
1040 			.cra_blocksize = DES_BLOCK_SIZE,
1041 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1042 			.cra_alignmask = 0,
1043 			.cra_module = THIS_MODULE,
1044 		},
1045 		.init = sec_alg_skcipher_init_with_queue,
1046 		.exit = sec_alg_skcipher_exit_with_queue,
1047 		.setkey = sec_alg_skcipher_setkey_des_cbc,
1048 		.decrypt = sec_alg_skcipher_decrypt,
1049 		.encrypt = sec_alg_skcipher_encrypt,
1050 		.min_keysize = DES_KEY_SIZE,
1051 		.max_keysize = DES_KEY_SIZE,
1052 		.ivsize = DES_BLOCK_SIZE,
1053 	}, {
1054 		.base = {
1055 			.cra_name = "cbc(des3_ede)",
1056 			.cra_driver_name = "hisi_sec_3des_cbc",
1057 			.cra_priority = 4001,
1058 			.cra_flags = CRYPTO_ALG_ASYNC |
1059 				     CRYPTO_ALG_ALLOCATES_MEMORY,
1060 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1061 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1062 			.cra_alignmask = 0,
1063 			.cra_module = THIS_MODULE,
1064 		},
1065 		.init = sec_alg_skcipher_init_with_queue,
1066 		.exit = sec_alg_skcipher_exit_with_queue,
1067 		.setkey = sec_alg_skcipher_setkey_3des_cbc,
1068 		.decrypt = sec_alg_skcipher_decrypt,
1069 		.encrypt = sec_alg_skcipher_encrypt,
1070 		.min_keysize = DES3_EDE_KEY_SIZE,
1071 		.max_keysize = DES3_EDE_KEY_SIZE,
1072 		.ivsize = DES3_EDE_BLOCK_SIZE,
1073 	}, {
1074 		.base = {
1075 			.cra_name = "ecb(des3_ede)",
1076 			.cra_driver_name = "hisi_sec_3des_ecb",
1077 			.cra_priority = 4001,
1078 			.cra_flags = CRYPTO_ALG_ASYNC |
1079 				     CRYPTO_ALG_ALLOCATES_MEMORY,
1080 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1081 			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1082 			.cra_alignmask = 0,
1083 			.cra_module = THIS_MODULE,
1084 		},
1085 		.init = sec_alg_skcipher_init,
1086 		.exit = sec_alg_skcipher_exit,
1087 		.setkey = sec_alg_skcipher_setkey_3des_ecb,
1088 		.decrypt = sec_alg_skcipher_decrypt,
1089 		.encrypt = sec_alg_skcipher_encrypt,
1090 		.min_keysize = DES3_EDE_KEY_SIZE,
1091 		.max_keysize = DES3_EDE_KEY_SIZE,
1092 		.ivsize = 0,
1093 	}
1094 };
1095 
1096 int sec_algs_register(void)
1097 {
1098 	int ret = 0;
1099 
1100 	mutex_lock(&algs_lock);
1101 	if (++active_devs != 1)
1102 		goto unlock;
1103 
1104 	ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1105 	if (ret)
1106 		--active_devs;
1107 unlock:
1108 	mutex_unlock(&algs_lock);
1109 
1110 	return ret;
1111 }
1112 
1113 void sec_algs_unregister(void)
1114 {
1115 	mutex_lock(&algs_lock);
1116 	if (--active_devs != 0)
1117 		goto unlock;
1118 	crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1119 
1120 unlock:
1121 	mutex_unlock(&algs_lock);
1122 }
1123