xref: /openbmc/linux/drivers/crypto/ccree/cc_cipher.c (revision 0ee43367)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
3 
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <crypto/algapi.h>
7 #include <crypto/internal/skcipher.h>
8 #include <crypto/internal/des.h>
9 #include <crypto/xts.h>
10 #include <crypto/sm4.h>
11 #include <crypto/scatterwalk.h>
12 
13 #include "cc_driver.h"
14 #include "cc_lli_defs.h"
15 #include "cc_buffer_mgr.h"
16 #include "cc_cipher.h"
17 #include "cc_request_mgr.h"
18 
19 #define MAX_SKCIPHER_SEQ_LEN 6
20 
21 #define template_skcipher	template_u.skcipher
22 
23 struct cc_user_key_info {
24 	u8 *key;
25 	dma_addr_t key_dma_addr;
26 };
27 
28 struct cc_hw_key_info {
29 	enum cc_hw_crypto_key key1_slot;
30 	enum cc_hw_crypto_key key2_slot;
31 };
32 
33 struct cc_cpp_key_info {
34 	u8 slot;
35 	enum cc_cpp_alg alg;
36 };
37 
38 enum cc_key_type {
39 	CC_UNPROTECTED_KEY,		/* User key */
40 	CC_HW_PROTECTED_KEY,		/* HW (FDE) key */
41 	CC_POLICY_PROTECTED_KEY,	/* CPP key */
42 	CC_INVALID_PROTECTED_KEY	/* Invalid key */
43 };
44 
45 struct cc_cipher_ctx {
46 	struct cc_drvdata *drvdata;
47 	int keylen;
48 	int cipher_mode;
49 	int flow_mode;
50 	unsigned int flags;
51 	enum cc_key_type key_type;
52 	struct cc_user_key_info user;
53 	union {
54 		struct cc_hw_key_info hw;
55 		struct cc_cpp_key_info cpp;
56 	};
57 	struct crypto_shash *shash_tfm;
58 	struct crypto_skcipher *fallback_tfm;
59 	bool fallback_on;
60 };
61 
62 static void cc_cipher_complete(struct device *dev, void *cc_req, int err);
63 
cc_key_type(struct crypto_tfm * tfm)64 static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm)
65 {
66 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
67 
68 	return ctx_p->key_type;
69 }
70 
validate_keys_sizes(struct cc_cipher_ctx * ctx_p,u32 size)71 static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size)
72 {
73 	switch (ctx_p->flow_mode) {
74 	case S_DIN_to_AES:
75 		switch (size) {
76 		case CC_AES_128_BIT_KEY_SIZE:
77 		case CC_AES_192_BIT_KEY_SIZE:
78 			if (ctx_p->cipher_mode != DRV_CIPHER_XTS)
79 				return 0;
80 			break;
81 		case CC_AES_256_BIT_KEY_SIZE:
82 			return 0;
83 		case (CC_AES_192_BIT_KEY_SIZE * 2):
84 		case (CC_AES_256_BIT_KEY_SIZE * 2):
85 			if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
86 			    ctx_p->cipher_mode == DRV_CIPHER_ESSIV)
87 				return 0;
88 			break;
89 		default:
90 			break;
91 		}
92 		break;
93 	case S_DIN_to_DES:
94 		if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE)
95 			return 0;
96 		break;
97 	case S_DIN_to_SM4:
98 		if (size == SM4_KEY_SIZE)
99 			return 0;
100 		break;
101 	default:
102 		break;
103 	}
104 	return -EINVAL;
105 }
106 
validate_data_size(struct cc_cipher_ctx * ctx_p,unsigned int size)107 static int validate_data_size(struct cc_cipher_ctx *ctx_p,
108 			      unsigned int size)
109 {
110 	switch (ctx_p->flow_mode) {
111 	case S_DIN_to_AES:
112 		switch (ctx_p->cipher_mode) {
113 		case DRV_CIPHER_XTS:
114 		case DRV_CIPHER_CBC_CTS:
115 			if (size >= AES_BLOCK_SIZE)
116 				return 0;
117 			break;
118 		case DRV_CIPHER_OFB:
119 		case DRV_CIPHER_CTR:
120 				return 0;
121 		case DRV_CIPHER_ECB:
122 		case DRV_CIPHER_CBC:
123 		case DRV_CIPHER_ESSIV:
124 			if (IS_ALIGNED(size, AES_BLOCK_SIZE))
125 				return 0;
126 			break;
127 		default:
128 			break;
129 		}
130 		break;
131 	case S_DIN_to_DES:
132 		if (IS_ALIGNED(size, DES_BLOCK_SIZE))
133 			return 0;
134 		break;
135 	case S_DIN_to_SM4:
136 		switch (ctx_p->cipher_mode) {
137 		case DRV_CIPHER_CTR:
138 			return 0;
139 		case DRV_CIPHER_ECB:
140 		case DRV_CIPHER_CBC:
141 			if (IS_ALIGNED(size, SM4_BLOCK_SIZE))
142 				return 0;
143 			break;
144 		default:
145 			break;
146 		}
147 		break;
148 	default:
149 		break;
150 	}
151 	return -EINVAL;
152 }
153 
cc_cipher_init(struct crypto_tfm * tfm)154 static int cc_cipher_init(struct crypto_tfm *tfm)
155 {
156 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
157 	struct cc_crypto_alg *cc_alg =
158 			container_of(tfm->__crt_alg, struct cc_crypto_alg,
159 				     skcipher_alg.base);
160 	struct device *dev = drvdata_to_dev(cc_alg->drvdata);
161 	unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
162 	unsigned int fallback_req_size = 0;
163 
164 	dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p,
165 		crypto_tfm_alg_name(tfm));
166 
167 	ctx_p->cipher_mode = cc_alg->cipher_mode;
168 	ctx_p->flow_mode = cc_alg->flow_mode;
169 	ctx_p->drvdata = cc_alg->drvdata;
170 
171 	if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
172 		const char *name = crypto_tfm_alg_name(tfm);
173 
174 		/* Alloc hash tfm for essiv */
175 		ctx_p->shash_tfm = crypto_alloc_shash("sha256", 0, 0);
176 		if (IS_ERR(ctx_p->shash_tfm)) {
177 			dev_err(dev, "Error allocating hash tfm for ESSIV.\n");
178 			return PTR_ERR(ctx_p->shash_tfm);
179 		}
180 		max_key_buf_size <<= 1;
181 
182 		/* Alloc fallabck tfm or essiv when key size != 256 bit */
183 		ctx_p->fallback_tfm =
184 			crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC);
185 
186 		if (IS_ERR(ctx_p->fallback_tfm)) {
187 			/* Note we're still allowing registration with no fallback since it's
188 			 * better to have most modes supported than none at all.
189 			 */
190 			dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n",
191 			       name);
192 			ctx_p->fallback_tfm = NULL;
193 		} else {
194 			fallback_req_size = crypto_skcipher_reqsize(ctx_p->fallback_tfm);
195 		}
196 	}
197 
198 	crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
199 				    sizeof(struct cipher_req_ctx) + fallback_req_size);
200 
201 	/* Allocate key buffer, cache line aligned */
202 	ctx_p->user.key = kzalloc(max_key_buf_size, GFP_KERNEL);
203 	if (!ctx_p->user.key)
204 		goto free_fallback;
205 
206 	dev_dbg(dev, "Allocated key buffer in context. key=@%p\n",
207 		ctx_p->user.key);
208 
209 	/* Map key buffer */
210 	ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key,
211 						  max_key_buf_size,
212 						  DMA_TO_DEVICE);
213 	if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) {
214 		dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n",
215 			max_key_buf_size, ctx_p->user.key);
216 		goto free_key;
217 	}
218 	dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n",
219 		max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr);
220 
221 	return 0;
222 
223 free_key:
224 	kfree(ctx_p->user.key);
225 free_fallback:
226 	crypto_free_skcipher(ctx_p->fallback_tfm);
227 	crypto_free_shash(ctx_p->shash_tfm);
228 
229 	return -ENOMEM;
230 }
231 
cc_cipher_exit(struct crypto_tfm * tfm)232 static void cc_cipher_exit(struct crypto_tfm *tfm)
233 {
234 	struct crypto_alg *alg = tfm->__crt_alg;
235 	struct cc_crypto_alg *cc_alg =
236 			container_of(alg, struct cc_crypto_alg,
237 				     skcipher_alg.base);
238 	unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
239 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
240 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
241 
242 	dev_dbg(dev, "Clearing context @%p for %s\n",
243 		crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm));
244 
245 	if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
246 		/* Free hash tfm for essiv */
247 		crypto_free_shash(ctx_p->shash_tfm);
248 		ctx_p->shash_tfm = NULL;
249 		crypto_free_skcipher(ctx_p->fallback_tfm);
250 		ctx_p->fallback_tfm = NULL;
251 	}
252 
253 	/* Unmap key buffer */
254 	dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size,
255 			 DMA_TO_DEVICE);
256 	dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n",
257 		&ctx_p->user.key_dma_addr);
258 
259 	/* Free key buffer in context */
260 	dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key);
261 	kfree_sensitive(ctx_p->user.key);
262 }
263 
264 struct tdes_keys {
265 	u8	key1[DES_KEY_SIZE];
266 	u8	key2[DES_KEY_SIZE];
267 	u8	key3[DES_KEY_SIZE];
268 };
269 
cc_slot_to_hw_key(u8 slot_num)270 static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num)
271 {
272 	switch (slot_num) {
273 	case 0:
274 		return KFDE0_KEY;
275 	case 1:
276 		return KFDE1_KEY;
277 	case 2:
278 		return KFDE2_KEY;
279 	case 3:
280 		return KFDE3_KEY;
281 	}
282 	return END_OF_KEYS;
283 }
284 
cc_slot_to_cpp_key(u8 slot_num)285 static u8 cc_slot_to_cpp_key(u8 slot_num)
286 {
287 	return (slot_num - CC_FIRST_CPP_KEY_SLOT);
288 }
289 
cc_slot_to_key_type(u8 slot_num)290 static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num)
291 {
292 	if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT)
293 		return CC_HW_PROTECTED_KEY;
294 	else if (slot_num >=  CC_FIRST_CPP_KEY_SLOT &&
295 		 slot_num <=  CC_LAST_CPP_KEY_SLOT)
296 		return CC_POLICY_PROTECTED_KEY;
297 	else
298 		return CC_INVALID_PROTECTED_KEY;
299 }
300 
cc_cipher_sethkey(struct crypto_skcipher * sktfm,const u8 * key,unsigned int keylen)301 static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key,
302 			     unsigned int keylen)
303 {
304 	struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm);
305 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
306 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
307 	struct cc_hkey_info hki;
308 
309 	dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n",
310 		ctx_p, crypto_tfm_alg_name(tfm), keylen);
311 	dump_byte_array("key", key, keylen);
312 
313 	/* STAT_PHASE_0: Init and sanity checks */
314 
315 	/* This check the size of the protected key token */
316 	if (keylen != sizeof(hki)) {
317 		dev_err(dev, "Unsupported protected key size %d.\n", keylen);
318 		return -EINVAL;
319 	}
320 
321 	memcpy(&hki, key, keylen);
322 
323 	/* The real key len for crypto op is the size of the HW key
324 	 * referenced by the HW key slot, not the hardware key token
325 	 */
326 	keylen = hki.keylen;
327 
328 	if (validate_keys_sizes(ctx_p, keylen)) {
329 		dev_dbg(dev, "Unsupported key size %d.\n", keylen);
330 		return -EINVAL;
331 	}
332 
333 	ctx_p->keylen = keylen;
334 	ctx_p->fallback_on = false;
335 
336 	switch (cc_slot_to_key_type(hki.hw_key1)) {
337 	case CC_HW_PROTECTED_KEY:
338 		if (ctx_p->flow_mode == S_DIN_to_SM4) {
339 			dev_err(dev, "Only AES HW protected keys are supported\n");
340 			return -EINVAL;
341 		}
342 
343 		ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1);
344 		if (ctx_p->hw.key1_slot == END_OF_KEYS) {
345 			dev_err(dev, "Unsupported hw key1 number (%d)\n",
346 				hki.hw_key1);
347 			return -EINVAL;
348 		}
349 
350 		if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
351 		    ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
352 			if (hki.hw_key1 == hki.hw_key2) {
353 				dev_err(dev, "Illegal hw key numbers (%d,%d)\n",
354 					hki.hw_key1, hki.hw_key2);
355 				return -EINVAL;
356 			}
357 
358 			ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2);
359 			if (ctx_p->hw.key2_slot == END_OF_KEYS) {
360 				dev_err(dev, "Unsupported hw key2 number (%d)\n",
361 					hki.hw_key2);
362 				return -EINVAL;
363 			}
364 		}
365 
366 		ctx_p->key_type = CC_HW_PROTECTED_KEY;
367 		dev_dbg(dev, "HW protected key  %d/%d set\n.",
368 			ctx_p->hw.key1_slot, ctx_p->hw.key2_slot);
369 		break;
370 
371 	case CC_POLICY_PROTECTED_KEY:
372 		if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) {
373 			dev_err(dev, "CPP keys not supported in this hardware revision.\n");
374 			return -EINVAL;
375 		}
376 
377 		if (ctx_p->cipher_mode != DRV_CIPHER_CBC &&
378 		    ctx_p->cipher_mode != DRV_CIPHER_CTR) {
379 			dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n");
380 			return -EINVAL;
381 		}
382 
383 		ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1);
384 		if (ctx_p->flow_mode == S_DIN_to_AES)
385 			ctx_p->cpp.alg = CC_CPP_AES;
386 		else /* Must be SM4 since due to sethkey registration */
387 			ctx_p->cpp.alg = CC_CPP_SM4;
388 		ctx_p->key_type = CC_POLICY_PROTECTED_KEY;
389 		dev_dbg(dev, "policy protected key alg: %d slot: %d.\n",
390 			ctx_p->cpp.alg, ctx_p->cpp.slot);
391 		break;
392 
393 	default:
394 		dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1);
395 		return -EINVAL;
396 	}
397 
398 	return 0;
399 }
400 
cc_cipher_setkey(struct crypto_skcipher * sktfm,const u8 * key,unsigned int keylen)401 static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
402 			    unsigned int keylen)
403 {
404 	struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm);
405 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
406 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
407 	struct cc_crypto_alg *cc_alg =
408 			container_of(tfm->__crt_alg, struct cc_crypto_alg,
409 				     skcipher_alg.base);
410 	unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
411 
412 	dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n",
413 		ctx_p, crypto_tfm_alg_name(tfm), keylen);
414 	dump_byte_array("key", key, keylen);
415 
416 	/* STAT_PHASE_0: Init and sanity checks */
417 
418 	if (validate_keys_sizes(ctx_p, keylen)) {
419 		dev_dbg(dev, "Invalid key size %d.\n", keylen);
420 		return -EINVAL;
421 	}
422 
423 	if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
424 
425 		/* We only support 256 bit ESSIV-CBC-AES keys */
426 		if (keylen != AES_KEYSIZE_256)  {
427 			unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK;
428 
429 			if (likely(ctx_p->fallback_tfm)) {
430 				ctx_p->fallback_on = true;
431 				crypto_skcipher_clear_flags(ctx_p->fallback_tfm,
432 							    CRYPTO_TFM_REQ_MASK);
433 				crypto_skcipher_clear_flags(ctx_p->fallback_tfm, flags);
434 				return crypto_skcipher_setkey(ctx_p->fallback_tfm, key, keylen);
435 			}
436 
437 			dev_dbg(dev, "Unsupported key size %d and no fallback.\n", keylen);
438 			return -EINVAL;
439 		}
440 
441 		/* Internal ESSIV key buffer is double sized */
442 		max_key_buf_size <<= 1;
443 	}
444 
445 	ctx_p->fallback_on = false;
446 	ctx_p->key_type = CC_UNPROTECTED_KEY;
447 
448 	/*
449 	 * Verify DES weak keys
450 	 * Note that we're dropping the expanded key since the
451 	 * HW does the expansion on its own.
452 	 */
453 	if (ctx_p->flow_mode == S_DIN_to_DES) {
454 		if ((keylen == DES3_EDE_KEY_SIZE &&
455 		     verify_skcipher_des3_key(sktfm, key)) ||
456 		    verify_skcipher_des_key(sktfm, key)) {
457 			dev_dbg(dev, "weak DES key");
458 			return -EINVAL;
459 		}
460 	}
461 
462 	if (ctx_p->cipher_mode == DRV_CIPHER_XTS &&
463 	    xts_verify_key(sktfm, key, keylen)) {
464 		dev_dbg(dev, "weak XTS key");
465 		return -EINVAL;
466 	}
467 
468 	/* STAT_PHASE_1: Copy key to ctx */
469 	dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr,
470 				max_key_buf_size, DMA_TO_DEVICE);
471 
472 	memcpy(ctx_p->user.key, key, keylen);
473 
474 	if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
475 		/* sha256 for key2 - use sw implementation */
476 		int err;
477 
478 		err = crypto_shash_tfm_digest(ctx_p->shash_tfm,
479 					      ctx_p->user.key, keylen,
480 					      ctx_p->user.key + keylen);
481 		if (err) {
482 			dev_err(dev, "Failed to hash ESSIV key.\n");
483 			return err;
484 		}
485 
486 		keylen <<= 1;
487 	}
488 	dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr,
489 				   max_key_buf_size, DMA_TO_DEVICE);
490 	ctx_p->keylen = keylen;
491 
492 	dev_dbg(dev, "return safely");
493 	return 0;
494 }
495 
cc_out_setup_mode(struct cc_cipher_ctx * ctx_p)496 static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p)
497 {
498 	switch (ctx_p->flow_mode) {
499 	case S_DIN_to_AES:
500 		return S_AES_to_DOUT;
501 	case S_DIN_to_DES:
502 		return S_DES_to_DOUT;
503 	case S_DIN_to_SM4:
504 		return S_SM4_to_DOUT;
505 	default:
506 		return ctx_p->flow_mode;
507 	}
508 }
509 
cc_setup_readiv_desc(struct crypto_tfm * tfm,struct cipher_req_ctx * req_ctx,unsigned int ivsize,struct cc_hw_desc desc[],unsigned int * seq_size)510 static void cc_setup_readiv_desc(struct crypto_tfm *tfm,
511 				 struct cipher_req_ctx *req_ctx,
512 				 unsigned int ivsize, struct cc_hw_desc desc[],
513 				 unsigned int *seq_size)
514 {
515 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
516 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
517 	int cipher_mode = ctx_p->cipher_mode;
518 	int flow_mode = cc_out_setup_mode(ctx_p);
519 	int direction = req_ctx->gen_ctx.op_type;
520 	dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
521 
522 	if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY)
523 		return;
524 
525 	switch (cipher_mode) {
526 	case DRV_CIPHER_ECB:
527 		break;
528 	case DRV_CIPHER_CBC:
529 	case DRV_CIPHER_CBC_CTS:
530 	case DRV_CIPHER_CTR:
531 	case DRV_CIPHER_OFB:
532 		/* Read next IV */
533 		hw_desc_init(&desc[*seq_size]);
534 		set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1);
535 		set_cipher_config0(&desc[*seq_size], direction);
536 		set_flow_mode(&desc[*seq_size], flow_mode);
537 		set_cipher_mode(&desc[*seq_size], cipher_mode);
538 		if (cipher_mode == DRV_CIPHER_CTR ||
539 		    cipher_mode == DRV_CIPHER_OFB) {
540 			set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1);
541 		} else {
542 			set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0);
543 		}
544 		set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
545 		(*seq_size)++;
546 		break;
547 	case DRV_CIPHER_XTS:
548 	case DRV_CIPHER_ESSIV:
549 		/*  IV */
550 		hw_desc_init(&desc[*seq_size]);
551 		set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1);
552 		set_cipher_mode(&desc[*seq_size], cipher_mode);
553 		set_cipher_config0(&desc[*seq_size], direction);
554 		set_flow_mode(&desc[*seq_size], flow_mode);
555 		set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE,
556 			     NS_BIT, 1);
557 		set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
558 		(*seq_size)++;
559 		break;
560 	default:
561 		dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
562 	}
563 }
564 
565 
cc_setup_state_desc(struct crypto_tfm * tfm,struct cipher_req_ctx * req_ctx,unsigned int ivsize,unsigned int nbytes,struct cc_hw_desc desc[],unsigned int * seq_size)566 static void cc_setup_state_desc(struct crypto_tfm *tfm,
567 				 struct cipher_req_ctx *req_ctx,
568 				 unsigned int ivsize, unsigned int nbytes,
569 				 struct cc_hw_desc desc[],
570 				 unsigned int *seq_size)
571 {
572 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
573 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
574 	int cipher_mode = ctx_p->cipher_mode;
575 	int flow_mode = ctx_p->flow_mode;
576 	int direction = req_ctx->gen_ctx.op_type;
577 	dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
578 
579 	switch (cipher_mode) {
580 	case DRV_CIPHER_ECB:
581 		break;
582 	case DRV_CIPHER_CBC:
583 	case DRV_CIPHER_CBC_CTS:
584 	case DRV_CIPHER_CTR:
585 	case DRV_CIPHER_OFB:
586 		/* Load IV */
587 		hw_desc_init(&desc[*seq_size]);
588 		set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize,
589 			     NS_BIT);
590 		set_cipher_config0(&desc[*seq_size], direction);
591 		set_flow_mode(&desc[*seq_size], flow_mode);
592 		set_cipher_mode(&desc[*seq_size], cipher_mode);
593 		if (cipher_mode == DRV_CIPHER_CTR ||
594 		    cipher_mode == DRV_CIPHER_OFB) {
595 			set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1);
596 		} else {
597 			set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0);
598 		}
599 		(*seq_size)++;
600 		break;
601 	case DRV_CIPHER_XTS:
602 	case DRV_CIPHER_ESSIV:
603 		break;
604 	default:
605 		dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
606 	}
607 }
608 
609 
cc_setup_xex_state_desc(struct crypto_tfm * tfm,struct cipher_req_ctx * req_ctx,unsigned int ivsize,unsigned int nbytes,struct cc_hw_desc desc[],unsigned int * seq_size)610 static void cc_setup_xex_state_desc(struct crypto_tfm *tfm,
611 				 struct cipher_req_ctx *req_ctx,
612 				 unsigned int ivsize, unsigned int nbytes,
613 				 struct cc_hw_desc desc[],
614 				 unsigned int *seq_size)
615 {
616 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
617 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
618 	int cipher_mode = ctx_p->cipher_mode;
619 	int flow_mode = ctx_p->flow_mode;
620 	int direction = req_ctx->gen_ctx.op_type;
621 	dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr;
622 	unsigned int key_len = (ctx_p->keylen / 2);
623 	dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
624 	unsigned int key_offset = key_len;
625 
626 	switch (cipher_mode) {
627 	case DRV_CIPHER_ECB:
628 		break;
629 	case DRV_CIPHER_CBC:
630 	case DRV_CIPHER_CBC_CTS:
631 	case DRV_CIPHER_CTR:
632 	case DRV_CIPHER_OFB:
633 		break;
634 	case DRV_CIPHER_XTS:
635 	case DRV_CIPHER_ESSIV:
636 
637 		if (cipher_mode == DRV_CIPHER_ESSIV)
638 			key_len = SHA256_DIGEST_SIZE;
639 
640 		/* load XEX key */
641 		hw_desc_init(&desc[*seq_size]);
642 		set_cipher_mode(&desc[*seq_size], cipher_mode);
643 		set_cipher_config0(&desc[*seq_size], direction);
644 		if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) {
645 			set_hw_crypto_key(&desc[*seq_size],
646 					  ctx_p->hw.key2_slot);
647 		} else {
648 			set_din_type(&desc[*seq_size], DMA_DLLI,
649 				     (key_dma_addr + key_offset),
650 				     key_len, NS_BIT);
651 		}
652 		set_xex_data_unit_size(&desc[*seq_size], nbytes);
653 		set_flow_mode(&desc[*seq_size], S_DIN_to_AES2);
654 		set_key_size_aes(&desc[*seq_size], key_len);
655 		set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY);
656 		(*seq_size)++;
657 
658 		/* Load IV */
659 		hw_desc_init(&desc[*seq_size]);
660 		set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1);
661 		set_cipher_mode(&desc[*seq_size], cipher_mode);
662 		set_cipher_config0(&desc[*seq_size], direction);
663 		set_key_size_aes(&desc[*seq_size], key_len);
664 		set_flow_mode(&desc[*seq_size], flow_mode);
665 		set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr,
666 			     CC_AES_BLOCK_SIZE, NS_BIT);
667 		(*seq_size)++;
668 		break;
669 	default:
670 		dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
671 	}
672 }
673 
cc_out_flow_mode(struct cc_cipher_ctx * ctx_p)674 static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p)
675 {
676 	switch (ctx_p->flow_mode) {
677 	case S_DIN_to_AES:
678 		return DIN_AES_DOUT;
679 	case S_DIN_to_DES:
680 		return DIN_DES_DOUT;
681 	case S_DIN_to_SM4:
682 		return DIN_SM4_DOUT;
683 	default:
684 		return ctx_p->flow_mode;
685 	}
686 }
687 
cc_setup_key_desc(struct crypto_tfm * tfm,struct cipher_req_ctx * req_ctx,unsigned int nbytes,struct cc_hw_desc desc[],unsigned int * seq_size)688 static void cc_setup_key_desc(struct crypto_tfm *tfm,
689 			      struct cipher_req_ctx *req_ctx,
690 			      unsigned int nbytes, struct cc_hw_desc desc[],
691 			      unsigned int *seq_size)
692 {
693 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
694 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
695 	int cipher_mode = ctx_p->cipher_mode;
696 	int flow_mode = ctx_p->flow_mode;
697 	int direction = req_ctx->gen_ctx.op_type;
698 	dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr;
699 	unsigned int key_len = ctx_p->keylen;
700 	unsigned int din_size;
701 
702 	switch (cipher_mode) {
703 	case DRV_CIPHER_CBC:
704 	case DRV_CIPHER_CBC_CTS:
705 	case DRV_CIPHER_CTR:
706 	case DRV_CIPHER_OFB:
707 	case DRV_CIPHER_ECB:
708 		/* Load key */
709 		hw_desc_init(&desc[*seq_size]);
710 		set_cipher_mode(&desc[*seq_size], cipher_mode);
711 		set_cipher_config0(&desc[*seq_size], direction);
712 
713 		if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) {
714 			/* We use the AES key size coding for all CPP algs */
715 			set_key_size_aes(&desc[*seq_size], key_len);
716 			set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot);
717 			flow_mode = cc_out_flow_mode(ctx_p);
718 		} else {
719 			if (flow_mode == S_DIN_to_AES) {
720 				if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) {
721 					set_hw_crypto_key(&desc[*seq_size],
722 							  ctx_p->hw.key1_slot);
723 				} else {
724 					/* CC_POLICY_UNPROTECTED_KEY
725 					 * Invalid keys are filtered out in
726 					 * sethkey()
727 					 */
728 					din_size = (key_len == 24) ?
729 						AES_MAX_KEY_SIZE : key_len;
730 
731 					set_din_type(&desc[*seq_size], DMA_DLLI,
732 						     key_dma_addr, din_size,
733 						     NS_BIT);
734 				}
735 				set_key_size_aes(&desc[*seq_size], key_len);
736 			} else {
737 				/*des*/
738 				set_din_type(&desc[*seq_size], DMA_DLLI,
739 					     key_dma_addr, key_len, NS_BIT);
740 				set_key_size_des(&desc[*seq_size], key_len);
741 			}
742 			set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0);
743 		}
744 		set_flow_mode(&desc[*seq_size], flow_mode);
745 		(*seq_size)++;
746 		break;
747 	case DRV_CIPHER_XTS:
748 	case DRV_CIPHER_ESSIV:
749 		/* Load AES key */
750 		hw_desc_init(&desc[*seq_size]);
751 		set_cipher_mode(&desc[*seq_size], cipher_mode);
752 		set_cipher_config0(&desc[*seq_size], direction);
753 		if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) {
754 			set_hw_crypto_key(&desc[*seq_size],
755 					  ctx_p->hw.key1_slot);
756 		} else {
757 			set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr,
758 				     (key_len / 2), NS_BIT);
759 		}
760 		set_key_size_aes(&desc[*seq_size], (key_len / 2));
761 		set_flow_mode(&desc[*seq_size], flow_mode);
762 		set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0);
763 		(*seq_size)++;
764 		break;
765 	default:
766 		dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
767 	}
768 }
769 
cc_setup_mlli_desc(struct crypto_tfm * tfm,struct cipher_req_ctx * req_ctx,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes,void * areq,struct cc_hw_desc desc[],unsigned int * seq_size)770 static void cc_setup_mlli_desc(struct crypto_tfm *tfm,
771 			       struct cipher_req_ctx *req_ctx,
772 			       struct scatterlist *dst, struct scatterlist *src,
773 			       unsigned int nbytes, void *areq,
774 			       struct cc_hw_desc desc[], unsigned int *seq_size)
775 {
776 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
777 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
778 
779 	if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
780 		/* bypass */
781 		dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n",
782 			&req_ctx->mlli_params.mlli_dma_addr,
783 			req_ctx->mlli_params.mlli_len,
784 			ctx_p->drvdata->mlli_sram_addr);
785 		hw_desc_init(&desc[*seq_size]);
786 		set_din_type(&desc[*seq_size], DMA_DLLI,
787 			     req_ctx->mlli_params.mlli_dma_addr,
788 			     req_ctx->mlli_params.mlli_len, NS_BIT);
789 		set_dout_sram(&desc[*seq_size],
790 			      ctx_p->drvdata->mlli_sram_addr,
791 			      req_ctx->mlli_params.mlli_len);
792 		set_flow_mode(&desc[*seq_size], BYPASS);
793 		(*seq_size)++;
794 	}
795 }
796 
cc_setup_flow_desc(struct crypto_tfm * tfm,struct cipher_req_ctx * req_ctx,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes,struct cc_hw_desc desc[],unsigned int * seq_size)797 static void cc_setup_flow_desc(struct crypto_tfm *tfm,
798 			       struct cipher_req_ctx *req_ctx,
799 			       struct scatterlist *dst, struct scatterlist *src,
800 			       unsigned int nbytes, struct cc_hw_desc desc[],
801 			       unsigned int *seq_size)
802 {
803 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
804 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
805 	unsigned int flow_mode = cc_out_flow_mode(ctx_p);
806 	bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY ||
807 			  ctx_p->cipher_mode == DRV_CIPHER_ECB);
808 
809 	/* Process */
810 	if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) {
811 		dev_dbg(dev, " data params addr %pad length 0x%X\n",
812 			&sg_dma_address(src), nbytes);
813 		dev_dbg(dev, " data params addr %pad length 0x%X\n",
814 			&sg_dma_address(dst), nbytes);
815 		hw_desc_init(&desc[*seq_size]);
816 		set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src),
817 			     nbytes, NS_BIT);
818 		set_dout_dlli(&desc[*seq_size], sg_dma_address(dst),
819 			      nbytes, NS_BIT, (!last_desc ? 0 : 1));
820 		if (last_desc)
821 			set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
822 
823 		set_flow_mode(&desc[*seq_size], flow_mode);
824 		(*seq_size)++;
825 	} else {
826 		hw_desc_init(&desc[*seq_size]);
827 		set_din_type(&desc[*seq_size], DMA_MLLI,
828 			     ctx_p->drvdata->mlli_sram_addr,
829 			     req_ctx->in_mlli_nents, NS_BIT);
830 		if (req_ctx->out_nents == 0) {
831 			dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n",
832 				ctx_p->drvdata->mlli_sram_addr,
833 				ctx_p->drvdata->mlli_sram_addr);
834 			set_dout_mlli(&desc[*seq_size],
835 				      ctx_p->drvdata->mlli_sram_addr,
836 				      req_ctx->in_mlli_nents, NS_BIT,
837 				      (!last_desc ? 0 : 1));
838 		} else {
839 			dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n",
840 				ctx_p->drvdata->mlli_sram_addr,
841 				ctx_p->drvdata->mlli_sram_addr +
842 				(u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents);
843 			set_dout_mlli(&desc[*seq_size],
844 				      (ctx_p->drvdata->mlli_sram_addr +
845 				       (LLI_ENTRY_BYTE_SIZE *
846 					req_ctx->in_mlli_nents)),
847 				      req_ctx->out_mlli_nents, NS_BIT,
848 				      (!last_desc ? 0 : 1));
849 		}
850 		if (last_desc)
851 			set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
852 
853 		set_flow_mode(&desc[*seq_size], flow_mode);
854 		(*seq_size)++;
855 	}
856 }
857 
cc_cipher_complete(struct device * dev,void * cc_req,int err)858 static void cc_cipher_complete(struct device *dev, void *cc_req, int err)
859 {
860 	struct skcipher_request *req = (struct skcipher_request *)cc_req;
861 	struct scatterlist *dst = req->dst;
862 	struct scatterlist *src = req->src;
863 	struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
864 	struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req);
865 	unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm);
866 
867 	if (err != -EINPROGRESS) {
868 		/* Not a BACKLOG notification */
869 		cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
870 		memcpy(req->iv, req_ctx->iv, ivsize);
871 		kfree_sensitive(req_ctx->iv);
872 	}
873 
874 	skcipher_request_complete(req, err);
875 }
876 
cc_cipher_process(struct skcipher_request * req,enum drv_crypto_direction direction)877 static int cc_cipher_process(struct skcipher_request *req,
878 			     enum drv_crypto_direction direction)
879 {
880 	struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req);
881 	struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm);
882 	struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
883 	unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm);
884 	struct scatterlist *dst = req->dst;
885 	struct scatterlist *src = req->src;
886 	unsigned int nbytes = req->cryptlen;
887 	void *iv = req->iv;
888 	struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
889 	struct device *dev = drvdata_to_dev(ctx_p->drvdata);
890 	struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN];
891 	struct cc_crypto_req cc_req = {};
892 	int rc;
893 	unsigned int seq_len = 0;
894 	gfp_t flags = cc_gfp_flags(&req->base);
895 
896 	dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n",
897 		((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
898 		"Encrypt" : "Decrypt"), req, iv, nbytes);
899 
900 	/* STAT_PHASE_0: Init and sanity checks */
901 
902 	if (validate_data_size(ctx_p, nbytes)) {
903 		dev_dbg(dev, "Unsupported data size %d.\n", nbytes);
904 		rc = -EINVAL;
905 		goto exit_process;
906 	}
907 	if (nbytes == 0) {
908 		/* No data to process is valid */
909 		rc = 0;
910 		goto exit_process;
911 	}
912 
913 	if (ctx_p->fallback_on) {
914 		struct skcipher_request *subreq = skcipher_request_ctx(req);
915 
916 		*subreq = *req;
917 		skcipher_request_set_tfm(subreq, ctx_p->fallback_tfm);
918 		if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT)
919 			return crypto_skcipher_encrypt(subreq);
920 		else
921 			return crypto_skcipher_decrypt(subreq);
922 	}
923 
924 	/* The IV we are handed may be allocated from the stack so
925 	 * we must copy it to a DMAable buffer before use.
926 	 */
927 	req_ctx->iv = kmemdup(iv, ivsize, flags);
928 	if (!req_ctx->iv) {
929 		rc = -ENOMEM;
930 		goto exit_process;
931 	}
932 
933 	/* Setup request structure */
934 	cc_req.user_cb = cc_cipher_complete;
935 	cc_req.user_arg = req;
936 
937 	/* Setup CPP operation details */
938 	if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) {
939 		cc_req.cpp.is_cpp = true;
940 		cc_req.cpp.alg = ctx_p->cpp.alg;
941 		cc_req.cpp.slot = ctx_p->cpp.slot;
942 	}
943 
944 	/* Setup request context */
945 	req_ctx->gen_ctx.op_type = direction;
946 
947 	/* STAT_PHASE_1: Map buffers */
948 
949 	rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes,
950 				      req_ctx->iv, src, dst, flags);
951 	if (rc) {
952 		dev_err(dev, "map_request() failed\n");
953 		goto exit_process;
954 	}
955 
956 	/* STAT_PHASE_2: Create sequence */
957 
958 	/* Setup state (IV)  */
959 	cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len);
960 	/* Setup MLLI line, if needed */
961 	cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len);
962 	/* Setup key */
963 	cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len);
964 	/* Setup state (IV and XEX key)  */
965 	cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len);
966 	/* Data processing */
967 	cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len);
968 	/* Read next IV */
969 	cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len);
970 
971 	/* STAT_PHASE_3: Lock HW and push sequence */
972 
973 	rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len,
974 			     &req->base);
975 	if (rc != -EINPROGRESS && rc != -EBUSY) {
976 		/* Failed to send the request or request completed
977 		 * synchronously
978 		 */
979 		cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
980 	}
981 
982 exit_process:
983 	if (rc != -EINPROGRESS && rc != -EBUSY) {
984 		kfree_sensitive(req_ctx->iv);
985 	}
986 
987 	return rc;
988 }
989 
cc_cipher_encrypt(struct skcipher_request * req)990 static int cc_cipher_encrypt(struct skcipher_request *req)
991 {
992 	struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
993 
994 	memset(req_ctx, 0, sizeof(*req_ctx));
995 
996 	return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
997 }
998 
cc_cipher_decrypt(struct skcipher_request * req)999 static int cc_cipher_decrypt(struct skcipher_request *req)
1000 {
1001 	struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
1002 
1003 	memset(req_ctx, 0, sizeof(*req_ctx));
1004 
1005 	return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT);
1006 }
1007 
1008 /* Block cipher alg */
1009 static const struct cc_alg_template skcipher_algs[] = {
1010 	{
1011 		.name = "xts(paes)",
1012 		.driver_name = "xts-paes-ccree",
1013 		.blocksize = 1,
1014 		.template_skcipher = {
1015 			.setkey = cc_cipher_sethkey,
1016 			.encrypt = cc_cipher_encrypt,
1017 			.decrypt = cc_cipher_decrypt,
1018 			.min_keysize = CC_HW_KEY_SIZE,
1019 			.max_keysize = CC_HW_KEY_SIZE,
1020 			.ivsize = AES_BLOCK_SIZE,
1021 			},
1022 		.cipher_mode = DRV_CIPHER_XTS,
1023 		.flow_mode = S_DIN_to_AES,
1024 		.min_hw_rev = CC_HW_REV_630,
1025 		.std_body = CC_STD_NIST,
1026 		.sec_func = true,
1027 	},
1028 	{
1029 		.name = "essiv(cbc(paes),sha256)",
1030 		.driver_name = "essiv-paes-ccree",
1031 		.blocksize = AES_BLOCK_SIZE,
1032 		.template_skcipher = {
1033 			.setkey = cc_cipher_sethkey,
1034 			.encrypt = cc_cipher_encrypt,
1035 			.decrypt = cc_cipher_decrypt,
1036 			.min_keysize = CC_HW_KEY_SIZE,
1037 			.max_keysize = CC_HW_KEY_SIZE,
1038 			.ivsize = AES_BLOCK_SIZE,
1039 			},
1040 		.cipher_mode = DRV_CIPHER_ESSIV,
1041 		.flow_mode = S_DIN_to_AES,
1042 		.min_hw_rev = CC_HW_REV_712,
1043 		.std_body = CC_STD_NIST,
1044 		.sec_func = true,
1045 	},
1046 	{
1047 		.name = "ecb(paes)",
1048 		.driver_name = "ecb-paes-ccree",
1049 		.blocksize = AES_BLOCK_SIZE,
1050 		.template_skcipher = {
1051 			.setkey = cc_cipher_sethkey,
1052 			.encrypt = cc_cipher_encrypt,
1053 			.decrypt = cc_cipher_decrypt,
1054 			.min_keysize = CC_HW_KEY_SIZE,
1055 			.max_keysize = CC_HW_KEY_SIZE,
1056 			.ivsize = 0,
1057 			},
1058 		.cipher_mode = DRV_CIPHER_ECB,
1059 		.flow_mode = S_DIN_to_AES,
1060 		.min_hw_rev = CC_HW_REV_712,
1061 		.std_body = CC_STD_NIST,
1062 		.sec_func = true,
1063 	},
1064 	{
1065 		.name = "cbc(paes)",
1066 		.driver_name = "cbc-paes-ccree",
1067 		.blocksize = AES_BLOCK_SIZE,
1068 		.template_skcipher = {
1069 			.setkey = cc_cipher_sethkey,
1070 			.encrypt = cc_cipher_encrypt,
1071 			.decrypt = cc_cipher_decrypt,
1072 			.min_keysize = CC_HW_KEY_SIZE,
1073 			.max_keysize = CC_HW_KEY_SIZE,
1074 			.ivsize = AES_BLOCK_SIZE,
1075 		},
1076 		.cipher_mode = DRV_CIPHER_CBC,
1077 		.flow_mode = S_DIN_to_AES,
1078 		.min_hw_rev = CC_HW_REV_712,
1079 		.std_body = CC_STD_NIST,
1080 		.sec_func = true,
1081 	},
1082 	{
1083 		.name = "ofb(paes)",
1084 		.driver_name = "ofb-paes-ccree",
1085 		.blocksize = AES_BLOCK_SIZE,
1086 		.template_skcipher = {
1087 			.setkey = cc_cipher_sethkey,
1088 			.encrypt = cc_cipher_encrypt,
1089 			.decrypt = cc_cipher_decrypt,
1090 			.min_keysize = CC_HW_KEY_SIZE,
1091 			.max_keysize = CC_HW_KEY_SIZE,
1092 			.ivsize = AES_BLOCK_SIZE,
1093 			},
1094 		.cipher_mode = DRV_CIPHER_OFB,
1095 		.flow_mode = S_DIN_to_AES,
1096 		.min_hw_rev = CC_HW_REV_712,
1097 		.std_body = CC_STD_NIST,
1098 		.sec_func = true,
1099 	},
1100 	{
1101 		.name = "cts(cbc(paes))",
1102 		.driver_name = "cts-cbc-paes-ccree",
1103 		.blocksize = AES_BLOCK_SIZE,
1104 		.template_skcipher = {
1105 			.setkey = cc_cipher_sethkey,
1106 			.encrypt = cc_cipher_encrypt,
1107 			.decrypt = cc_cipher_decrypt,
1108 			.min_keysize = CC_HW_KEY_SIZE,
1109 			.max_keysize = CC_HW_KEY_SIZE,
1110 			.ivsize = AES_BLOCK_SIZE,
1111 			},
1112 		.cipher_mode = DRV_CIPHER_CBC_CTS,
1113 		.flow_mode = S_DIN_to_AES,
1114 		.min_hw_rev = CC_HW_REV_712,
1115 		.std_body = CC_STD_NIST,
1116 		.sec_func = true,
1117 	},
1118 	{
1119 		.name = "ctr(paes)",
1120 		.driver_name = "ctr-paes-ccree",
1121 		.blocksize = 1,
1122 		.template_skcipher = {
1123 			.setkey = cc_cipher_sethkey,
1124 			.encrypt = cc_cipher_encrypt,
1125 			.decrypt = cc_cipher_decrypt,
1126 			.min_keysize = CC_HW_KEY_SIZE,
1127 			.max_keysize = CC_HW_KEY_SIZE,
1128 			.ivsize = AES_BLOCK_SIZE,
1129 			},
1130 		.cipher_mode = DRV_CIPHER_CTR,
1131 		.flow_mode = S_DIN_to_AES,
1132 		.min_hw_rev = CC_HW_REV_712,
1133 		.std_body = CC_STD_NIST,
1134 		.sec_func = true,
1135 	},
1136 	{
1137 		/* See https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg40576.html
1138 		 * for the reason why this differs from the generic
1139 		 * implementation.
1140 		 */
1141 		.name = "xts(aes)",
1142 		.driver_name = "xts-aes-ccree",
1143 		.blocksize = 1,
1144 		.template_skcipher = {
1145 			.setkey = cc_cipher_setkey,
1146 			.encrypt = cc_cipher_encrypt,
1147 			.decrypt = cc_cipher_decrypt,
1148 			.min_keysize = AES_MIN_KEY_SIZE * 2,
1149 			.max_keysize = AES_MAX_KEY_SIZE * 2,
1150 			.ivsize = AES_BLOCK_SIZE,
1151 			},
1152 		.cipher_mode = DRV_CIPHER_XTS,
1153 		.flow_mode = S_DIN_to_AES,
1154 		.min_hw_rev = CC_HW_REV_630,
1155 		.std_body = CC_STD_NIST,
1156 	},
1157 	{
1158 		.name = "essiv(cbc(aes),sha256)",
1159 		.driver_name = "essiv-aes-ccree",
1160 		.blocksize = AES_BLOCK_SIZE,
1161 		.template_skcipher = {
1162 			.setkey = cc_cipher_setkey,
1163 			.encrypt = cc_cipher_encrypt,
1164 			.decrypt = cc_cipher_decrypt,
1165 			.min_keysize = AES_MIN_KEY_SIZE,
1166 			.max_keysize = AES_MAX_KEY_SIZE,
1167 			.ivsize = AES_BLOCK_SIZE,
1168 			},
1169 		.cipher_mode = DRV_CIPHER_ESSIV,
1170 		.flow_mode = S_DIN_to_AES,
1171 		.min_hw_rev = CC_HW_REV_712,
1172 		.std_body = CC_STD_NIST,
1173 	},
1174 	{
1175 		.name = "ecb(aes)",
1176 		.driver_name = "ecb-aes-ccree",
1177 		.blocksize = AES_BLOCK_SIZE,
1178 		.template_skcipher = {
1179 			.setkey = cc_cipher_setkey,
1180 			.encrypt = cc_cipher_encrypt,
1181 			.decrypt = cc_cipher_decrypt,
1182 			.min_keysize = AES_MIN_KEY_SIZE,
1183 			.max_keysize = AES_MAX_KEY_SIZE,
1184 			.ivsize = 0,
1185 			},
1186 		.cipher_mode = DRV_CIPHER_ECB,
1187 		.flow_mode = S_DIN_to_AES,
1188 		.min_hw_rev = CC_HW_REV_630,
1189 		.std_body = CC_STD_NIST,
1190 	},
1191 	{
1192 		.name = "cbc(aes)",
1193 		.driver_name = "cbc-aes-ccree",
1194 		.blocksize = AES_BLOCK_SIZE,
1195 		.template_skcipher = {
1196 			.setkey = cc_cipher_setkey,
1197 			.encrypt = cc_cipher_encrypt,
1198 			.decrypt = cc_cipher_decrypt,
1199 			.min_keysize = AES_MIN_KEY_SIZE,
1200 			.max_keysize = AES_MAX_KEY_SIZE,
1201 			.ivsize = AES_BLOCK_SIZE,
1202 		},
1203 		.cipher_mode = DRV_CIPHER_CBC,
1204 		.flow_mode = S_DIN_to_AES,
1205 		.min_hw_rev = CC_HW_REV_630,
1206 		.std_body = CC_STD_NIST,
1207 	},
1208 	{
1209 		.name = "ofb(aes)",
1210 		.driver_name = "ofb-aes-ccree",
1211 		.blocksize = 1,
1212 		.template_skcipher = {
1213 			.setkey = cc_cipher_setkey,
1214 			.encrypt = cc_cipher_encrypt,
1215 			.decrypt = cc_cipher_decrypt,
1216 			.min_keysize = AES_MIN_KEY_SIZE,
1217 			.max_keysize = AES_MAX_KEY_SIZE,
1218 			.ivsize = AES_BLOCK_SIZE,
1219 			},
1220 		.cipher_mode = DRV_CIPHER_OFB,
1221 		.flow_mode = S_DIN_to_AES,
1222 		.min_hw_rev = CC_HW_REV_630,
1223 		.std_body = CC_STD_NIST,
1224 	},
1225 	{
1226 		.name = "cts(cbc(aes))",
1227 		.driver_name = "cts-cbc-aes-ccree",
1228 		.blocksize = AES_BLOCK_SIZE,
1229 		.template_skcipher = {
1230 			.setkey = cc_cipher_setkey,
1231 			.encrypt = cc_cipher_encrypt,
1232 			.decrypt = cc_cipher_decrypt,
1233 			.min_keysize = AES_MIN_KEY_SIZE,
1234 			.max_keysize = AES_MAX_KEY_SIZE,
1235 			.ivsize = AES_BLOCK_SIZE,
1236 			},
1237 		.cipher_mode = DRV_CIPHER_CBC_CTS,
1238 		.flow_mode = S_DIN_to_AES,
1239 		.min_hw_rev = CC_HW_REV_630,
1240 		.std_body = CC_STD_NIST,
1241 	},
1242 	{
1243 		.name = "ctr(aes)",
1244 		.driver_name = "ctr-aes-ccree",
1245 		.blocksize = 1,
1246 		.template_skcipher = {
1247 			.setkey = cc_cipher_setkey,
1248 			.encrypt = cc_cipher_encrypt,
1249 			.decrypt = cc_cipher_decrypt,
1250 			.min_keysize = AES_MIN_KEY_SIZE,
1251 			.max_keysize = AES_MAX_KEY_SIZE,
1252 			.ivsize = AES_BLOCK_SIZE,
1253 			},
1254 		.cipher_mode = DRV_CIPHER_CTR,
1255 		.flow_mode = S_DIN_to_AES,
1256 		.min_hw_rev = CC_HW_REV_630,
1257 		.std_body = CC_STD_NIST,
1258 	},
1259 	{
1260 		.name = "cbc(des3_ede)",
1261 		.driver_name = "cbc-3des-ccree",
1262 		.blocksize = DES3_EDE_BLOCK_SIZE,
1263 		.template_skcipher = {
1264 			.setkey = cc_cipher_setkey,
1265 			.encrypt = cc_cipher_encrypt,
1266 			.decrypt = cc_cipher_decrypt,
1267 			.min_keysize = DES3_EDE_KEY_SIZE,
1268 			.max_keysize = DES3_EDE_KEY_SIZE,
1269 			.ivsize = DES3_EDE_BLOCK_SIZE,
1270 			},
1271 		.cipher_mode = DRV_CIPHER_CBC,
1272 		.flow_mode = S_DIN_to_DES,
1273 		.min_hw_rev = CC_HW_REV_630,
1274 		.std_body = CC_STD_NIST,
1275 	},
1276 	{
1277 		.name = "ecb(des3_ede)",
1278 		.driver_name = "ecb-3des-ccree",
1279 		.blocksize = DES3_EDE_BLOCK_SIZE,
1280 		.template_skcipher = {
1281 			.setkey = cc_cipher_setkey,
1282 			.encrypt = cc_cipher_encrypt,
1283 			.decrypt = cc_cipher_decrypt,
1284 			.min_keysize = DES3_EDE_KEY_SIZE,
1285 			.max_keysize = DES3_EDE_KEY_SIZE,
1286 			.ivsize = 0,
1287 			},
1288 		.cipher_mode = DRV_CIPHER_ECB,
1289 		.flow_mode = S_DIN_to_DES,
1290 		.min_hw_rev = CC_HW_REV_630,
1291 		.std_body = CC_STD_NIST,
1292 	},
1293 	{
1294 		.name = "cbc(des)",
1295 		.driver_name = "cbc-des-ccree",
1296 		.blocksize = DES_BLOCK_SIZE,
1297 		.template_skcipher = {
1298 			.setkey = cc_cipher_setkey,
1299 			.encrypt = cc_cipher_encrypt,
1300 			.decrypt = cc_cipher_decrypt,
1301 			.min_keysize = DES_KEY_SIZE,
1302 			.max_keysize = DES_KEY_SIZE,
1303 			.ivsize = DES_BLOCK_SIZE,
1304 			},
1305 		.cipher_mode = DRV_CIPHER_CBC,
1306 		.flow_mode = S_DIN_to_DES,
1307 		.min_hw_rev = CC_HW_REV_630,
1308 		.std_body = CC_STD_NIST,
1309 	},
1310 	{
1311 		.name = "ecb(des)",
1312 		.driver_name = "ecb-des-ccree",
1313 		.blocksize = DES_BLOCK_SIZE,
1314 		.template_skcipher = {
1315 			.setkey = cc_cipher_setkey,
1316 			.encrypt = cc_cipher_encrypt,
1317 			.decrypt = cc_cipher_decrypt,
1318 			.min_keysize = DES_KEY_SIZE,
1319 			.max_keysize = DES_KEY_SIZE,
1320 			.ivsize = 0,
1321 			},
1322 		.cipher_mode = DRV_CIPHER_ECB,
1323 		.flow_mode = S_DIN_to_DES,
1324 		.min_hw_rev = CC_HW_REV_630,
1325 		.std_body = CC_STD_NIST,
1326 	},
1327 	{
1328 		.name = "cbc(sm4)",
1329 		.driver_name = "cbc-sm4-ccree",
1330 		.blocksize = SM4_BLOCK_SIZE,
1331 		.template_skcipher = {
1332 			.setkey = cc_cipher_setkey,
1333 			.encrypt = cc_cipher_encrypt,
1334 			.decrypt = cc_cipher_decrypt,
1335 			.min_keysize = SM4_KEY_SIZE,
1336 			.max_keysize = SM4_KEY_SIZE,
1337 			.ivsize = SM4_BLOCK_SIZE,
1338 			},
1339 		.cipher_mode = DRV_CIPHER_CBC,
1340 		.flow_mode = S_DIN_to_SM4,
1341 		.min_hw_rev = CC_HW_REV_713,
1342 		.std_body = CC_STD_OSCCA,
1343 	},
1344 	{
1345 		.name = "ecb(sm4)",
1346 		.driver_name = "ecb-sm4-ccree",
1347 		.blocksize = SM4_BLOCK_SIZE,
1348 		.template_skcipher = {
1349 			.setkey = cc_cipher_setkey,
1350 			.encrypt = cc_cipher_encrypt,
1351 			.decrypt = cc_cipher_decrypt,
1352 			.min_keysize = SM4_KEY_SIZE,
1353 			.max_keysize = SM4_KEY_SIZE,
1354 			.ivsize = 0,
1355 			},
1356 		.cipher_mode = DRV_CIPHER_ECB,
1357 		.flow_mode = S_DIN_to_SM4,
1358 		.min_hw_rev = CC_HW_REV_713,
1359 		.std_body = CC_STD_OSCCA,
1360 	},
1361 	{
1362 		.name = "ctr(sm4)",
1363 		.driver_name = "ctr-sm4-ccree",
1364 		.blocksize = 1,
1365 		.template_skcipher = {
1366 			.setkey = cc_cipher_setkey,
1367 			.encrypt = cc_cipher_encrypt,
1368 			.decrypt = cc_cipher_decrypt,
1369 			.min_keysize = SM4_KEY_SIZE,
1370 			.max_keysize = SM4_KEY_SIZE,
1371 			.ivsize = SM4_BLOCK_SIZE,
1372 			},
1373 		.cipher_mode = DRV_CIPHER_CTR,
1374 		.flow_mode = S_DIN_to_SM4,
1375 		.min_hw_rev = CC_HW_REV_713,
1376 		.std_body = CC_STD_OSCCA,
1377 	},
1378 	{
1379 		.name = "cbc(psm4)",
1380 		.driver_name = "cbc-psm4-ccree",
1381 		.blocksize = SM4_BLOCK_SIZE,
1382 		.template_skcipher = {
1383 			.setkey = cc_cipher_sethkey,
1384 			.encrypt = cc_cipher_encrypt,
1385 			.decrypt = cc_cipher_decrypt,
1386 			.min_keysize = CC_HW_KEY_SIZE,
1387 			.max_keysize = CC_HW_KEY_SIZE,
1388 			.ivsize = SM4_BLOCK_SIZE,
1389 			},
1390 		.cipher_mode = DRV_CIPHER_CBC,
1391 		.flow_mode = S_DIN_to_SM4,
1392 		.min_hw_rev = CC_HW_REV_713,
1393 		.std_body = CC_STD_OSCCA,
1394 		.sec_func = true,
1395 	},
1396 	{
1397 		.name = "ctr(psm4)",
1398 		.driver_name = "ctr-psm4-ccree",
1399 		.blocksize = SM4_BLOCK_SIZE,
1400 		.template_skcipher = {
1401 			.setkey = cc_cipher_sethkey,
1402 			.encrypt = cc_cipher_encrypt,
1403 			.decrypt = cc_cipher_decrypt,
1404 			.min_keysize = CC_HW_KEY_SIZE,
1405 			.max_keysize = CC_HW_KEY_SIZE,
1406 			.ivsize = SM4_BLOCK_SIZE,
1407 			},
1408 		.cipher_mode = DRV_CIPHER_CTR,
1409 		.flow_mode = S_DIN_to_SM4,
1410 		.min_hw_rev = CC_HW_REV_713,
1411 		.std_body = CC_STD_OSCCA,
1412 		.sec_func = true,
1413 	},
1414 };
1415 
cc_create_alg(const struct cc_alg_template * tmpl,struct device * dev)1416 static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl,
1417 					   struct device *dev)
1418 {
1419 	struct cc_crypto_alg *t_alg;
1420 	struct skcipher_alg *alg;
1421 
1422 	t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL);
1423 	if (!t_alg)
1424 		return ERR_PTR(-ENOMEM);
1425 
1426 	alg = &t_alg->skcipher_alg;
1427 
1428 	memcpy(alg, &tmpl->template_skcipher, sizeof(*alg));
1429 
1430 	snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1431 	snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1432 		 tmpl->driver_name);
1433 	alg->base.cra_module = THIS_MODULE;
1434 	alg->base.cra_priority = CC_CRA_PRIO;
1435 	alg->base.cra_blocksize = tmpl->blocksize;
1436 	alg->base.cra_alignmask = 0;
1437 	alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx);
1438 
1439 	alg->base.cra_init = cc_cipher_init;
1440 	alg->base.cra_exit = cc_cipher_exit;
1441 	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
1442 
1443 	t_alg->cipher_mode = tmpl->cipher_mode;
1444 	t_alg->flow_mode = tmpl->flow_mode;
1445 
1446 	return t_alg;
1447 }
1448 
cc_cipher_free(struct cc_drvdata * drvdata)1449 int cc_cipher_free(struct cc_drvdata *drvdata)
1450 {
1451 	struct cc_crypto_alg *t_alg, *n;
1452 
1453 	/* Remove registered algs */
1454 	list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) {
1455 		crypto_unregister_skcipher(&t_alg->skcipher_alg);
1456 		list_del(&t_alg->entry);
1457 	}
1458 	return 0;
1459 }
1460 
cc_cipher_alloc(struct cc_drvdata * drvdata)1461 int cc_cipher_alloc(struct cc_drvdata *drvdata)
1462 {
1463 	struct cc_crypto_alg *t_alg;
1464 	struct device *dev = drvdata_to_dev(drvdata);
1465 	int rc = -ENOMEM;
1466 	int alg;
1467 
1468 	INIT_LIST_HEAD(&drvdata->alg_list);
1469 
1470 	/* Linux crypto */
1471 	dev_dbg(dev, "Number of algorithms = %zu\n",
1472 		ARRAY_SIZE(skcipher_algs));
1473 	for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) {
1474 		if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) ||
1475 		    !(drvdata->std_bodies & skcipher_algs[alg].std_body) ||
1476 		    (drvdata->sec_disabled && skcipher_algs[alg].sec_func))
1477 			continue;
1478 
1479 		dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name);
1480 		t_alg = cc_create_alg(&skcipher_algs[alg], dev);
1481 		if (IS_ERR(t_alg)) {
1482 			rc = PTR_ERR(t_alg);
1483 			dev_err(dev, "%s alg allocation failed\n",
1484 				skcipher_algs[alg].driver_name);
1485 			goto fail0;
1486 		}
1487 		t_alg->drvdata = drvdata;
1488 
1489 		dev_dbg(dev, "registering %s\n",
1490 			skcipher_algs[alg].driver_name);
1491 		rc = crypto_register_skcipher(&t_alg->skcipher_alg);
1492 		dev_dbg(dev, "%s alg registration rc = %x\n",
1493 			t_alg->skcipher_alg.base.cra_driver_name, rc);
1494 		if (rc) {
1495 			dev_err(dev, "%s alg registration failed\n",
1496 				t_alg->skcipher_alg.base.cra_driver_name);
1497 			goto fail0;
1498 		}
1499 
1500 		list_add_tail(&t_alg->entry, &drvdata->alg_list);
1501 		dev_dbg(dev, "Registered %s\n",
1502 			t_alg->skcipher_alg.base.cra_driver_name);
1503 	}
1504 	return 0;
1505 
1506 fail0:
1507 	cc_cipher_free(drvdata);
1508 	return rc;
1509 }
1510