1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Marvell. */
3
4 #include <crypto/aes.h>
5 #include <crypto/authenc.h>
6 #include <crypto/cryptd.h>
7 #include <crypto/des.h>
8 #include <crypto/internal/aead.h>
9 #include <crypto/sha1.h>
10 #include <crypto/sha2.h>
11 #include <crypto/xts.h>
12 #include <crypto/gcm.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/sort.h>
16 #include <linux/module.h>
17 #include "otx2_cptvf.h"
18 #include "otx2_cptvf_algs.h"
19 #include "otx2_cpt_reqmgr.h"
20
21 /* Size of salt in AES GCM mode */
22 #define AES_GCM_SALT_SIZE 4
23 /* Size of IV in AES GCM mode */
24 #define AES_GCM_IV_SIZE 8
25 /* Size of ICV (Integrity Check Value) in AES GCM mode */
26 #define AES_GCM_ICV_SIZE 16
27 /* Offset of IV in AES GCM mode */
28 #define AES_GCM_IV_OFFSET 8
29 #define CONTROL_WORD_LEN 8
30 #define KEY2_OFFSET 48
31 #define DMA_MODE_FLAG(dma_mode) \
32 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
33
34 /* Truncated SHA digest size */
35 #define SHA1_TRUNC_DIGEST_SIZE 12
36 #define SHA256_TRUNC_DIGEST_SIZE 16
37 #define SHA384_TRUNC_DIGEST_SIZE 24
38 #define SHA512_TRUNC_DIGEST_SIZE 32
39
40 static DEFINE_MUTEX(mutex);
41 static int is_crypto_registered;
42
43 struct cpt_device_desc {
44 struct pci_dev *dev;
45 int num_queues;
46 };
47
48 struct cpt_device_table {
49 atomic_t count;
50 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
51 };
52
53 static struct cpt_device_table se_devices = {
54 .count = ATOMIC_INIT(0)
55 };
56
get_se_device(struct pci_dev ** pdev,int * cpu_num)57 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
58 {
59 int count;
60
61 count = atomic_read(&se_devices.count);
62 if (count < 1)
63 return -ENODEV;
64
65 *cpu_num = get_cpu();
66 /*
67 * On OcteonTX2 platform CPT instruction queue is bound to each
68 * local function LF, in turn LFs can be attached to PF
69 * or VF therefore we always use first device. We get maximum
70 * performance if one CPT queue is available for each cpu
71 * otherwise CPT queues need to be shared between cpus.
72 */
73 if (*cpu_num >= se_devices.desc[0].num_queues)
74 *cpu_num %= se_devices.desc[0].num_queues;
75 *pdev = se_devices.desc[0].dev;
76
77 put_cpu();
78
79 return 0;
80 }
81
validate_hmac_cipher_null(struct otx2_cpt_req_info * cpt_req)82 static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
83 {
84 struct otx2_cpt_req_ctx *rctx;
85 struct aead_request *req;
86 struct crypto_aead *tfm;
87
88 req = container_of(cpt_req->areq, struct aead_request, base);
89 tfm = crypto_aead_reqtfm(req);
90 rctx = aead_request_ctx_dma(req);
91 if (memcmp(rctx->fctx.hmac.s.hmac_calc,
92 rctx->fctx.hmac.s.hmac_recv,
93 crypto_aead_authsize(tfm)) != 0)
94 return -EBADMSG;
95
96 return 0;
97 }
98
otx2_cpt_aead_callback(int status,void * arg1,void * arg2)99 static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
100 {
101 struct otx2_cpt_inst_info *inst_info = arg2;
102 struct crypto_async_request *areq = arg1;
103 struct otx2_cpt_req_info *cpt_req;
104 struct pci_dev *pdev;
105
106 if (inst_info) {
107 cpt_req = inst_info->req;
108 if (!status) {
109 /*
110 * When selected cipher is NULL we need to manually
111 * verify whether calculated hmac value matches
112 * received hmac value
113 */
114 if (cpt_req->req_type ==
115 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
116 !cpt_req->is_enc)
117 status = validate_hmac_cipher_null(cpt_req);
118 }
119 pdev = inst_info->pdev;
120 otx2_cpt_info_destroy(pdev, inst_info);
121 }
122 if (areq)
123 crypto_request_complete(areq, status);
124 }
125
output_iv_copyback(struct crypto_async_request * areq)126 static void output_iv_copyback(struct crypto_async_request *areq)
127 {
128 struct otx2_cpt_req_info *req_info;
129 struct otx2_cpt_req_ctx *rctx;
130 struct skcipher_request *sreq;
131 struct crypto_skcipher *stfm;
132 struct otx2_cpt_enc_ctx *ctx;
133 u32 start, ivsize;
134
135 sreq = container_of(areq, struct skcipher_request, base);
136 stfm = crypto_skcipher_reqtfm(sreq);
137 ctx = crypto_skcipher_ctx(stfm);
138 if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
139 ctx->cipher_type == OTX2_CPT_DES3_CBC) {
140 rctx = skcipher_request_ctx_dma(sreq);
141 req_info = &rctx->cpt_req;
142 ivsize = crypto_skcipher_ivsize(stfm);
143 start = sreq->cryptlen - ivsize;
144
145 if (req_info->is_enc) {
146 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
147 ivsize, 0);
148 } else {
149 if (sreq->src != sreq->dst) {
150 scatterwalk_map_and_copy(sreq->iv, sreq->src,
151 start, ivsize, 0);
152 } else {
153 memcpy(sreq->iv, req_info->iv_out, ivsize);
154 kfree(req_info->iv_out);
155 }
156 }
157 }
158 }
159
otx2_cpt_skcipher_callback(int status,void * arg1,void * arg2)160 static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
161 {
162 struct otx2_cpt_inst_info *inst_info = arg2;
163 struct crypto_async_request *areq = arg1;
164 struct pci_dev *pdev;
165
166 if (areq) {
167 if (!status)
168 output_iv_copyback(areq);
169 if (inst_info) {
170 pdev = inst_info->pdev;
171 otx2_cpt_info_destroy(pdev, inst_info);
172 }
173 crypto_request_complete(areq, status);
174 }
175 }
176
update_input_data(struct otx2_cpt_req_info * req_info,struct scatterlist * inp_sg,u32 nbytes,u32 * argcnt)177 static inline void update_input_data(struct otx2_cpt_req_info *req_info,
178 struct scatterlist *inp_sg,
179 u32 nbytes, u32 *argcnt)
180 {
181 req_info->req.dlen += nbytes;
182
183 while (nbytes) {
184 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
185 u8 *ptr = sg_virt(inp_sg);
186
187 req_info->in[*argcnt].vptr = (void *)ptr;
188 req_info->in[*argcnt].size = len;
189 nbytes -= len;
190 ++(*argcnt);
191 inp_sg = sg_next(inp_sg);
192 }
193 }
194
update_output_data(struct otx2_cpt_req_info * req_info,struct scatterlist * outp_sg,u32 offset,u32 nbytes,u32 * argcnt)195 static inline void update_output_data(struct otx2_cpt_req_info *req_info,
196 struct scatterlist *outp_sg,
197 u32 offset, u32 nbytes, u32 *argcnt)
198 {
199 u32 len, sg_len;
200 u8 *ptr;
201
202 req_info->rlen += nbytes;
203
204 while (nbytes) {
205 sg_len = outp_sg->length - offset;
206 len = (nbytes < sg_len) ? nbytes : sg_len;
207 ptr = sg_virt(outp_sg);
208
209 req_info->out[*argcnt].vptr = (void *) (ptr + offset);
210 req_info->out[*argcnt].size = len;
211 nbytes -= len;
212 ++(*argcnt);
213 offset = 0;
214 outp_sg = sg_next(outp_sg);
215 }
216 }
217
create_ctx_hdr(struct skcipher_request * req,u32 enc,u32 * argcnt)218 static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
219 u32 *argcnt)
220 {
221 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
222 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
223 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
224 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
225 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
226 int ivsize = crypto_skcipher_ivsize(stfm);
227 u32 start = req->cryptlen - ivsize;
228 gfp_t flags;
229
230 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
231 GFP_KERNEL : GFP_ATOMIC;
232 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
233 req_info->ctrl.s.se_req = 1;
234
235 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
236 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
237 if (enc) {
238 req_info->req.opcode.s.minor = 2;
239 } else {
240 req_info->req.opcode.s.minor = 3;
241 if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
242 ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
243 req->src == req->dst) {
244 req_info->iv_out = kmalloc(ivsize, flags);
245 if (!req_info->iv_out)
246 return -ENOMEM;
247
248 scatterwalk_map_and_copy(req_info->iv_out, req->src,
249 start, ivsize, 0);
250 }
251 }
252 /* Encryption data length */
253 req_info->req.param1 = req->cryptlen;
254 /* Authentication data length */
255 req_info->req.param2 = 0;
256
257 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
258 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
259 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
260
261 if (ctx->cipher_type == OTX2_CPT_AES_XTS)
262 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
263 else
264 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
265
266 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
267
268 cpu_to_be64s(&fctx->enc.enc_ctrl.u);
269
270 /*
271 * Storing Packet Data Information in offset
272 * Control Word First 8 bytes
273 */
274 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
275 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
276 req_info->req.dlen += CONTROL_WORD_LEN;
277 ++(*argcnt);
278
279 req_info->in[*argcnt].vptr = (u8 *)fctx;
280 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
281 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
282
283 ++(*argcnt);
284
285 return 0;
286 }
287
create_input_list(struct skcipher_request * req,u32 enc,u32 enc_iv_len)288 static inline int create_input_list(struct skcipher_request *req, u32 enc,
289 u32 enc_iv_len)
290 {
291 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
292 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
293 u32 argcnt = 0;
294 int ret;
295
296 ret = create_ctx_hdr(req, enc, &argcnt);
297 if (ret)
298 return ret;
299
300 update_input_data(req_info, req->src, req->cryptlen, &argcnt);
301 req_info->in_cnt = argcnt;
302
303 return 0;
304 }
305
create_output_list(struct skcipher_request * req,u32 enc_iv_len)306 static inline void create_output_list(struct skcipher_request *req,
307 u32 enc_iv_len)
308 {
309 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
310 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
311 u32 argcnt = 0;
312
313 /*
314 * OUTPUT Buffer Processing
315 * AES encryption/decryption output would be
316 * received in the following format
317 *
318 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
319 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
320 */
321 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
322 req_info->out_cnt = argcnt;
323 }
324
skcipher_do_fallback(struct skcipher_request * req,bool is_enc)325 static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
326 {
327 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
328 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
329 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
330 int ret;
331
332 if (ctx->fbk_cipher) {
333 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);
334 skcipher_request_set_callback(&rctx->sk_fbk_req,
335 req->base.flags,
336 req->base.complete,
337 req->base.data);
338 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,
339 req->dst, req->cryptlen, req->iv);
340 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :
341 crypto_skcipher_decrypt(&rctx->sk_fbk_req);
342 } else {
343 ret = -EINVAL;
344 }
345 return ret;
346 }
347
cpt_enc_dec(struct skcipher_request * req,u32 enc)348 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
349 {
350 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
351 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
352 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
353 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
354 u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
355 struct pci_dev *pdev;
356 int status, cpu_num;
357
358 if (req->cryptlen == 0)
359 return 0;
360
361 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
362 return -EINVAL;
363
364 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
365 return skcipher_do_fallback(req, enc);
366
367 /* Clear control words */
368 rctx->ctrl_word.flags = 0;
369 rctx->fctx.enc.enc_ctrl.u = 0;
370
371 status = create_input_list(req, enc, enc_iv_len);
372 if (status)
373 return status;
374 create_output_list(req, enc_iv_len);
375
376 status = get_se_device(&pdev, &cpu_num);
377 if (status)
378 return status;
379
380 req_info->callback = otx2_cpt_skcipher_callback;
381 req_info->areq = &req->base;
382 req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
383 req_info->is_enc = enc;
384 req_info->is_trunc_hmac = false;
385 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
386
387 /*
388 * We perform an asynchronous send and once
389 * the request is completed the driver would
390 * intimate through registered call back functions
391 */
392 status = otx2_cpt_do_request(pdev, req_info, cpu_num);
393
394 return status;
395 }
396
otx2_cpt_skcipher_encrypt(struct skcipher_request * req)397 static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
398 {
399 return cpt_enc_dec(req, true);
400 }
401
otx2_cpt_skcipher_decrypt(struct skcipher_request * req)402 static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
403 {
404 return cpt_enc_dec(req, false);
405 }
406
otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)407 static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
408 const u8 *key, u32 keylen)
409 {
410 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
411 const u8 *key2 = key + (keylen / 2);
412 const u8 *key1 = key;
413 int ret;
414
415 ret = xts_verify_key(tfm, key, keylen);
416 if (ret)
417 return ret;
418 ctx->key_len = keylen;
419 ctx->enc_align_len = 1;
420 memcpy(ctx->enc_key, key1, keylen / 2);
421 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
422 ctx->cipher_type = OTX2_CPT_AES_XTS;
423 switch (ctx->key_len) {
424 case 2 * AES_KEYSIZE_128:
425 ctx->key_type = OTX2_CPT_AES_128_BIT;
426 break;
427 case 2 * AES_KEYSIZE_192:
428 ctx->key_type = OTX2_CPT_AES_192_BIT;
429 break;
430 case 2 * AES_KEYSIZE_256:
431 ctx->key_type = OTX2_CPT_AES_256_BIT;
432 break;
433 default:
434 return -EINVAL;
435 }
436 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
437 }
438
cpt_des_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen,u8 cipher_type)439 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
440 u32 keylen, u8 cipher_type)
441 {
442 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
443
444 if (keylen != DES3_EDE_KEY_SIZE)
445 return -EINVAL;
446
447 ctx->key_len = keylen;
448 ctx->cipher_type = cipher_type;
449 ctx->enc_align_len = 8;
450
451 memcpy(ctx->enc_key, key, keylen);
452
453 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
454 }
455
cpt_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen,u8 cipher_type)456 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
457 u32 keylen, u8 cipher_type)
458 {
459 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
460
461 switch (keylen) {
462 case AES_KEYSIZE_128:
463 ctx->key_type = OTX2_CPT_AES_128_BIT;
464 break;
465 case AES_KEYSIZE_192:
466 ctx->key_type = OTX2_CPT_AES_192_BIT;
467 break;
468 case AES_KEYSIZE_256:
469 ctx->key_type = OTX2_CPT_AES_256_BIT;
470 break;
471 default:
472 return -EINVAL;
473 }
474 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
475 ctx->enc_align_len = 16;
476 else
477 ctx->enc_align_len = 1;
478
479 ctx->key_len = keylen;
480 ctx->cipher_type = cipher_type;
481
482 memcpy(ctx->enc_key, key, keylen);
483
484 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
485 }
486
otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)487 static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
488 const u8 *key, u32 keylen)
489 {
490 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);
491 }
492
otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)493 static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
494 const u8 *key, u32 keylen)
495 {
496 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);
497 }
498
otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)499 static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
500 const u8 *key, u32 keylen)
501 {
502 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);
503 }
504
otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher * tfm,const u8 * key,u32 keylen)505 static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
506 const u8 *key, u32 keylen)
507 {
508 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);
509 }
510
cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx * ctx,struct crypto_alg * alg)511 static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
512 struct crypto_alg *alg)
513 {
514 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
515 ctx->fbk_cipher =
516 crypto_alloc_skcipher(alg->cra_name, 0,
517 CRYPTO_ALG_ASYNC |
518 CRYPTO_ALG_NEED_FALLBACK);
519 if (IS_ERR(ctx->fbk_cipher)) {
520 pr_err("%s() failed to allocate fallback for %s\n",
521 __func__, alg->cra_name);
522 return PTR_ERR(ctx->fbk_cipher);
523 }
524 }
525 return 0;
526 }
527
otx2_cpt_enc_dec_init(struct crypto_skcipher * stfm)528 static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
529 {
530 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
531 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
532 struct crypto_alg *alg = tfm->__crt_alg;
533
534 memset(ctx, 0, sizeof(*ctx));
535 /*
536 * Additional memory for skcipher_request is
537 * allocated since the cryptd daemon uses
538 * this memory for request_ctx information
539 */
540 crypto_skcipher_set_reqsize_dma(
541 stfm, sizeof(struct otx2_cpt_req_ctx) +
542 sizeof(struct skcipher_request));
543
544 return cpt_skcipher_fallback_init(ctx, alg);
545 }
546
otx2_cpt_skcipher_exit(struct crypto_skcipher * tfm)547 static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
548 {
549 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
550
551 if (ctx->fbk_cipher) {
552 crypto_free_skcipher(ctx->fbk_cipher);
553 ctx->fbk_cipher = NULL;
554 }
555 }
556
cpt_aead_fallback_init(struct otx2_cpt_aead_ctx * ctx,struct crypto_alg * alg)557 static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
558 struct crypto_alg *alg)
559 {
560 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
561 ctx->fbk_cipher =
562 crypto_alloc_aead(alg->cra_name, 0,
563 CRYPTO_ALG_ASYNC |
564 CRYPTO_ALG_NEED_FALLBACK);
565 if (IS_ERR(ctx->fbk_cipher)) {
566 pr_err("%s() failed to allocate fallback for %s\n",
567 __func__, alg->cra_name);
568 return PTR_ERR(ctx->fbk_cipher);
569 }
570 }
571 return 0;
572 }
573
cpt_aead_init(struct crypto_aead * atfm,u8 cipher_type,u8 mac_type)574 static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
575 {
576 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);
577 struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
578 struct crypto_alg *alg = tfm->__crt_alg;
579
580 ctx->cipher_type = cipher_type;
581 ctx->mac_type = mac_type;
582
583 /*
584 * When selected cipher is NULL we use HMAC opcode instead of
585 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
586 * for calculating ipad and opad
587 */
588 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) {
589 switch (ctx->mac_type) {
590 case OTX2_CPT_SHA1:
591 ctx->hashalg = crypto_alloc_shash("sha1", 0,
592 CRYPTO_ALG_ASYNC);
593 if (IS_ERR(ctx->hashalg))
594 return PTR_ERR(ctx->hashalg);
595 break;
596
597 case OTX2_CPT_SHA256:
598 ctx->hashalg = crypto_alloc_shash("sha256", 0,
599 CRYPTO_ALG_ASYNC);
600 if (IS_ERR(ctx->hashalg))
601 return PTR_ERR(ctx->hashalg);
602 break;
603
604 case OTX2_CPT_SHA384:
605 ctx->hashalg = crypto_alloc_shash("sha384", 0,
606 CRYPTO_ALG_ASYNC);
607 if (IS_ERR(ctx->hashalg))
608 return PTR_ERR(ctx->hashalg);
609 break;
610
611 case OTX2_CPT_SHA512:
612 ctx->hashalg = crypto_alloc_shash("sha512", 0,
613 CRYPTO_ALG_ASYNC);
614 if (IS_ERR(ctx->hashalg))
615 return PTR_ERR(ctx->hashalg);
616 break;
617 }
618 }
619 switch (ctx->cipher_type) {
620 case OTX2_CPT_AES_CBC:
621 case OTX2_CPT_AES_ECB:
622 ctx->enc_align_len = 16;
623 break;
624 case OTX2_CPT_DES3_CBC:
625 case OTX2_CPT_DES3_ECB:
626 ctx->enc_align_len = 8;
627 break;
628 case OTX2_CPT_AES_GCM:
629 case OTX2_CPT_CIPHER_NULL:
630 ctx->enc_align_len = 1;
631 break;
632 }
633 crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));
634
635 return cpt_aead_fallback_init(ctx, alg);
636 }
637
otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead * tfm)638 static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
639 {
640 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);
641 }
642
otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead * tfm)643 static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
644 {
645 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);
646 }
647
otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead * tfm)648 static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
649 {
650 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);
651 }
652
otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead * tfm)653 static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
654 {
655 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);
656 }
657
otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead * tfm)658 static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
659 {
660 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);
661 }
662
otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead * tfm)663 static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
664 {
665 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);
666 }
667
otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead * tfm)668 static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
669 {
670 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);
671 }
672
otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead * tfm)673 static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
674 {
675 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);
676 }
677
otx2_cpt_aead_gcm_aes_init(struct crypto_aead * tfm)678 static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
679 {
680 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);
681 }
682
otx2_cpt_aead_exit(struct crypto_aead * tfm)683 static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
684 {
685 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
686
687 kfree(ctx->ipad);
688 kfree(ctx->opad);
689 if (ctx->hashalg)
690 crypto_free_shash(ctx->hashalg);
691 kfree(ctx->sdesc);
692
693 if (ctx->fbk_cipher) {
694 crypto_free_aead(ctx->fbk_cipher);
695 ctx->fbk_cipher = NULL;
696 }
697 }
698
otx2_cpt_aead_gcm_set_authsize(struct crypto_aead * tfm,unsigned int authsize)699 static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
700 unsigned int authsize)
701 {
702 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
703
704 if (crypto_rfc4106_check_authsize(authsize))
705 return -EINVAL;
706
707 tfm->authsize = authsize;
708 /* Set authsize for fallback case */
709 if (ctx->fbk_cipher)
710 ctx->fbk_cipher->authsize = authsize;
711
712 return 0;
713 }
714
otx2_cpt_aead_set_authsize(struct crypto_aead * tfm,unsigned int authsize)715 static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
716 unsigned int authsize)
717 {
718 tfm->authsize = authsize;
719
720 return 0;
721 }
722
otx2_cpt_aead_null_set_authsize(struct crypto_aead * tfm,unsigned int authsize)723 static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
724 unsigned int authsize)
725 {
726 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
727
728 ctx->is_trunc_hmac = true;
729 tfm->authsize = authsize;
730
731 return 0;
732 }
733
alloc_sdesc(struct crypto_shash * alg)734 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
735 {
736 struct otx2_cpt_sdesc *sdesc;
737 int size;
738
739 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
740 sdesc = kmalloc(size, GFP_KERNEL);
741 if (!sdesc)
742 return NULL;
743
744 sdesc->shash.tfm = alg;
745
746 return sdesc;
747 }
748
swap_data32(void * buf,u32 len)749 static inline void swap_data32(void *buf, u32 len)
750 {
751 cpu_to_be32_array(buf, buf, len / 4);
752 }
753
swap_data64(void * buf,u32 len)754 static inline void swap_data64(void *buf, u32 len)
755 {
756 u64 *src = buf;
757 int i = 0;
758
759 for (i = 0 ; i < len / 8; i++, src++)
760 cpu_to_be64s(src);
761 }
762
copy_pad(u8 mac_type,u8 * out_pad,u8 * in_pad)763 static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
764 {
765 struct sha512_state *sha512;
766 struct sha256_state *sha256;
767 struct sha1_state *sha1;
768
769 switch (mac_type) {
770 case OTX2_CPT_SHA1:
771 sha1 = (struct sha1_state *) in_pad;
772 swap_data32(sha1->state, SHA1_DIGEST_SIZE);
773 memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
774 break;
775
776 case OTX2_CPT_SHA256:
777 sha256 = (struct sha256_state *) in_pad;
778 swap_data32(sha256->state, SHA256_DIGEST_SIZE);
779 memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
780 break;
781
782 case OTX2_CPT_SHA384:
783 case OTX2_CPT_SHA512:
784 sha512 = (struct sha512_state *) in_pad;
785 swap_data64(sha512->state, SHA512_DIGEST_SIZE);
786 memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
787 break;
788
789 default:
790 return -EINVAL;
791 }
792
793 return 0;
794 }
795
aead_hmac_init(struct crypto_aead * cipher)796 static int aead_hmac_init(struct crypto_aead *cipher)
797 {
798 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
799 int state_size = crypto_shash_statesize(ctx->hashalg);
800 int ds = crypto_shash_digestsize(ctx->hashalg);
801 int bs = crypto_shash_blocksize(ctx->hashalg);
802 int authkeylen = ctx->auth_key_len;
803 u8 *ipad = NULL, *opad = NULL;
804 int ret = 0, icount = 0;
805
806 ctx->sdesc = alloc_sdesc(ctx->hashalg);
807 if (!ctx->sdesc)
808 return -ENOMEM;
809
810 ctx->ipad = kzalloc(bs, GFP_KERNEL);
811 if (!ctx->ipad) {
812 ret = -ENOMEM;
813 goto calc_fail;
814 }
815
816 ctx->opad = kzalloc(bs, GFP_KERNEL);
817 if (!ctx->opad) {
818 ret = -ENOMEM;
819 goto calc_fail;
820 }
821
822 ipad = kzalloc(state_size, GFP_KERNEL);
823 if (!ipad) {
824 ret = -ENOMEM;
825 goto calc_fail;
826 }
827
828 opad = kzalloc(state_size, GFP_KERNEL);
829 if (!opad) {
830 ret = -ENOMEM;
831 goto calc_fail;
832 }
833
834 if (authkeylen > bs) {
835 ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key,
836 authkeylen, ipad);
837 if (ret)
838 goto calc_fail;
839
840 authkeylen = ds;
841 } else {
842 memcpy(ipad, ctx->key, authkeylen);
843 }
844
845 memset(ipad + authkeylen, 0, bs - authkeylen);
846 memcpy(opad, ipad, bs);
847
848 for (icount = 0; icount < bs; icount++) {
849 ipad[icount] ^= 0x36;
850 opad[icount] ^= 0x5c;
851 }
852
853 /*
854 * Partial Hash calculated from the software
855 * algorithm is retrieved for IPAD & OPAD
856 */
857
858 /* IPAD Calculation */
859 crypto_shash_init(&ctx->sdesc->shash);
860 crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
861 crypto_shash_export(&ctx->sdesc->shash, ipad);
862 ret = copy_pad(ctx->mac_type, ctx->ipad, ipad);
863 if (ret)
864 goto calc_fail;
865
866 /* OPAD Calculation */
867 crypto_shash_init(&ctx->sdesc->shash);
868 crypto_shash_update(&ctx->sdesc->shash, opad, bs);
869 crypto_shash_export(&ctx->sdesc->shash, opad);
870 ret = copy_pad(ctx->mac_type, ctx->opad, opad);
871 if (ret)
872 goto calc_fail;
873
874 kfree(ipad);
875 kfree(opad);
876
877 return 0;
878
879 calc_fail:
880 kfree(ctx->ipad);
881 ctx->ipad = NULL;
882 kfree(ctx->opad);
883 ctx->opad = NULL;
884 kfree(ipad);
885 kfree(opad);
886 kfree(ctx->sdesc);
887 ctx->sdesc = NULL;
888
889 return ret;
890 }
891
otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead * cipher,const unsigned char * key,unsigned int keylen)892 static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
893 const unsigned char *key,
894 unsigned int keylen)
895 {
896 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
897 struct crypto_authenc_key_param *param;
898 int enckeylen = 0, authkeylen = 0;
899 struct rtattr *rta = (void *)key;
900
901 if (!RTA_OK(rta, keylen))
902 return -EINVAL;
903
904 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
905 return -EINVAL;
906
907 if (RTA_PAYLOAD(rta) < sizeof(*param))
908 return -EINVAL;
909
910 param = RTA_DATA(rta);
911 enckeylen = be32_to_cpu(param->enckeylen);
912 key += RTA_ALIGN(rta->rta_len);
913 keylen -= RTA_ALIGN(rta->rta_len);
914 if (keylen < enckeylen)
915 return -EINVAL;
916
917 if (keylen > OTX2_CPT_MAX_KEY_SIZE)
918 return -EINVAL;
919
920 authkeylen = keylen - enckeylen;
921 memcpy(ctx->key, key, keylen);
922
923 switch (enckeylen) {
924 case AES_KEYSIZE_128:
925 ctx->key_type = OTX2_CPT_AES_128_BIT;
926 break;
927 case AES_KEYSIZE_192:
928 ctx->key_type = OTX2_CPT_AES_192_BIT;
929 break;
930 case AES_KEYSIZE_256:
931 ctx->key_type = OTX2_CPT_AES_256_BIT;
932 break;
933 default:
934 /* Invalid key length */
935 return -EINVAL;
936 }
937
938 ctx->enc_key_len = enckeylen;
939 ctx->auth_key_len = authkeylen;
940
941 return aead_hmac_init(cipher);
942 }
943
otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead * cipher,const unsigned char * key,unsigned int keylen)944 static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
945 const unsigned char *key,
946 unsigned int keylen)
947 {
948 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
949 struct crypto_authenc_key_param *param;
950 struct rtattr *rta = (void *)key;
951 int enckeylen = 0;
952
953 if (!RTA_OK(rta, keylen))
954 return -EINVAL;
955
956 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
957 return -EINVAL;
958
959 if (RTA_PAYLOAD(rta) < sizeof(*param))
960 return -EINVAL;
961
962 param = RTA_DATA(rta);
963 enckeylen = be32_to_cpu(param->enckeylen);
964 key += RTA_ALIGN(rta->rta_len);
965 keylen -= RTA_ALIGN(rta->rta_len);
966 if (enckeylen != 0)
967 return -EINVAL;
968
969 if (keylen > OTX2_CPT_MAX_KEY_SIZE)
970 return -EINVAL;
971
972 memcpy(ctx->key, key, keylen);
973 ctx->enc_key_len = enckeylen;
974 ctx->auth_key_len = keylen;
975
976 return 0;
977 }
978
otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead * cipher,const unsigned char * key,unsigned int keylen)979 static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
980 const unsigned char *key,
981 unsigned int keylen)
982 {
983 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
984
985 /*
986 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
987 * and salt (4 bytes)
988 */
989 switch (keylen) {
990 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
991 ctx->key_type = OTX2_CPT_AES_128_BIT;
992 ctx->enc_key_len = AES_KEYSIZE_128;
993 break;
994 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
995 ctx->key_type = OTX2_CPT_AES_192_BIT;
996 ctx->enc_key_len = AES_KEYSIZE_192;
997 break;
998 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
999 ctx->key_type = OTX2_CPT_AES_256_BIT;
1000 ctx->enc_key_len = AES_KEYSIZE_256;
1001 break;
1002 default:
1003 /* Invalid key and salt length */
1004 return -EINVAL;
1005 }
1006
1007 /* Store encryption key and salt */
1008 memcpy(ctx->key, key, keylen);
1009
1010 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);
1011 }
1012
create_aead_ctx_hdr(struct aead_request * req,u32 enc,u32 * argcnt)1013 static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
1014 u32 *argcnt)
1015 {
1016 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1017 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1018 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1019 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1020 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
1021 int mac_len = crypto_aead_authsize(tfm);
1022 int ds;
1023
1024 rctx->ctrl_word.e.enc_data_offset = req->assoclen;
1025
1026 switch (ctx->cipher_type) {
1027 case OTX2_CPT_AES_CBC:
1028 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
1029 return -EINVAL;
1030
1031 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
1032 /* Copy encryption key to context */
1033 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
1034 ctx->enc_key_len);
1035 /* Copy IV to context */
1036 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
1037
1038 ds = crypto_shash_digestsize(ctx->hashalg);
1039 if (ctx->mac_type == OTX2_CPT_SHA384)
1040 ds = SHA512_DIGEST_SIZE;
1041 if (ctx->ipad)
1042 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
1043 if (ctx->opad)
1044 memcpy(fctx->hmac.e.opad, ctx->opad, ds);
1045 break;
1046
1047 case OTX2_CPT_AES_GCM:
1048 if (crypto_ipsec_check_assoclen(req->assoclen))
1049 return -EINVAL;
1050
1051 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1052 /* Copy encryption key to context */
1053 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1054 /* Copy salt to context */
1055 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1056 AES_GCM_SALT_SIZE);
1057
1058 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1059 break;
1060
1061 default:
1062 /* Unknown cipher type */
1063 return -EINVAL;
1064 }
1065 cpu_to_be64s(&rctx->ctrl_word.flags);
1066
1067 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1068 req_info->ctrl.s.se_req = 1;
1069 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1070 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1071 if (enc) {
1072 req_info->req.opcode.s.minor = 2;
1073 req_info->req.param1 = req->cryptlen;
1074 req_info->req.param2 = req->cryptlen + req->assoclen;
1075 } else {
1076 req_info->req.opcode.s.minor = 3;
1077 req_info->req.param1 = req->cryptlen - mac_len;
1078 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1079 }
1080
1081 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1082 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1083 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1084 fctx->enc.enc_ctrl.e.mac_len = mac_len;
1085 cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1086
1087 /*
1088 * Storing Packet Data Information in offset
1089 * Control Word First 8 bytes
1090 */
1091 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1092 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1093 req_info->req.dlen += CONTROL_WORD_LEN;
1094 ++(*argcnt);
1095
1096 req_info->in[*argcnt].vptr = (u8 *)fctx;
1097 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1098 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1099 ++(*argcnt);
1100
1101 return 0;
1102 }
1103
create_hmac_ctx_hdr(struct aead_request * req,u32 * argcnt,u32 enc)1104 static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1105 u32 enc)
1106 {
1107 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1108 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1109 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1110 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1111
1112 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1113 req_info->ctrl.s.se_req = 1;
1114 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1115 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1116 req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1117
1118 req_info->req.opcode.s.minor = 0;
1119 req_info->req.param1 = ctx->auth_key_len;
1120 req_info->req.param2 = ctx->mac_type << 8;
1121
1122 /* Add authentication key */
1123 req_info->in[*argcnt].vptr = ctx->key;
1124 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1125 req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1126 ++(*argcnt);
1127 }
1128
create_aead_input_list(struct aead_request * req,u32 enc)1129 static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1130 {
1131 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1132 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1133 u32 inputlen = req->cryptlen + req->assoclen;
1134 u32 status, argcnt = 0;
1135
1136 status = create_aead_ctx_hdr(req, enc, &argcnt);
1137 if (status)
1138 return status;
1139 update_input_data(req_info, req->src, inputlen, &argcnt);
1140 req_info->in_cnt = argcnt;
1141
1142 return 0;
1143 }
1144
create_aead_output_list(struct aead_request * req,u32 enc,u32 mac_len)1145 static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1146 u32 mac_len)
1147 {
1148 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1149 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1150 u32 argcnt = 0, outputlen = 0;
1151
1152 if (enc)
1153 outputlen = req->cryptlen + req->assoclen + mac_len;
1154 else
1155 outputlen = req->cryptlen + req->assoclen - mac_len;
1156
1157 update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1158 req_info->out_cnt = argcnt;
1159 }
1160
create_aead_null_input_list(struct aead_request * req,u32 enc,u32 mac_len)1161 static inline void create_aead_null_input_list(struct aead_request *req,
1162 u32 enc, u32 mac_len)
1163 {
1164 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1165 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1166 u32 inputlen, argcnt = 0;
1167
1168 if (enc)
1169 inputlen = req->cryptlen + req->assoclen;
1170 else
1171 inputlen = req->cryptlen + req->assoclen - mac_len;
1172
1173 create_hmac_ctx_hdr(req, &argcnt, enc);
1174 update_input_data(req_info, req->src, inputlen, &argcnt);
1175 req_info->in_cnt = argcnt;
1176 }
1177
create_aead_null_output_list(struct aead_request * req,u32 enc,u32 mac_len)1178 static inline int create_aead_null_output_list(struct aead_request *req,
1179 u32 enc, u32 mac_len)
1180 {
1181 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1182 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1183 struct scatterlist *dst;
1184 u8 *ptr = NULL;
1185 int argcnt = 0, status, offset;
1186 u32 inputlen;
1187
1188 if (enc)
1189 inputlen = req->cryptlen + req->assoclen;
1190 else
1191 inputlen = req->cryptlen + req->assoclen - mac_len;
1192
1193 /*
1194 * If source and destination are different
1195 * then copy payload to destination
1196 */
1197 if (req->src != req->dst) {
1198
1199 ptr = kmalloc(inputlen, (req_info->areq->flags &
1200 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1201 GFP_KERNEL : GFP_ATOMIC);
1202 if (!ptr)
1203 return -ENOMEM;
1204
1205 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1206 inputlen);
1207 if (status != inputlen) {
1208 status = -EINVAL;
1209 goto error_free;
1210 }
1211 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1212 inputlen);
1213 if (status != inputlen) {
1214 status = -EINVAL;
1215 goto error_free;
1216 }
1217 kfree(ptr);
1218 }
1219
1220 if (enc) {
1221 /*
1222 * In an encryption scenario hmac needs
1223 * to be appended after payload
1224 */
1225 dst = req->dst;
1226 offset = inputlen;
1227 while (offset >= dst->length) {
1228 offset -= dst->length;
1229 dst = sg_next(dst);
1230 if (!dst)
1231 return -ENOENT;
1232 }
1233
1234 update_output_data(req_info, dst, offset, mac_len, &argcnt);
1235 } else {
1236 /*
1237 * In a decryption scenario calculated hmac for received
1238 * payload needs to be compare with hmac received
1239 */
1240 status = sg_copy_buffer(req->src, sg_nents(req->src),
1241 rctx->fctx.hmac.s.hmac_recv, mac_len,
1242 inputlen, true);
1243 if (status != mac_len)
1244 return -EINVAL;
1245
1246 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1247 req_info->out[argcnt].size = mac_len;
1248 argcnt++;
1249 }
1250
1251 req_info->out_cnt = argcnt;
1252 return 0;
1253
1254 error_free:
1255 kfree(ptr);
1256 return status;
1257 }
1258
aead_do_fallback(struct aead_request * req,bool is_enc)1259 static int aead_do_fallback(struct aead_request *req, bool is_enc)
1260 {
1261 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1262 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1263 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead);
1264 int ret;
1265
1266 if (ctx->fbk_cipher) {
1267 /* Store the cipher tfm and then use the fallback tfm */
1268 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);
1269 aead_request_set_callback(&rctx->fbk_req, req->base.flags,
1270 req->base.complete, req->base.data);
1271 aead_request_set_crypt(&rctx->fbk_req, req->src,
1272 req->dst, req->cryptlen, req->iv);
1273 aead_request_set_ad(&rctx->fbk_req, req->assoclen);
1274 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
1275 crypto_aead_decrypt(&rctx->fbk_req);
1276 } else {
1277 ret = -EINVAL;
1278 }
1279
1280 return ret;
1281 }
1282
cpt_aead_enc_dec(struct aead_request * req,u8 reg_type,u8 enc)1283 static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1284 {
1285 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1286 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1287 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1288 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1289 struct pci_dev *pdev;
1290 int status, cpu_num;
1291
1292 /* Clear control words */
1293 rctx->ctrl_word.flags = 0;
1294 rctx->fctx.enc.enc_ctrl.u = 0;
1295
1296 req_info->callback = otx2_cpt_aead_callback;
1297 req_info->areq = &req->base;
1298 req_info->req_type = reg_type;
1299 req_info->is_enc = enc;
1300 req_info->is_trunc_hmac = false;
1301
1302 switch (reg_type) {
1303 case OTX2_CPT_AEAD_ENC_DEC_REQ:
1304 status = create_aead_input_list(req, enc);
1305 if (status)
1306 return status;
1307 create_aead_output_list(req, enc, crypto_aead_authsize(tfm));
1308 break;
1309
1310 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1311 create_aead_null_input_list(req, enc,
1312 crypto_aead_authsize(tfm));
1313 status = create_aead_null_output_list(req, enc,
1314 crypto_aead_authsize(tfm));
1315 if (status)
1316 return status;
1317 break;
1318
1319 default:
1320 return -EINVAL;
1321 }
1322 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1323 return -EINVAL;
1324
1325 if (!req_info->req.param2 ||
1326 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1327 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1328 return aead_do_fallback(req, enc);
1329
1330 status = get_se_device(&pdev, &cpu_num);
1331 if (status)
1332 return status;
1333
1334 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1335
1336 /*
1337 * We perform an asynchronous send and once
1338 * the request is completed the driver would
1339 * intimate through registered call back functions
1340 */
1341 return otx2_cpt_do_request(pdev, req_info, cpu_num);
1342 }
1343
otx2_cpt_aead_encrypt(struct aead_request * req)1344 static int otx2_cpt_aead_encrypt(struct aead_request *req)
1345 {
1346 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);
1347 }
1348
otx2_cpt_aead_decrypt(struct aead_request * req)1349 static int otx2_cpt_aead_decrypt(struct aead_request *req)
1350 {
1351 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);
1352 }
1353
otx2_cpt_aead_null_encrypt(struct aead_request * req)1354 static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1355 {
1356 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1357 }
1358
otx2_cpt_aead_null_decrypt(struct aead_request * req)1359 static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1360 {
1361 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1362 }
1363
1364 static struct skcipher_alg otx2_cpt_skciphers[] = { {
1365 .base.cra_name = "xts(aes)",
1366 .base.cra_driver_name = "cpt_xts_aes",
1367 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1368 .base.cra_blocksize = AES_BLOCK_SIZE,
1369 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1370 .base.cra_alignmask = 7,
1371 .base.cra_priority = 4001,
1372 .base.cra_module = THIS_MODULE,
1373
1374 .init = otx2_cpt_enc_dec_init,
1375 .exit = otx2_cpt_skcipher_exit,
1376 .ivsize = AES_BLOCK_SIZE,
1377 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1378 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1379 .setkey = otx2_cpt_skcipher_xts_setkey,
1380 .encrypt = otx2_cpt_skcipher_encrypt,
1381 .decrypt = otx2_cpt_skcipher_decrypt,
1382 }, {
1383 .base.cra_name = "cbc(aes)",
1384 .base.cra_driver_name = "cpt_cbc_aes",
1385 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1386 .base.cra_blocksize = AES_BLOCK_SIZE,
1387 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1388 .base.cra_alignmask = 7,
1389 .base.cra_priority = 4001,
1390 .base.cra_module = THIS_MODULE,
1391
1392 .init = otx2_cpt_enc_dec_init,
1393 .exit = otx2_cpt_skcipher_exit,
1394 .ivsize = AES_BLOCK_SIZE,
1395 .min_keysize = AES_MIN_KEY_SIZE,
1396 .max_keysize = AES_MAX_KEY_SIZE,
1397 .setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1398 .encrypt = otx2_cpt_skcipher_encrypt,
1399 .decrypt = otx2_cpt_skcipher_decrypt,
1400 }, {
1401 .base.cra_name = "ecb(aes)",
1402 .base.cra_driver_name = "cpt_ecb_aes",
1403 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1404 .base.cra_blocksize = AES_BLOCK_SIZE,
1405 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1406 .base.cra_alignmask = 7,
1407 .base.cra_priority = 4001,
1408 .base.cra_module = THIS_MODULE,
1409
1410 .init = otx2_cpt_enc_dec_init,
1411 .exit = otx2_cpt_skcipher_exit,
1412 .ivsize = 0,
1413 .min_keysize = AES_MIN_KEY_SIZE,
1414 .max_keysize = AES_MAX_KEY_SIZE,
1415 .setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1416 .encrypt = otx2_cpt_skcipher_encrypt,
1417 .decrypt = otx2_cpt_skcipher_decrypt,
1418 }, {
1419 .base.cra_name = "cbc(des3_ede)",
1420 .base.cra_driver_name = "cpt_cbc_des3_ede",
1421 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1422 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1423 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1424 .base.cra_alignmask = 7,
1425 .base.cra_priority = 4001,
1426 .base.cra_module = THIS_MODULE,
1427
1428 .init = otx2_cpt_enc_dec_init,
1429 .exit = otx2_cpt_skcipher_exit,
1430 .min_keysize = DES3_EDE_KEY_SIZE,
1431 .max_keysize = DES3_EDE_KEY_SIZE,
1432 .ivsize = DES_BLOCK_SIZE,
1433 .setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1434 .encrypt = otx2_cpt_skcipher_encrypt,
1435 .decrypt = otx2_cpt_skcipher_decrypt,
1436 }, {
1437 .base.cra_name = "ecb(des3_ede)",
1438 .base.cra_driver_name = "cpt_ecb_des3_ede",
1439 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1440 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1441 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1442 .base.cra_alignmask = 7,
1443 .base.cra_priority = 4001,
1444 .base.cra_module = THIS_MODULE,
1445
1446 .init = otx2_cpt_enc_dec_init,
1447 .exit = otx2_cpt_skcipher_exit,
1448 .min_keysize = DES3_EDE_KEY_SIZE,
1449 .max_keysize = DES3_EDE_KEY_SIZE,
1450 .ivsize = 0,
1451 .setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1452 .encrypt = otx2_cpt_skcipher_encrypt,
1453 .decrypt = otx2_cpt_skcipher_decrypt,
1454 } };
1455
1456 static struct aead_alg otx2_cpt_aeads[] = { {
1457 .base = {
1458 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1459 .cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1460 .cra_blocksize = AES_BLOCK_SIZE,
1461 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1462 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1463 .cra_priority = 4001,
1464 .cra_alignmask = 0,
1465 .cra_module = THIS_MODULE,
1466 },
1467 .init = otx2_cpt_aead_cbc_aes_sha1_init,
1468 .exit = otx2_cpt_aead_exit,
1469 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1470 .setauthsize = otx2_cpt_aead_set_authsize,
1471 .encrypt = otx2_cpt_aead_encrypt,
1472 .decrypt = otx2_cpt_aead_decrypt,
1473 .ivsize = AES_BLOCK_SIZE,
1474 .maxauthsize = SHA1_DIGEST_SIZE,
1475 }, {
1476 .base = {
1477 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1478 .cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1479 .cra_blocksize = AES_BLOCK_SIZE,
1480 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1481 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1482 .cra_priority = 4001,
1483 .cra_alignmask = 0,
1484 .cra_module = THIS_MODULE,
1485 },
1486 .init = otx2_cpt_aead_cbc_aes_sha256_init,
1487 .exit = otx2_cpt_aead_exit,
1488 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1489 .setauthsize = otx2_cpt_aead_set_authsize,
1490 .encrypt = otx2_cpt_aead_encrypt,
1491 .decrypt = otx2_cpt_aead_decrypt,
1492 .ivsize = AES_BLOCK_SIZE,
1493 .maxauthsize = SHA256_DIGEST_SIZE,
1494 }, {
1495 .base = {
1496 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1497 .cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1498 .cra_blocksize = AES_BLOCK_SIZE,
1499 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1500 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1501 .cra_priority = 4001,
1502 .cra_alignmask = 0,
1503 .cra_module = THIS_MODULE,
1504 },
1505 .init = otx2_cpt_aead_cbc_aes_sha384_init,
1506 .exit = otx2_cpt_aead_exit,
1507 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1508 .setauthsize = otx2_cpt_aead_set_authsize,
1509 .encrypt = otx2_cpt_aead_encrypt,
1510 .decrypt = otx2_cpt_aead_decrypt,
1511 .ivsize = AES_BLOCK_SIZE,
1512 .maxauthsize = SHA384_DIGEST_SIZE,
1513 }, {
1514 .base = {
1515 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1516 .cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1517 .cra_blocksize = AES_BLOCK_SIZE,
1518 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1519 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1520 .cra_priority = 4001,
1521 .cra_alignmask = 0,
1522 .cra_module = THIS_MODULE,
1523 },
1524 .init = otx2_cpt_aead_cbc_aes_sha512_init,
1525 .exit = otx2_cpt_aead_exit,
1526 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1527 .setauthsize = otx2_cpt_aead_set_authsize,
1528 .encrypt = otx2_cpt_aead_encrypt,
1529 .decrypt = otx2_cpt_aead_decrypt,
1530 .ivsize = AES_BLOCK_SIZE,
1531 .maxauthsize = SHA512_DIGEST_SIZE,
1532 }, {
1533 .base = {
1534 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1535 .cra_driver_name = "cpt_hmac_sha1_ecb_null",
1536 .cra_blocksize = 1,
1537 .cra_flags = CRYPTO_ALG_ASYNC,
1538 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1539 .cra_priority = 4001,
1540 .cra_alignmask = 0,
1541 .cra_module = THIS_MODULE,
1542 },
1543 .init = otx2_cpt_aead_ecb_null_sha1_init,
1544 .exit = otx2_cpt_aead_exit,
1545 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1546 .setauthsize = otx2_cpt_aead_null_set_authsize,
1547 .encrypt = otx2_cpt_aead_null_encrypt,
1548 .decrypt = otx2_cpt_aead_null_decrypt,
1549 .ivsize = 0,
1550 .maxauthsize = SHA1_DIGEST_SIZE,
1551 }, {
1552 .base = {
1553 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1554 .cra_driver_name = "cpt_hmac_sha256_ecb_null",
1555 .cra_blocksize = 1,
1556 .cra_flags = CRYPTO_ALG_ASYNC,
1557 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1558 .cra_priority = 4001,
1559 .cra_alignmask = 0,
1560 .cra_module = THIS_MODULE,
1561 },
1562 .init = otx2_cpt_aead_ecb_null_sha256_init,
1563 .exit = otx2_cpt_aead_exit,
1564 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1565 .setauthsize = otx2_cpt_aead_null_set_authsize,
1566 .encrypt = otx2_cpt_aead_null_encrypt,
1567 .decrypt = otx2_cpt_aead_null_decrypt,
1568 .ivsize = 0,
1569 .maxauthsize = SHA256_DIGEST_SIZE,
1570 }, {
1571 .base = {
1572 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1573 .cra_driver_name = "cpt_hmac_sha384_ecb_null",
1574 .cra_blocksize = 1,
1575 .cra_flags = CRYPTO_ALG_ASYNC,
1576 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1577 .cra_priority = 4001,
1578 .cra_alignmask = 0,
1579 .cra_module = THIS_MODULE,
1580 },
1581 .init = otx2_cpt_aead_ecb_null_sha384_init,
1582 .exit = otx2_cpt_aead_exit,
1583 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1584 .setauthsize = otx2_cpt_aead_null_set_authsize,
1585 .encrypt = otx2_cpt_aead_null_encrypt,
1586 .decrypt = otx2_cpt_aead_null_decrypt,
1587 .ivsize = 0,
1588 .maxauthsize = SHA384_DIGEST_SIZE,
1589 }, {
1590 .base = {
1591 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1592 .cra_driver_name = "cpt_hmac_sha512_ecb_null",
1593 .cra_blocksize = 1,
1594 .cra_flags = CRYPTO_ALG_ASYNC,
1595 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1596 .cra_priority = 4001,
1597 .cra_alignmask = 0,
1598 .cra_module = THIS_MODULE,
1599 },
1600 .init = otx2_cpt_aead_ecb_null_sha512_init,
1601 .exit = otx2_cpt_aead_exit,
1602 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1603 .setauthsize = otx2_cpt_aead_null_set_authsize,
1604 .encrypt = otx2_cpt_aead_null_encrypt,
1605 .decrypt = otx2_cpt_aead_null_decrypt,
1606 .ivsize = 0,
1607 .maxauthsize = SHA512_DIGEST_SIZE,
1608 }, {
1609 .base = {
1610 .cra_name = "rfc4106(gcm(aes))",
1611 .cra_driver_name = "cpt_rfc4106_gcm_aes",
1612 .cra_blocksize = 1,
1613 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1614 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1615 .cra_priority = 4001,
1616 .cra_alignmask = 0,
1617 .cra_module = THIS_MODULE,
1618 },
1619 .init = otx2_cpt_aead_gcm_aes_init,
1620 .exit = otx2_cpt_aead_exit,
1621 .setkey = otx2_cpt_aead_gcm_aes_setkey,
1622 .setauthsize = otx2_cpt_aead_gcm_set_authsize,
1623 .encrypt = otx2_cpt_aead_encrypt,
1624 .decrypt = otx2_cpt_aead_decrypt,
1625 .ivsize = AES_GCM_IV_SIZE,
1626 .maxauthsize = AES_GCM_ICV_SIZE,
1627 } };
1628
cpt_register_algs(void)1629 static inline int cpt_register_algs(void)
1630 {
1631 int i, err = 0;
1632
1633 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1634 otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1635
1636 err = crypto_register_skciphers(otx2_cpt_skciphers,
1637 ARRAY_SIZE(otx2_cpt_skciphers));
1638 if (err)
1639 return err;
1640
1641 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1642 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1643
1644 err = crypto_register_aeads(otx2_cpt_aeads,
1645 ARRAY_SIZE(otx2_cpt_aeads));
1646 if (err) {
1647 crypto_unregister_skciphers(otx2_cpt_skciphers,
1648 ARRAY_SIZE(otx2_cpt_skciphers));
1649 return err;
1650 }
1651
1652 return 0;
1653 }
1654
cpt_unregister_algs(void)1655 static inline void cpt_unregister_algs(void)
1656 {
1657 crypto_unregister_skciphers(otx2_cpt_skciphers,
1658 ARRAY_SIZE(otx2_cpt_skciphers));
1659 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1660 }
1661
compare_func(const void * lptr,const void * rptr)1662 static int compare_func(const void *lptr, const void *rptr)
1663 {
1664 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1665 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1666
1667 if (ldesc->dev->devfn < rdesc->dev->devfn)
1668 return -1;
1669 if (ldesc->dev->devfn > rdesc->dev->devfn)
1670 return 1;
1671 return 0;
1672 }
1673
swap_func(void * lptr,void * rptr,int size)1674 static void swap_func(void *lptr, void *rptr, int size)
1675 {
1676 struct cpt_device_desc *ldesc = lptr;
1677 struct cpt_device_desc *rdesc = rptr;
1678
1679 swap(*ldesc, *rdesc);
1680 }
1681
otx2_cpt_crypto_init(struct pci_dev * pdev,struct module * mod,int num_queues,int num_devices)1682 int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1683 int num_queues, int num_devices)
1684 {
1685 int ret = 0;
1686 int count;
1687
1688 mutex_lock(&mutex);
1689 count = atomic_read(&se_devices.count);
1690 if (count >= OTX2_CPT_MAX_LFS_NUM) {
1691 dev_err(&pdev->dev, "No space to add a new device\n");
1692 ret = -ENOSPC;
1693 goto unlock;
1694 }
1695 se_devices.desc[count].num_queues = num_queues;
1696 se_devices.desc[count++].dev = pdev;
1697 atomic_inc(&se_devices.count);
1698
1699 if (atomic_read(&se_devices.count) == num_devices &&
1700 is_crypto_registered == false) {
1701 if (cpt_register_algs()) {
1702 dev_err(&pdev->dev,
1703 "Error in registering crypto algorithms\n");
1704 ret = -EINVAL;
1705 goto unlock;
1706 }
1707 try_module_get(mod);
1708 is_crypto_registered = true;
1709 }
1710 sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1711 compare_func, swap_func);
1712
1713 unlock:
1714 mutex_unlock(&mutex);
1715 return ret;
1716 }
1717
otx2_cpt_crypto_exit(struct pci_dev * pdev,struct module * mod)1718 void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1719 {
1720 struct cpt_device_table *dev_tbl;
1721 bool dev_found = false;
1722 int i, j, count;
1723
1724 mutex_lock(&mutex);
1725
1726 dev_tbl = &se_devices;
1727 count = atomic_read(&dev_tbl->count);
1728 for (i = 0; i < count; i++) {
1729 if (pdev == dev_tbl->desc[i].dev) {
1730 for (j = i; j < count-1; j++)
1731 dev_tbl->desc[j] = dev_tbl->desc[j+1];
1732 dev_found = true;
1733 break;
1734 }
1735 }
1736
1737 if (!dev_found) {
1738 dev_err(&pdev->dev, "%s device not found\n", __func__);
1739 goto unlock;
1740 }
1741 if (atomic_dec_and_test(&se_devices.count)) {
1742 cpt_unregister_algs();
1743 module_put(mod);
1744 is_crypto_registered = false;
1745 }
1746
1747 unlock:
1748 mutex_unlock(&mutex);
1749 }
1750