1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __NITROX_REQ_H
3 #define __NITROX_REQ_H
4 
5 #include <linux/dma-mapping.h>
6 #include <crypto/aes.h>
7 
8 #include "nitrox_dev.h"
9 
10 #define PENDING_SIG	0xFFFFFFFFFFFFFFFFUL
11 #define PRIO 4001
12 
13 typedef void (*sereq_completion_t)(void *req, int err);
14 
15 /**
16  * struct gphdr - General purpose Header
17  * @param0: first parameter.
18  * @param1: second parameter.
19  * @param2: third parameter.
20  * @param3: fourth parameter.
21  *
22  * Params tell the iv and enc/dec data offsets.
23  */
24 struct gphdr {
25 	__be16 param0;
26 	__be16 param1;
27 	__be16 param2;
28 	__be16 param3;
29 };
30 
31 /**
32  * struct se_req_ctrl - SE request information.
33  * @arg: Minor number of the opcode
34  * @ctxc: Context control.
35  * @unca: Uncertainity enabled.
36  * @info: Additional information for SE cores.
37  * @ctxl: Context length in bytes.
38  * @uddl: User defined data length
39  */
40 union se_req_ctrl {
41 	u64 value;
42 	struct {
43 		u64 raz	: 22;
44 		u64 arg	: 8;
45 		u64 ctxc : 2;
46 		u64 unca : 1;
47 		u64 info : 3;
48 		u64 unc : 8;
49 		u64 ctxl : 12;
50 		u64 uddl : 8;
51 	} s;
52 };
53 
54 #define MAX_IV_LEN 16
55 
56 /**
57  * struct se_crypto_request - SE crypto request structure.
58  * @opcode: Request opcode (enc/dec)
59  * @flags: flags from crypto subsystem
60  * @ctx_handle: Crypto context handle.
61  * @gph: GP Header
62  * @ctrl: Request Information.
63  * @orh: ORH address
64  * @comp: completion address
65  * @src: Input sglist
66  * @dst: Output sglist
67  */
68 struct se_crypto_request {
69 	u8 opcode;
70 	gfp_t gfp;
71 	u32 flags;
72 	u64 ctx_handle;
73 
74 	struct gphdr gph;
75 	union se_req_ctrl ctrl;
76 	u64 *orh;
77 	u64 *comp;
78 
79 	struct scatterlist *src;
80 	struct scatterlist *dst;
81 };
82 
83 /* Crypto opcodes */
84 #define FLEXI_CRYPTO_ENCRYPT_HMAC	0x33
85 #define ENCRYPT	0
86 #define DECRYPT 1
87 
88 /* IV from context */
89 #define IV_FROM_CTX	0
90 /* IV from Input data */
91 #define IV_FROM_DPTR	1
92 
93 /**
94  * cipher opcodes for firmware
95  */
96 enum flexi_cipher {
97 	CIPHER_NULL = 0,
98 	CIPHER_3DES_CBC,
99 	CIPHER_3DES_ECB,
100 	CIPHER_AES_CBC,
101 	CIPHER_AES_ECB,
102 	CIPHER_AES_CFB,
103 	CIPHER_AES_CTR,
104 	CIPHER_AES_GCM,
105 	CIPHER_AES_XTS,
106 	CIPHER_AES_CCM,
107 	CIPHER_AES_CBC_CTS,
108 	CIPHER_AES_ECB_CTS,
109 	CIPHER_INVALID
110 };
111 
112 enum flexi_auth {
113 	AUTH_NULL = 0,
114 	AUTH_MD5,
115 	AUTH_SHA1,
116 	AUTH_SHA2_SHA224,
117 	AUTH_SHA2_SHA256,
118 	AUTH_SHA2_SHA384,
119 	AUTH_SHA2_SHA512,
120 	AUTH_GMAC,
121 	AUTH_INVALID
122 };
123 
124 /**
125  * struct crypto_keys - Crypto keys
126  * @key: Encryption key or KEY1 for AES-XTS
127  * @iv: Encryption IV or Tweak for AES-XTS
128  */
129 struct crypto_keys {
130 	union {
131 		u8 key[AES_MAX_KEY_SIZE];
132 		u8 key1[AES_MAX_KEY_SIZE];
133 	} u;
134 	u8 iv[AES_BLOCK_SIZE];
135 };
136 
137 /**
138  * struct auth_keys - Authentication keys
139  * @ipad: IPAD or KEY2 for AES-XTS
140  * @opad: OPAD or AUTH KEY if auth_input_type = 1
141  */
142 struct auth_keys {
143 	union {
144 		u8 ipad[64];
145 		u8 key2[64];
146 	} u;
147 	u8 opad[64];
148 };
149 
150 union fc_ctx_flags {
151 	__be64 f;
152 	u64 fu;
153 	struct {
154 #if defined(__BIG_ENDIAN_BITFIELD)
155 		u64 cipher_type	: 4;
156 		u64 reserved_59	: 1;
157 		u64 aes_keylen : 2;
158 		u64 iv_source : 1;
159 		u64 hash_type : 4;
160 		u64 reserved_49_51 : 3;
161 		u64 auth_input_type: 1;
162 		u64 mac_len : 8;
163 		u64 reserved_0_39 : 40;
164 #else
165 		u64 reserved_0_39 : 40;
166 		u64 mac_len : 8;
167 		u64 auth_input_type: 1;
168 		u64 reserved_49_51 : 3;
169 		u64 hash_type : 4;
170 		u64 iv_source : 1;
171 		u64 aes_keylen : 2;
172 		u64 reserved_59	: 1;
173 		u64 cipher_type	: 4;
174 #endif
175 	} w0;
176 };
177 /**
178  * struct flexi_crypto_context - Crypto context
179  * @cipher_type: Encryption cipher type
180  * @aes_keylen: AES key length
181  * @iv_source: Encryption IV source
182  * @hash_type: Authentication type
183  * @auth_input_type: Authentication input type
184  *   1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
185  *   0 - Authentication OPAD/IPAD
186  * @mac_len: mac length
187  * @crypto: Crypto keys
188  * @auth: Authentication keys
189  */
190 struct flexi_crypto_context {
191 	union fc_ctx_flags flags;
192 	struct crypto_keys crypto;
193 	struct auth_keys auth;
194 };
195 
196 struct crypto_ctx_hdr {
197 	struct dma_pool *pool;
198 	dma_addr_t dma;
199 	void *vaddr;
200 };
201 
202 struct nitrox_crypto_ctx {
203 	struct nitrox_device *ndev;
204 	union {
205 		u64 ctx_handle;
206 		struct flexi_crypto_context *fctx;
207 	} u;
208 	struct crypto_ctx_hdr *chdr;
209 	sereq_completion_t callback;
210 };
211 
212 struct nitrox_kcrypt_request {
213 	struct se_crypto_request creq;
214 	u8 *src;
215 	u8 *dst;
216 	u8 *iv_out;
217 };
218 
219 /**
220  * struct nitrox_aead_rctx - AEAD request context
221  * @nkreq: Base request context
222  * @cryptlen: Encryption/Decryption data length
223  * @assoclen: AAD length
224  * @srclen: Input buffer length
225  * @dstlen: Output buffer length
226  * @iv: IV data
227  * @ivsize: IV data length
228  * @flags: AEAD req flags
229  * @ctx_handle: Device context handle
230  * @src: Source sglist
231  * @dst: Destination sglist
232  * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
233  */
234 struct nitrox_aead_rctx {
235 	struct nitrox_kcrypt_request nkreq;
236 	unsigned int cryptlen;
237 	unsigned int assoclen;
238 	unsigned int srclen;
239 	unsigned int dstlen;
240 	u8 *iv;
241 	int ivsize;
242 	u32 flags;
243 	u64 ctx_handle;
244 	struct scatterlist *src;
245 	struct scatterlist *dst;
246 	u8 ctrl_arg;
247 };
248 
249 /**
250  * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
251  * @base: AEAD request context
252  * @src: Source sglist
253  * @dst: Destination sglist
254  * @assoc: AAD
255  */
256 struct nitrox_rfc4106_rctx {
257 	struct nitrox_aead_rctx base;
258 	struct scatterlist src[3];
259 	struct scatterlist dst[3];
260 	u8 assoc[20];
261 };
262 
263 /**
264  * struct pkt_instr_hdr - Packet Instruction Header
265  * @g: Gather used
266  *   When [G] is set and [GSZ] != 0, the instruction is
267  *   indirect gather instruction.
268  *   When [G] is set and [GSZ] = 0, the instruction is
269  *   direct gather instruction.
270  * @gsz: Number of pointers in the indirect gather list
271  * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
272  *   and adds them to the packet after the pkt_instr_hdr but before any UDD
273  * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
274  *   when [IHI] is set.
275  * @fsz: The number of front data bytes directly included in the
276  *   PCIe instruction.
277  * @tlen: The length of the input packet in bytes, include:
278  *   - 16B pkt_hdr
279  *   - Inline context bytes if any,
280  *   - UDD if any,
281  *   - packet payload bytes
282  */
283 union pkt_instr_hdr {
284 	__be64 bev;
285 	u64 value;
286 	struct {
287 #if defined(__BIG_ENDIAN_BITFIELD)
288 		u64 raz_48_63 : 16;
289 		u64 g : 1;
290 		u64 gsz	: 7;
291 		u64 ihi	: 1;
292 		u64 ssz	: 7;
293 		u64 raz_30_31 : 2;
294 		u64 fsz	: 6;
295 		u64 raz_16_23 : 8;
296 		u64 tlen : 16;
297 #else
298 		u64 tlen : 16;
299 		u64 raz_16_23 : 8;
300 		u64 fsz	: 6;
301 		u64 raz_30_31 : 2;
302 		u64 ssz	: 7;
303 		u64 ihi	: 1;
304 		u64 gsz	: 7;
305 		u64 g : 1;
306 		u64 raz_48_63 : 16;
307 #endif
308 	} s;
309 };
310 
311 /**
312  * struct pkt_hdr - Packet Input Header
313  * @opcode: Request opcode (Major)
314  * @arg: Request opcode (Minor)
315  * @ctxc: Context control.
316  * @unca: When set [UNC] is the uncertainty count for an input packet.
317  *        The hardware uses uncertainty counts to predict
318  *        output buffer use and avoid deadlock.
319  * @info: Not used by input hardware. Available for use
320  *        during SE processing.
321  * @destport: The expected destination port/ring/channel for the packet.
322  * @unc: Uncertainty count for an input packet.
323  * @grp: SE group that will process the input packet.
324  * @ctxl: Context Length in 64-bit words.
325  * @uddl: User-defined data (UDD) length in bytes.
326  * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
327  */
328 union pkt_hdr {
329 	__be64 bev[2];
330 	u64 value[2];
331 	struct {
332 #if defined(__BIG_ENDIAN_BITFIELD)
333 		u64 opcode : 8;
334 		u64 arg	: 8;
335 		u64 ctxc : 2;
336 		u64 unca : 1;
337 		u64 raz_44 : 1;
338 		u64 info : 3;
339 		u64 destport : 9;
340 		u64 unc	: 8;
341 		u64 raz_19_23 : 5;
342 		u64 grp	: 3;
343 		u64 raz_15 : 1;
344 		u64 ctxl : 7;
345 		u64 uddl : 8;
346 #else
347 		u64 uddl : 8;
348 		u64 ctxl : 7;
349 		u64 raz_15 : 1;
350 		u64 grp	: 3;
351 		u64 raz_19_23 : 5;
352 		u64 unc	: 8;
353 		u64 destport : 9;
354 		u64 info : 3;
355 		u64 raz_44 : 1;
356 		u64 unca : 1;
357 		u64 ctxc : 2;
358 		u64 arg	: 8;
359 		u64 opcode : 8;
360 #endif
361 		__be64 ctxp;
362 	} s;
363 };
364 
365 /**
366  * struct slc_store_info - Solicited Paceket Output Store Information.
367  * @ssz: The number of scatterlist pointers for the solicited output port
368  *       packet.
369  * @rptr: The result pointer for the solicited output port packet.
370  *        If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
371  *        host that is large enough to hold the entire output packet.
372  *        If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
373  *        sglist components at [RPTR] on the remote host.
374  */
375 union slc_store_info {
376 	__be64 bev[2];
377 	u64 value[2];
378 	struct {
379 #if defined(__BIG_ENDIAN_BITFIELD)
380 		u64 raz_39_63 : 25;
381 		u64 ssz	: 7;
382 		u64 raz_0_31 : 32;
383 #else
384 		u64 raz_0_31 : 32;
385 		u64 ssz	: 7;
386 		u64 raz_39_63 : 25;
387 #endif
388 		__be64 rptr;
389 	} s;
390 };
391 
392 /**
393  * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
394  * @dptr0 : Input pointer points to buffer in remote host.
395  * @ih: Packet Instruction Header (8 bytes)
396  * @irh: Packet Input Header (16 bytes)
397  * @slc: Solicited Packet Output Store Information (16 bytes)
398  * @fdata: Front data
399  *
400  * 64-Byte Instruction Format
401  */
402 struct nps_pkt_instr {
403 	__be64 dptr0;
404 	union pkt_instr_hdr ih;
405 	union pkt_hdr irh;
406 	union slc_store_info slc;
407 	u64 fdata[2];
408 };
409 
410 /**
411  * struct aqmq_command_s - The 32 byte command for AE processing.
412  * @opcode: Request opcode
413  * @param1: Request control parameter 1
414  * @param2: Request control parameter 2
415  * @dlen: Input length
416  * @dptr: Input pointer points to buffer in remote host
417  * @rptr: Result pointer points to buffer in remote host
418  * @grp: AQM Group (0..7)
419  * @cptr: Context pointer
420  */
421 struct aqmq_command_s {
422 	__be16 opcode;
423 	__be16 param1;
424 	__be16 param2;
425 	__be16 dlen;
426 	__be64 dptr;
427 	__be64 rptr;
428 	union {
429 		__be64 word3;
430 #if defined(__BIG_ENDIAN_BITFIELD)
431 		u64 grp : 3;
432 		u64 cptr : 61;
433 #else
434 		u64 cptr : 61;
435 		u64 grp : 3;
436 #endif
437 	};
438 };
439 
440 /**
441  * struct ctx_hdr - Book keeping data about the crypto context
442  * @pool: Pool used to allocate crypto context
443  * @dma: Base DMA address of the crypto context
444  * @ctx_dma: Actual usable crypto context for NITROX
445  */
446 struct ctx_hdr {
447 	struct dma_pool *pool;
448 	dma_addr_t dma;
449 	dma_addr_t ctx_dma;
450 };
451 
452 /*
453  * struct sglist_component - SG list component format
454  * @len0: The number of bytes at [PTR0] on the remote host.
455  * @len1: The number of bytes at [PTR1] on the remote host.
456  * @len2: The number of bytes at [PTR2] on the remote host.
457  * @len3: The number of bytes at [PTR3] on the remote host.
458  * @dma0: First pointer point to buffer in remote host.
459  * @dma1: Second pointer point to buffer in remote host.
460  * @dma2: Third pointer point to buffer in remote host.
461  * @dma3: Fourth pointer point to buffer in remote host.
462  */
463 struct nitrox_sgcomp {
464 	__be16 len[4];
465 	__be64 dma[4];
466 };
467 
468 /*
469  * strutct nitrox_sgtable - SG list information
470  * @sgmap_cnt: Number of buffers mapped
471  * @total_bytes: Total bytes in sglist.
472  * @sgcomp_len: Total sglist components length.
473  * @sgcomp_dma: DMA address of sglist component.
474  * @sg: crypto request buffer.
475  * @sgcomp: sglist component for NITROX.
476  */
477 struct nitrox_sgtable {
478 	u8 sgmap_cnt;
479 	u16 total_bytes;
480 	u32 sgcomp_len;
481 	dma_addr_t sgcomp_dma;
482 	struct scatterlist *sg;
483 	struct nitrox_sgcomp *sgcomp;
484 };
485 
486 /* Response Header Length */
487 #define ORH_HLEN	8
488 /* Completion bytes Length */
489 #define COMP_HLEN	8
490 
491 struct resp_hdr {
492 	u64 *orh;
493 	u64 *completion;
494 };
495 
496 typedef void (*completion_t)(void *arg, int err);
497 
498 /**
499  * struct nitrox_softreq - Represents the NIROX Request.
500  * @response: response list entry
501  * @backlog: Backlog list entry
502  * @ndev: Device used to submit the request
503  * @cmdq: Command queue for submission
504  * @resp: Response headers
505  * @instr: 64B instruction
506  * @in: SG table for input
507  * @out SG table for output
508  * @tstamp: Request submitted time in jiffies
509  * @callback: callback after request completion/timeout
510  * @cb_arg: callback argument
511  */
512 struct nitrox_softreq {
513 	struct list_head response;
514 	struct list_head backlog;
515 
516 	u32 flags;
517 	gfp_t gfp;
518 	atomic_t status;
519 
520 	struct nitrox_device *ndev;
521 	struct nitrox_cmdq *cmdq;
522 
523 	struct nps_pkt_instr instr;
524 	struct resp_hdr resp;
525 	struct nitrox_sgtable in;
526 	struct nitrox_sgtable out;
527 
528 	unsigned long tstamp;
529 
530 	completion_t callback;
531 	void *cb_arg;
532 };
533 
534 static inline int flexi_aes_keylen(int keylen)
535 {
536 	int aes_keylen;
537 
538 	switch (keylen) {
539 	case AES_KEYSIZE_128:
540 		aes_keylen = 1;
541 		break;
542 	case AES_KEYSIZE_192:
543 		aes_keylen = 2;
544 		break;
545 	case AES_KEYSIZE_256:
546 		aes_keylen = 3;
547 		break;
548 	default:
549 		aes_keylen = -EINVAL;
550 		break;
551 	}
552 	return aes_keylen;
553 }
554 
555 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
556 {
557 	size_t size;
558 
559 	size = sizeof(struct scatterlist) * nents;
560 	size += extralen;
561 
562 	return kzalloc(size, gfp);
563 }
564 
565 /**
566  * create_single_sg - Point SG entry to the data
567  * @sg:		Destination SG list
568  * @buf:	Data
569  * @buflen:	Data length
570  *
571  * Returns next free entry in the destination SG list
572  **/
573 static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
574 						   void *buf, int buflen)
575 {
576 	sg_set_buf(sg, buf, buflen);
577 	sg++;
578 	return sg;
579 }
580 
581 /**
582  * create_multi_sg - Create multiple sg entries with buflen data length from
583  *		     source sglist
584  * @to_sg:	Destination SG list
585  * @from_sg:	Source SG list
586  * @buflen:	Data length
587  *
588  * Returns next free entry in the destination SG list
589  **/
590 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
591 						  struct scatterlist *from_sg,
592 						  int buflen)
593 {
594 	struct scatterlist *sg = to_sg;
595 	unsigned int sglen;
596 
597 	for (; buflen && from_sg; buflen -= sglen) {
598 		sglen = from_sg->length;
599 		if (sglen > buflen)
600 			sglen = buflen;
601 
602 		sg_set_buf(sg, sg_virt(from_sg), sglen);
603 		from_sg = sg_next(from_sg);
604 		sg++;
605 	}
606 
607 	return sg;
608 }
609 
610 static inline void set_orh_value(u64 *orh)
611 {
612 	WRITE_ONCE(*orh, PENDING_SIG);
613 }
614 
615 static inline void set_comp_value(u64 *comp)
616 {
617 	WRITE_ONCE(*comp, PENDING_SIG);
618 }
619 
620 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
621 				    int nents, int ivsize)
622 {
623 	struct se_crypto_request *creq = &nkreq->creq;
624 
625 	nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
626 	if (!nkreq->src)
627 		return -ENOMEM;
628 
629 	return 0;
630 }
631 
632 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
633 {
634 	memcpy(dst, src, size);
635 }
636 
637 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
638 {
639 	return (struct scatterlist *)(iv + ivsize);
640 }
641 
642 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
643 					  int nents, int ivsize,
644 					  struct scatterlist *src, int buflen)
645 {
646 	char *iv = nkreq->src;
647 	struct scatterlist *sg;
648 	struct se_crypto_request *creq = &nkreq->creq;
649 
650 	creq->src = nitrox_creq_src_sg(iv, ivsize);
651 	sg = creq->src;
652 	sg_init_table(sg, nents);
653 
654 	/* Input format:
655 	 * +----+----------------+
656 	 * | IV | SRC sg entries |
657 	 * +----+----------------+
658 	 */
659 
660 	/* IV */
661 	sg = create_single_sg(sg, iv, ivsize);
662 	/* SRC entries */
663 	create_multi_sg(sg, src, buflen);
664 }
665 
666 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
667 				    int nents)
668 {
669 	int extralen = ORH_HLEN + COMP_HLEN;
670 	struct se_crypto_request *creq = &nkreq->creq;
671 
672 	nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
673 	if (!nkreq->dst)
674 		return -ENOMEM;
675 
676 	return 0;
677 }
678 
679 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
680 {
681 	struct se_crypto_request *creq = &nkreq->creq;
682 
683 	creq->orh = (u64 *)(nkreq->dst);
684 	set_orh_value(creq->orh);
685 }
686 
687 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
688 {
689 	struct se_crypto_request *creq = &nkreq->creq;
690 
691 	creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
692 	set_comp_value(creq->comp);
693 }
694 
695 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
696 {
697 	return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
698 }
699 
700 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
701 					  int nents, int ivsize,
702 					  struct scatterlist *dst, int buflen)
703 {
704 	struct se_crypto_request *creq = &nkreq->creq;
705 	struct scatterlist *sg;
706 	char *iv = nkreq->src;
707 
708 	creq->dst = nitrox_creq_dst_sg(nkreq->dst);
709 	sg = creq->dst;
710 	sg_init_table(sg, nents);
711 
712 	/* Output format:
713 	 * +-----+----+----------------+-----------------+
714 	 * | ORH | IV | DST sg entries | COMPLETION Bytes|
715 	 * +-----+----+----------------+-----------------+
716 	 */
717 
718 	/* ORH */
719 	sg = create_single_sg(sg, creq->orh, ORH_HLEN);
720 	/* IV */
721 	sg = create_single_sg(sg, iv, ivsize);
722 	/* DST entries */
723 	sg = create_multi_sg(sg, dst, buflen);
724 	/* COMPLETION Bytes */
725 	create_single_sg(sg, creq->comp, COMP_HLEN);
726 }
727 
728 #endif /* __NITROX_REQ_H */
729