1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __NITROX_REQ_H
3 #define __NITROX_REQ_H
4 
5 #include <linux/dma-mapping.h>
6 #include <crypto/aes.h>
7 
8 #include "nitrox_dev.h"
9 
10 #define PENDING_SIG	0xFFFFFFFFFFFFFFFFUL
11 #define PRIO 4001
12 
13 /**
14  * struct gphdr - General purpose Header
15  * @param0: first parameter.
16  * @param1: second parameter.
17  * @param2: third parameter.
18  * @param3: fourth parameter.
19  *
20  * Params tell the iv and enc/dec data offsets.
21  */
22 struct gphdr {
23 	__be16 param0;
24 	__be16 param1;
25 	__be16 param2;
26 	__be16 param3;
27 };
28 
29 /**
30  * struct se_req_ctrl - SE request information.
31  * @arg: Minor number of the opcode
32  * @ctxc: Context control.
33  * @unca: Uncertainity enabled.
34  * @info: Additional information for SE cores.
35  * @ctxl: Context length in bytes.
36  * @uddl: User defined data length
37  */
38 union se_req_ctrl {
39 	u64 value;
40 	struct {
41 		u64 raz	: 22;
42 		u64 arg	: 8;
43 		u64 ctxc : 2;
44 		u64 unca : 1;
45 		u64 info : 3;
46 		u64 unc : 8;
47 		u64 ctxl : 12;
48 		u64 uddl : 8;
49 	} s;
50 };
51 
52 #define MAX_IV_LEN 16
53 
54 /**
55  * struct se_crypto_request - SE crypto request structure.
56  * @opcode: Request opcode (enc/dec)
57  * @flags: flags from crypto subsystem
58  * @ctx_handle: Crypto context handle.
59  * @gph: GP Header
60  * @ctrl: Request Information.
61  * @orh: ORH address
62  * @comp: completion address
63  * @src: Input sglist
64  * @dst: Output sglist
65  */
66 struct se_crypto_request {
67 	u8 opcode;
68 	gfp_t gfp;
69 	u32 flags;
70 	u64 ctx_handle;
71 
72 	struct gphdr gph;
73 	union se_req_ctrl ctrl;
74 	u64 *orh;
75 	u64 *comp;
76 
77 	struct scatterlist *src;
78 	struct scatterlist *dst;
79 };
80 
81 /* Crypto opcodes */
82 #define FLEXI_CRYPTO_ENCRYPT_HMAC	0x33
83 #define ENCRYPT	0
84 #define DECRYPT 1
85 
86 /* IV from context */
87 #define IV_FROM_CTX	0
88 /* IV from Input data */
89 #define IV_FROM_DPTR	1
90 
91 /**
92  * cipher opcodes for firmware
93  */
94 enum flexi_cipher {
95 	CIPHER_NULL = 0,
96 	CIPHER_3DES_CBC,
97 	CIPHER_3DES_ECB,
98 	CIPHER_AES_CBC,
99 	CIPHER_AES_ECB,
100 	CIPHER_AES_CFB,
101 	CIPHER_AES_CTR,
102 	CIPHER_AES_GCM,
103 	CIPHER_AES_XTS,
104 	CIPHER_AES_CCM,
105 	CIPHER_AES_CBC_CTS,
106 	CIPHER_AES_ECB_CTS,
107 	CIPHER_INVALID
108 };
109 
110 enum flexi_auth {
111 	AUTH_NULL = 0,
112 	AUTH_MD5,
113 	AUTH_SHA1,
114 	AUTH_SHA2_SHA224,
115 	AUTH_SHA2_SHA256,
116 	AUTH_SHA2_SHA384,
117 	AUTH_SHA2_SHA512,
118 	AUTH_GMAC,
119 	AUTH_INVALID
120 };
121 
122 /**
123  * struct crypto_keys - Crypto keys
124  * @key: Encryption key or KEY1 for AES-XTS
125  * @iv: Encryption IV or Tweak for AES-XTS
126  */
127 struct crypto_keys {
128 	union {
129 		u8 key[AES_MAX_KEY_SIZE];
130 		u8 key1[AES_MAX_KEY_SIZE];
131 	} u;
132 	u8 iv[AES_BLOCK_SIZE];
133 };
134 
135 /**
136  * struct auth_keys - Authentication keys
137  * @ipad: IPAD or KEY2 for AES-XTS
138  * @opad: OPAD or AUTH KEY if auth_input_type = 1
139  */
140 struct auth_keys {
141 	union {
142 		u8 ipad[64];
143 		u8 key2[64];
144 	} u;
145 	u8 opad[64];
146 };
147 
148 union fc_ctx_flags {
149 	__be64 f;
150 	struct {
151 #if defined(__BIG_ENDIAN_BITFIELD)
152 		u64 cipher_type	: 4;
153 		u64 reserved_59	: 1;
154 		u64 aes_keylen : 2;
155 		u64 iv_source : 1;
156 		u64 hash_type : 4;
157 		u64 reserved_49_51 : 3;
158 		u64 auth_input_type: 1;
159 		u64 mac_len : 8;
160 		u64 reserved_0_39 : 40;
161 #else
162 		u64 reserved_0_39 : 40;
163 		u64 mac_len : 8;
164 		u64 auth_input_type: 1;
165 		u64 reserved_49_51 : 3;
166 		u64 hash_type : 4;
167 		u64 iv_source : 1;
168 		u64 aes_keylen : 2;
169 		u64 reserved_59	: 1;
170 		u64 cipher_type	: 4;
171 #endif
172 	} w0;
173 };
174 /**
175  * struct flexi_crypto_context - Crypto context
176  * @cipher_type: Encryption cipher type
177  * @aes_keylen: AES key length
178  * @iv_source: Encryption IV source
179  * @hash_type: Authentication type
180  * @auth_input_type: Authentication input type
181  *   1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
182  *   0 - Authentication OPAD/IPAD
183  * @mac_len: mac length
184  * @crypto: Crypto keys
185  * @auth: Authentication keys
186  */
187 struct flexi_crypto_context {
188 	union fc_ctx_flags flags;
189 	struct crypto_keys crypto;
190 	struct auth_keys auth;
191 };
192 
193 struct crypto_ctx_hdr {
194 	struct dma_pool *pool;
195 	dma_addr_t dma;
196 	void *vaddr;
197 };
198 
199 struct nitrox_crypto_ctx {
200 	struct nitrox_device *ndev;
201 	union {
202 		u64 ctx_handle;
203 		struct flexi_crypto_context *fctx;
204 	} u;
205 	struct crypto_ctx_hdr *chdr;
206 };
207 
208 struct nitrox_kcrypt_request {
209 	struct se_crypto_request creq;
210 	u8 *src;
211 	u8 *dst;
212 };
213 
214 /**
215  * struct nitrox_aead_rctx - AEAD request context
216  * @nkreq: Base request context
217  * @cryptlen: Encryption/Decryption data length
218  * @assoclen: AAD length
219  * @srclen: Input buffer length
220  * @dstlen: Output buffer length
221  * @iv: IV data
222  * @ivsize: IV data length
223  * @flags: AEAD req flags
224  * @ctx_handle: Device context handle
225  * @src: Source sglist
226  * @dst: Destination sglist
227  * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
228  */
229 struct nitrox_aead_rctx {
230 	struct nitrox_kcrypt_request nkreq;
231 	unsigned int cryptlen;
232 	unsigned int assoclen;
233 	unsigned int srclen;
234 	unsigned int dstlen;
235 	u8 *iv;
236 	int ivsize;
237 	u32 flags;
238 	u64 ctx_handle;
239 	struct scatterlist *src;
240 	struct scatterlist *dst;
241 	u8 ctrl_arg;
242 };
243 
244 /**
245  * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
246  * @base: AEAD request context
247  * @src: Source sglist
248  * @dst: Destination sglist
249  * @assoc: AAD
250  */
251 struct nitrox_rfc4106_rctx {
252 	struct nitrox_aead_rctx base;
253 	struct scatterlist src[3];
254 	struct scatterlist dst[3];
255 	u8 assoc[20];
256 };
257 
258 /**
259  * struct pkt_instr_hdr - Packet Instruction Header
260  * @g: Gather used
261  *   When [G] is set and [GSZ] != 0, the instruction is
262  *   indirect gather instruction.
263  *   When [G] is set and [GSZ] = 0, the instruction is
264  *   direct gather instruction.
265  * @gsz: Number of pointers in the indirect gather list
266  * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
267  *   and adds them to the packet after the pkt_instr_hdr but before any UDD
268  * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
269  *   when [IHI] is set.
270  * @fsz: The number of front data bytes directly included in the
271  *   PCIe instruction.
272  * @tlen: The length of the input packet in bytes, include:
273  *   - 16B pkt_hdr
274  *   - Inline context bytes if any,
275  *   - UDD if any,
276  *   - packet payload bytes
277  */
278 union pkt_instr_hdr {
279 	u64 value;
280 	struct {
281 #if defined(__BIG_ENDIAN_BITFIELD)
282 		u64 raz_48_63 : 16;
283 		u64 g : 1;
284 		u64 gsz	: 7;
285 		u64 ihi	: 1;
286 		u64 ssz	: 7;
287 		u64 raz_30_31 : 2;
288 		u64 fsz	: 6;
289 		u64 raz_16_23 : 8;
290 		u64 tlen : 16;
291 #else
292 		u64 tlen : 16;
293 		u64 raz_16_23 : 8;
294 		u64 fsz	: 6;
295 		u64 raz_30_31 : 2;
296 		u64 ssz	: 7;
297 		u64 ihi	: 1;
298 		u64 gsz	: 7;
299 		u64 g : 1;
300 		u64 raz_48_63 : 16;
301 #endif
302 	} s;
303 };
304 
305 /**
306  * struct pkt_hdr - Packet Input Header
307  * @opcode: Request opcode (Major)
308  * @arg: Request opcode (Minor)
309  * @ctxc: Context control.
310  * @unca: When set [UNC] is the uncertainty count for an input packet.
311  *        The hardware uses uncertainty counts to predict
312  *        output buffer use and avoid deadlock.
313  * @info: Not used by input hardware. Available for use
314  *        during SE processing.
315  * @destport: The expected destination port/ring/channel for the packet.
316  * @unc: Uncertainty count for an input packet.
317  * @grp: SE group that will process the input packet.
318  * @ctxl: Context Length in 64-bit words.
319  * @uddl: User-defined data (UDD) length in bytes.
320  * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
321  */
322 union pkt_hdr {
323 	u64 value[2];
324 	struct {
325 #if defined(__BIG_ENDIAN_BITFIELD)
326 		u64 opcode : 8;
327 		u64 arg	: 8;
328 		u64 ctxc : 2;
329 		u64 unca : 1;
330 		u64 raz_44 : 1;
331 		u64 info : 3;
332 		u64 destport : 9;
333 		u64 unc	: 8;
334 		u64 raz_19_23 : 5;
335 		u64 grp	: 3;
336 		u64 raz_15 : 1;
337 		u64 ctxl : 7;
338 		u64 uddl : 8;
339 #else
340 		u64 uddl : 8;
341 		u64 ctxl : 7;
342 		u64 raz_15 : 1;
343 		u64 grp	: 3;
344 		u64 raz_19_23 : 5;
345 		u64 unc	: 8;
346 		u64 destport : 9;
347 		u64 info : 3;
348 		u64 raz_44 : 1;
349 		u64 unca : 1;
350 		u64 ctxc : 2;
351 		u64 arg	: 8;
352 		u64 opcode : 8;
353 #endif
354 		__be64 ctxp;
355 	} s;
356 };
357 
358 /**
359  * struct slc_store_info - Solicited Paceket Output Store Information.
360  * @ssz: The number of scatterlist pointers for the solicited output port
361  *       packet.
362  * @rptr: The result pointer for the solicited output port packet.
363  *        If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
364  *        host that is large enough to hold the entire output packet.
365  *        If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
366  *        sglist components at [RPTR] on the remote host.
367  */
368 union slc_store_info {
369 	u64 value[2];
370 	struct {
371 #if defined(__BIG_ENDIAN_BITFIELD)
372 		u64 raz_39_63 : 25;
373 		u64 ssz	: 7;
374 		u64 raz_0_31 : 32;
375 #else
376 		u64 raz_0_31 : 32;
377 		u64 ssz	: 7;
378 		u64 raz_39_63 : 25;
379 #endif
380 		__be64 rptr;
381 	} s;
382 };
383 
384 /**
385  * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
386  * @dptr0 : Input pointer points to buffer in remote host.
387  * @ih: Packet Instruction Header (8 bytes)
388  * @irh: Packet Input Header (16 bytes)
389  * @slc: Solicited Packet Output Store Information (16 bytes)
390  * @fdata: Front data
391  *
392  * 64-Byte Instruction Format
393  */
394 struct nps_pkt_instr {
395 	__be64 dptr0;
396 	union pkt_instr_hdr ih;
397 	union pkt_hdr irh;
398 	union slc_store_info slc;
399 	u64 fdata[2];
400 };
401 
402 /**
403  * struct ctx_hdr - Book keeping data about the crypto context
404  * @pool: Pool used to allocate crypto context
405  * @dma: Base DMA address of the cypto context
406  * @ctx_dma: Actual usable crypto context for NITROX
407  */
408 struct ctx_hdr {
409 	struct dma_pool *pool;
410 	dma_addr_t dma;
411 	dma_addr_t ctx_dma;
412 };
413 
414 /*
415  * struct sglist_component - SG list component format
416  * @len0: The number of bytes at [PTR0] on the remote host.
417  * @len1: The number of bytes at [PTR1] on the remote host.
418  * @len2: The number of bytes at [PTR2] on the remote host.
419  * @len3: The number of bytes at [PTR3] on the remote host.
420  * @dma0: First pointer point to buffer in remote host.
421  * @dma1: Second pointer point to buffer in remote host.
422  * @dma2: Third pointer point to buffer in remote host.
423  * @dma3: Fourth pointer point to buffer in remote host.
424  */
425 struct nitrox_sgcomp {
426 	__be16 len[4];
427 	__be64 dma[4];
428 };
429 
430 /*
431  * strutct nitrox_sgtable - SG list information
432  * @sgmap_cnt: Number of buffers mapped
433  * @total_bytes: Total bytes in sglist.
434  * @sgcomp_len: Total sglist components length.
435  * @sgcomp_dma: DMA address of sglist component.
436  * @sg: crypto request buffer.
437  * @sgcomp: sglist component for NITROX.
438  */
439 struct nitrox_sgtable {
440 	u8 sgmap_cnt;
441 	u16 total_bytes;
442 	u32 sgcomp_len;
443 	dma_addr_t sgcomp_dma;
444 	struct scatterlist *sg;
445 	struct nitrox_sgcomp *sgcomp;
446 };
447 
448 /* Response Header Length */
449 #define ORH_HLEN	8
450 /* Completion bytes Length */
451 #define COMP_HLEN	8
452 
453 struct resp_hdr {
454 	u64 *orh;
455 	u64 *completion;
456 };
457 
458 typedef void (*completion_t)(void *arg, int err);
459 
460 /**
461  * struct nitrox_softreq - Represents the NIROX Request.
462  * @response: response list entry
463  * @backlog: Backlog list entry
464  * @ndev: Device used to submit the request
465  * @cmdq: Command queue for submission
466  * @resp: Response headers
467  * @instr: 64B instruction
468  * @in: SG table for input
469  * @out SG table for output
470  * @tstamp: Request submitted time in jiffies
471  * @callback: callback after request completion/timeout
472  * @cb_arg: callback argument
473  */
474 struct nitrox_softreq {
475 	struct list_head response;
476 	struct list_head backlog;
477 
478 	u32 flags;
479 	gfp_t gfp;
480 	atomic_t status;
481 
482 	struct nitrox_device *ndev;
483 	struct nitrox_cmdq *cmdq;
484 
485 	struct nps_pkt_instr instr;
486 	struct resp_hdr resp;
487 	struct nitrox_sgtable in;
488 	struct nitrox_sgtable out;
489 
490 	unsigned long tstamp;
491 
492 	completion_t callback;
493 	void *cb_arg;
494 };
495 
496 static inline int flexi_aes_keylen(int keylen)
497 {
498 	int aes_keylen;
499 
500 	switch (keylen) {
501 	case AES_KEYSIZE_128:
502 		aes_keylen = 1;
503 		break;
504 	case AES_KEYSIZE_192:
505 		aes_keylen = 2;
506 		break;
507 	case AES_KEYSIZE_256:
508 		aes_keylen = 3;
509 		break;
510 	default:
511 		aes_keylen = -EINVAL;
512 		break;
513 	}
514 	return aes_keylen;
515 }
516 
517 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
518 {
519 	size_t size;
520 
521 	size = sizeof(struct scatterlist) * nents;
522 	size += extralen;
523 
524 	return kzalloc(size, gfp);
525 }
526 
527 /**
528  * create_single_sg - Point SG entry to the data
529  * @sg:		Destination SG list
530  * @buf:	Data
531  * @buflen:	Data length
532  *
533  * Returns next free entry in the destination SG list
534  **/
535 static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
536 						   void *buf, int buflen)
537 {
538 	sg_set_buf(sg, buf, buflen);
539 	sg++;
540 	return sg;
541 }
542 
543 /**
544  * create_multi_sg - Create multiple sg entries with buflen data length from
545  *		     source sglist
546  * @to_sg:	Destination SG list
547  * @from_sg:	Source SG list
548  * @buflen:	Data length
549  *
550  * Returns next free entry in the destination SG list
551  **/
552 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
553 						  struct scatterlist *from_sg,
554 						  int buflen)
555 {
556 	struct scatterlist *sg = to_sg;
557 	unsigned int sglen;
558 
559 	for (; buflen && from_sg; buflen -= sglen) {
560 		sglen = from_sg->length;
561 		if (sglen > buflen)
562 			sglen = buflen;
563 
564 		sg_set_buf(sg, sg_virt(from_sg), sglen);
565 		from_sg = sg_next(from_sg);
566 		sg++;
567 	}
568 
569 	return sg;
570 }
571 
572 static inline void set_orh_value(u64 *orh)
573 {
574 	WRITE_ONCE(*orh, PENDING_SIG);
575 }
576 
577 static inline void set_comp_value(u64 *comp)
578 {
579 	WRITE_ONCE(*comp, PENDING_SIG);
580 }
581 
582 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
583 				    int nents, int ivsize)
584 {
585 	struct se_crypto_request *creq = &nkreq->creq;
586 
587 	nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
588 	if (!nkreq->src)
589 		return -ENOMEM;
590 
591 	return 0;
592 }
593 
594 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
595 {
596 	memcpy(dst, src, size);
597 }
598 
599 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
600 {
601 	return (struct scatterlist *)(iv + ivsize);
602 }
603 
604 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
605 					  int nents, int ivsize,
606 					  struct scatterlist *src, int buflen)
607 {
608 	char *iv = nkreq->src;
609 	struct scatterlist *sg;
610 	struct se_crypto_request *creq = &nkreq->creq;
611 
612 	creq->src = nitrox_creq_src_sg(iv, ivsize);
613 	sg = creq->src;
614 	sg_init_table(sg, nents);
615 
616 	/* Input format:
617 	 * +----+----------------+
618 	 * | IV | SRC sg entries |
619 	 * +----+----------------+
620 	 */
621 
622 	/* IV */
623 	sg = create_single_sg(sg, iv, ivsize);
624 	/* SRC entries */
625 	create_multi_sg(sg, src, buflen);
626 }
627 
628 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
629 				    int nents)
630 {
631 	int extralen = ORH_HLEN + COMP_HLEN;
632 	struct se_crypto_request *creq = &nkreq->creq;
633 
634 	nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
635 	if (!nkreq->dst)
636 		return -ENOMEM;
637 
638 	return 0;
639 }
640 
641 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
642 {
643 	struct se_crypto_request *creq = &nkreq->creq;
644 
645 	creq->orh = (u64 *)(nkreq->dst);
646 	set_orh_value(creq->orh);
647 }
648 
649 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
650 {
651 	struct se_crypto_request *creq = &nkreq->creq;
652 
653 	creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
654 	set_comp_value(creq->comp);
655 }
656 
657 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
658 {
659 	return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
660 }
661 
662 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
663 					  int nents, int ivsize,
664 					  struct scatterlist *dst, int buflen)
665 {
666 	struct se_crypto_request *creq = &nkreq->creq;
667 	struct scatterlist *sg;
668 	char *iv = nkreq->src;
669 
670 	creq->dst = nitrox_creq_dst_sg(nkreq->dst);
671 	sg = creq->dst;
672 	sg_init_table(sg, nents);
673 
674 	/* Output format:
675 	 * +-----+----+----------------+-----------------+
676 	 * | ORH | IV | DST sg entries | COMPLETION Bytes|
677 	 * +-----+----+----------------+-----------------+
678 	 */
679 
680 	/* ORH */
681 	sg = create_single_sg(sg, creq->orh, ORH_HLEN);
682 	/* IV */
683 	sg = create_single_sg(sg, iv, ivsize);
684 	/* DST entries */
685 	sg = create_multi_sg(sg, dst, buflen);
686 	/* COMPLETION Bytes */
687 	create_single_sg(sg, creq->comp, COMP_HLEN);
688 }
689 
690 #endif /* __NITROX_REQ_H */
691