1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NITROX_REQ_H 3 #define __NITROX_REQ_H 4 5 #include <linux/dma-mapping.h> 6 #include <crypto/aes.h> 7 8 #include "nitrox_dev.h" 9 10 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL 11 #define PRIO 4001 12 13 /** 14 * struct gphdr - General purpose Header 15 * @param0: first parameter. 16 * @param1: second parameter. 17 * @param2: third parameter. 18 * @param3: fourth parameter. 19 * 20 * Params tell the iv and enc/dec data offsets. 21 */ 22 struct gphdr { 23 __be16 param0; 24 __be16 param1; 25 __be16 param2; 26 __be16 param3; 27 }; 28 29 /** 30 * struct se_req_ctrl - SE request information. 31 * @arg: Minor number of the opcode 32 * @ctxc: Context control. 33 * @unca: Uncertainity enabled. 34 * @info: Additional information for SE cores. 35 * @ctxl: Context length in bytes. 36 * @uddl: User defined data length 37 */ 38 union se_req_ctrl { 39 u64 value; 40 struct { 41 u64 raz : 22; 42 u64 arg : 8; 43 u64 ctxc : 2; 44 u64 unca : 1; 45 u64 info : 3; 46 u64 unc : 8; 47 u64 ctxl : 12; 48 u64 uddl : 8; 49 } s; 50 }; 51 52 #define MAX_IV_LEN 16 53 54 /** 55 * struct se_crypto_request - SE crypto request structure. 56 * @opcode: Request opcode (enc/dec) 57 * @flags: flags from crypto subsystem 58 * @ctx_handle: Crypto context handle. 59 * @gph: GP Header 60 * @ctrl: Request Information. 61 * @orh: ORH address 62 * @comp: completion address 63 * @src: Input sglist 64 * @dst: Output sglist 65 */ 66 struct se_crypto_request { 67 u8 opcode; 68 gfp_t gfp; 69 u32 flags; 70 u64 ctx_handle; 71 72 struct gphdr gph; 73 union se_req_ctrl ctrl; 74 u64 *orh; 75 u64 *comp; 76 77 struct scatterlist *src; 78 struct scatterlist *dst; 79 }; 80 81 /* Crypto opcodes */ 82 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 83 #define ENCRYPT 0 84 #define DECRYPT 1 85 86 /* IV from context */ 87 #define IV_FROM_CTX 0 88 /* IV from Input data */ 89 #define IV_FROM_DPTR 1 90 91 /** 92 * cipher opcodes for firmware 93 */ 94 enum flexi_cipher { 95 CIPHER_NULL = 0, 96 CIPHER_3DES_CBC, 97 CIPHER_3DES_ECB, 98 CIPHER_AES_CBC, 99 CIPHER_AES_ECB, 100 CIPHER_AES_CFB, 101 CIPHER_AES_CTR, 102 CIPHER_AES_GCM, 103 CIPHER_AES_XTS, 104 CIPHER_AES_CCM, 105 CIPHER_AES_CBC_CTS, 106 CIPHER_AES_ECB_CTS, 107 CIPHER_INVALID 108 }; 109 110 enum flexi_auth { 111 AUTH_NULL = 0, 112 AUTH_MD5, 113 AUTH_SHA1, 114 AUTH_SHA2_SHA224, 115 AUTH_SHA2_SHA256, 116 AUTH_SHA2_SHA384, 117 AUTH_SHA2_SHA512, 118 AUTH_GMAC, 119 AUTH_INVALID 120 }; 121 122 /** 123 * struct crypto_keys - Crypto keys 124 * @key: Encryption key or KEY1 for AES-XTS 125 * @iv: Encryption IV or Tweak for AES-XTS 126 */ 127 struct crypto_keys { 128 union { 129 u8 key[AES_MAX_KEY_SIZE]; 130 u8 key1[AES_MAX_KEY_SIZE]; 131 } u; 132 u8 iv[AES_BLOCK_SIZE]; 133 }; 134 135 /** 136 * struct auth_keys - Authentication keys 137 * @ipad: IPAD or KEY2 for AES-XTS 138 * @opad: OPAD or AUTH KEY if auth_input_type = 1 139 */ 140 struct auth_keys { 141 union { 142 u8 ipad[64]; 143 u8 key2[64]; 144 } u; 145 u8 opad[64]; 146 }; 147 148 union fc_ctx_flags { 149 __be64 f; 150 struct { 151 #if defined(__BIG_ENDIAN_BITFIELD) 152 u64 cipher_type : 4; 153 u64 reserved_59 : 1; 154 u64 aes_keylen : 2; 155 u64 iv_source : 1; 156 u64 hash_type : 4; 157 u64 reserved_49_51 : 3; 158 u64 auth_input_type: 1; 159 u64 mac_len : 8; 160 u64 reserved_0_39 : 40; 161 #else 162 u64 reserved_0_39 : 40; 163 u64 mac_len : 8; 164 u64 auth_input_type: 1; 165 u64 reserved_49_51 : 3; 166 u64 hash_type : 4; 167 u64 iv_source : 1; 168 u64 aes_keylen : 2; 169 u64 reserved_59 : 1; 170 u64 cipher_type : 4; 171 #endif 172 } w0; 173 }; 174 /** 175 * struct flexi_crypto_context - Crypto context 176 * @cipher_type: Encryption cipher type 177 * @aes_keylen: AES key length 178 * @iv_source: Encryption IV source 179 * @hash_type: Authentication type 180 * @auth_input_type: Authentication input type 181 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD 182 * 0 - Authentication OPAD/IPAD 183 * @mac_len: mac length 184 * @crypto: Crypto keys 185 * @auth: Authentication keys 186 */ 187 struct flexi_crypto_context { 188 union fc_ctx_flags flags; 189 struct crypto_keys crypto; 190 struct auth_keys auth; 191 }; 192 193 struct crypto_ctx_hdr { 194 struct dma_pool *pool; 195 dma_addr_t dma; 196 void *vaddr; 197 }; 198 199 struct nitrox_crypto_ctx { 200 struct nitrox_device *ndev; 201 union { 202 u64 ctx_handle; 203 struct flexi_crypto_context *fctx; 204 } u; 205 struct crypto_ctx_hdr *chdr; 206 }; 207 208 struct nitrox_kcrypt_request { 209 struct se_crypto_request creq; 210 u8 *src; 211 u8 *dst; 212 }; 213 214 /** 215 * struct nitrox_aead_rctx - AEAD request context 216 * @nkreq: Base request context 217 * @cryptlen: Encryption/Decryption data length 218 * @assoclen: AAD length 219 * @srclen: Input buffer length 220 * @dstlen: Output buffer length 221 * @iv: IV data 222 * @ivsize: IV data length 223 * @flags: AEAD req flags 224 * @ctx_handle: Device context handle 225 * @src: Source sglist 226 * @dst: Destination sglist 227 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) 228 */ 229 struct nitrox_aead_rctx { 230 struct nitrox_kcrypt_request nkreq; 231 unsigned int cryptlen; 232 unsigned int assoclen; 233 unsigned int srclen; 234 unsigned int dstlen; 235 u8 *iv; 236 int ivsize; 237 u32 flags; 238 u64 ctx_handle; 239 struct scatterlist *src; 240 struct scatterlist *dst; 241 u8 ctrl_arg; 242 }; 243 244 /** 245 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context 246 * @base: AEAD request context 247 * @src: Source sglist 248 * @dst: Destination sglist 249 * @assoc: AAD 250 */ 251 struct nitrox_rfc4106_rctx { 252 struct nitrox_aead_rctx base; 253 struct scatterlist src[3]; 254 struct scatterlist dst[3]; 255 u8 assoc[20]; 256 }; 257 258 /** 259 * struct pkt_instr_hdr - Packet Instruction Header 260 * @g: Gather used 261 * When [G] is set and [GSZ] != 0, the instruction is 262 * indirect gather instruction. 263 * When [G] is set and [GSZ] = 0, the instruction is 264 * direct gather instruction. 265 * @gsz: Number of pointers in the indirect gather list 266 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr 267 * and adds them to the packet after the pkt_instr_hdr but before any UDD 268 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] 269 * when [IHI] is set. 270 * @fsz: The number of front data bytes directly included in the 271 * PCIe instruction. 272 * @tlen: The length of the input packet in bytes, include: 273 * - 16B pkt_hdr 274 * - Inline context bytes if any, 275 * - UDD if any, 276 * - packet payload bytes 277 */ 278 union pkt_instr_hdr { 279 u64 value; 280 struct { 281 #if defined(__BIG_ENDIAN_BITFIELD) 282 u64 raz_48_63 : 16; 283 u64 g : 1; 284 u64 gsz : 7; 285 u64 ihi : 1; 286 u64 ssz : 7; 287 u64 raz_30_31 : 2; 288 u64 fsz : 6; 289 u64 raz_16_23 : 8; 290 u64 tlen : 16; 291 #else 292 u64 tlen : 16; 293 u64 raz_16_23 : 8; 294 u64 fsz : 6; 295 u64 raz_30_31 : 2; 296 u64 ssz : 7; 297 u64 ihi : 1; 298 u64 gsz : 7; 299 u64 g : 1; 300 u64 raz_48_63 : 16; 301 #endif 302 } s; 303 }; 304 305 /** 306 * struct pkt_hdr - Packet Input Header 307 * @opcode: Request opcode (Major) 308 * @arg: Request opcode (Minor) 309 * @ctxc: Context control. 310 * @unca: When set [UNC] is the uncertainty count for an input packet. 311 * The hardware uses uncertainty counts to predict 312 * output buffer use and avoid deadlock. 313 * @info: Not used by input hardware. Available for use 314 * during SE processing. 315 * @destport: The expected destination port/ring/channel for the packet. 316 * @unc: Uncertainty count for an input packet. 317 * @grp: SE group that will process the input packet. 318 * @ctxl: Context Length in 64-bit words. 319 * @uddl: User-defined data (UDD) length in bytes. 320 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. 321 */ 322 union pkt_hdr { 323 u64 value[2]; 324 struct { 325 #if defined(__BIG_ENDIAN_BITFIELD) 326 u64 opcode : 8; 327 u64 arg : 8; 328 u64 ctxc : 2; 329 u64 unca : 1; 330 u64 raz_44 : 1; 331 u64 info : 3; 332 u64 destport : 9; 333 u64 unc : 8; 334 u64 raz_19_23 : 5; 335 u64 grp : 3; 336 u64 raz_15 : 1; 337 u64 ctxl : 7; 338 u64 uddl : 8; 339 #else 340 u64 uddl : 8; 341 u64 ctxl : 7; 342 u64 raz_15 : 1; 343 u64 grp : 3; 344 u64 raz_19_23 : 5; 345 u64 unc : 8; 346 u64 destport : 9; 347 u64 info : 3; 348 u64 raz_44 : 1; 349 u64 unca : 1; 350 u64 ctxc : 2; 351 u64 arg : 8; 352 u64 opcode : 8; 353 #endif 354 __be64 ctxp; 355 } s; 356 }; 357 358 /** 359 * struct slc_store_info - Solicited Paceket Output Store Information. 360 * @ssz: The number of scatterlist pointers for the solicited output port 361 * packet. 362 * @rptr: The result pointer for the solicited output port packet. 363 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote 364 * host that is large enough to hold the entire output packet. 365 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 366 * sglist components at [RPTR] on the remote host. 367 */ 368 union slc_store_info { 369 u64 value[2]; 370 struct { 371 #if defined(__BIG_ENDIAN_BITFIELD) 372 u64 raz_39_63 : 25; 373 u64 ssz : 7; 374 u64 raz_0_31 : 32; 375 #else 376 u64 raz_0_31 : 32; 377 u64 ssz : 7; 378 u64 raz_39_63 : 25; 379 #endif 380 __be64 rptr; 381 } s; 382 }; 383 384 /** 385 * struct nps_pkt_instr - NPS Packet Instruction of SE cores. 386 * @dptr0 : Input pointer points to buffer in remote host. 387 * @ih: Packet Instruction Header (8 bytes) 388 * @irh: Packet Input Header (16 bytes) 389 * @slc: Solicited Packet Output Store Information (16 bytes) 390 * @fdata: Front data 391 * 392 * 64-Byte Instruction Format 393 */ 394 struct nps_pkt_instr { 395 __be64 dptr0; 396 union pkt_instr_hdr ih; 397 union pkt_hdr irh; 398 union slc_store_info slc; 399 u64 fdata[2]; 400 }; 401 402 /** 403 * struct aqmq_command_s - The 32 byte command for AE processing. 404 * @opcode: Request opcode 405 * @param1: Request control parameter 1 406 * @param2: Request control parameter 2 407 * @dlen: Input length 408 * @dptr: Input pointer points to buffer in remote host 409 * @rptr: Result pointer points to buffer in remote host 410 * @grp: AQM Group (0..7) 411 * @cptr: Context pointer 412 */ 413 struct aqmq_command_s { 414 __be16 opcode; 415 __be16 param1; 416 __be16 param2; 417 __be16 dlen; 418 __be64 dptr; 419 __be64 rptr; 420 union { 421 __be64 word3; 422 #if defined(__BIG_ENDIAN_BITFIELD) 423 u64 grp : 3; 424 u64 cptr : 61; 425 #else 426 u64 cptr : 61; 427 u64 grp : 3; 428 #endif 429 }; 430 }; 431 432 /** 433 * struct ctx_hdr - Book keeping data about the crypto context 434 * @pool: Pool used to allocate crypto context 435 * @dma: Base DMA address of the cypto context 436 * @ctx_dma: Actual usable crypto context for NITROX 437 */ 438 struct ctx_hdr { 439 struct dma_pool *pool; 440 dma_addr_t dma; 441 dma_addr_t ctx_dma; 442 }; 443 444 /* 445 * struct sglist_component - SG list component format 446 * @len0: The number of bytes at [PTR0] on the remote host. 447 * @len1: The number of bytes at [PTR1] on the remote host. 448 * @len2: The number of bytes at [PTR2] on the remote host. 449 * @len3: The number of bytes at [PTR3] on the remote host. 450 * @dma0: First pointer point to buffer in remote host. 451 * @dma1: Second pointer point to buffer in remote host. 452 * @dma2: Third pointer point to buffer in remote host. 453 * @dma3: Fourth pointer point to buffer in remote host. 454 */ 455 struct nitrox_sgcomp { 456 __be16 len[4]; 457 __be64 dma[4]; 458 }; 459 460 /* 461 * strutct nitrox_sgtable - SG list information 462 * @sgmap_cnt: Number of buffers mapped 463 * @total_bytes: Total bytes in sglist. 464 * @sgcomp_len: Total sglist components length. 465 * @sgcomp_dma: DMA address of sglist component. 466 * @sg: crypto request buffer. 467 * @sgcomp: sglist component for NITROX. 468 */ 469 struct nitrox_sgtable { 470 u8 sgmap_cnt; 471 u16 total_bytes; 472 u32 sgcomp_len; 473 dma_addr_t sgcomp_dma; 474 struct scatterlist *sg; 475 struct nitrox_sgcomp *sgcomp; 476 }; 477 478 /* Response Header Length */ 479 #define ORH_HLEN 8 480 /* Completion bytes Length */ 481 #define COMP_HLEN 8 482 483 struct resp_hdr { 484 u64 *orh; 485 u64 *completion; 486 }; 487 488 typedef void (*completion_t)(void *arg, int err); 489 490 /** 491 * struct nitrox_softreq - Represents the NIROX Request. 492 * @response: response list entry 493 * @backlog: Backlog list entry 494 * @ndev: Device used to submit the request 495 * @cmdq: Command queue for submission 496 * @resp: Response headers 497 * @instr: 64B instruction 498 * @in: SG table for input 499 * @out SG table for output 500 * @tstamp: Request submitted time in jiffies 501 * @callback: callback after request completion/timeout 502 * @cb_arg: callback argument 503 */ 504 struct nitrox_softreq { 505 struct list_head response; 506 struct list_head backlog; 507 508 u32 flags; 509 gfp_t gfp; 510 atomic_t status; 511 512 struct nitrox_device *ndev; 513 struct nitrox_cmdq *cmdq; 514 515 struct nps_pkt_instr instr; 516 struct resp_hdr resp; 517 struct nitrox_sgtable in; 518 struct nitrox_sgtable out; 519 520 unsigned long tstamp; 521 522 completion_t callback; 523 void *cb_arg; 524 }; 525 526 static inline int flexi_aes_keylen(int keylen) 527 { 528 int aes_keylen; 529 530 switch (keylen) { 531 case AES_KEYSIZE_128: 532 aes_keylen = 1; 533 break; 534 case AES_KEYSIZE_192: 535 aes_keylen = 2; 536 break; 537 case AES_KEYSIZE_256: 538 aes_keylen = 3; 539 break; 540 default: 541 aes_keylen = -EINVAL; 542 break; 543 } 544 return aes_keylen; 545 } 546 547 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp) 548 { 549 size_t size; 550 551 size = sizeof(struct scatterlist) * nents; 552 size += extralen; 553 554 return kzalloc(size, gfp); 555 } 556 557 /** 558 * create_single_sg - Point SG entry to the data 559 * @sg: Destination SG list 560 * @buf: Data 561 * @buflen: Data length 562 * 563 * Returns next free entry in the destination SG list 564 **/ 565 static inline struct scatterlist *create_single_sg(struct scatterlist *sg, 566 void *buf, int buflen) 567 { 568 sg_set_buf(sg, buf, buflen); 569 sg++; 570 return sg; 571 } 572 573 /** 574 * create_multi_sg - Create multiple sg entries with buflen data length from 575 * source sglist 576 * @to_sg: Destination SG list 577 * @from_sg: Source SG list 578 * @buflen: Data length 579 * 580 * Returns next free entry in the destination SG list 581 **/ 582 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, 583 struct scatterlist *from_sg, 584 int buflen) 585 { 586 struct scatterlist *sg = to_sg; 587 unsigned int sglen; 588 589 for (; buflen && from_sg; buflen -= sglen) { 590 sglen = from_sg->length; 591 if (sglen > buflen) 592 sglen = buflen; 593 594 sg_set_buf(sg, sg_virt(from_sg), sglen); 595 from_sg = sg_next(from_sg); 596 sg++; 597 } 598 599 return sg; 600 } 601 602 static inline void set_orh_value(u64 *orh) 603 { 604 WRITE_ONCE(*orh, PENDING_SIG); 605 } 606 607 static inline void set_comp_value(u64 *comp) 608 { 609 WRITE_ONCE(*comp, PENDING_SIG); 610 } 611 612 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq, 613 int nents, int ivsize) 614 { 615 struct se_crypto_request *creq = &nkreq->creq; 616 617 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp); 618 if (!nkreq->src) 619 return -ENOMEM; 620 621 return 0; 622 } 623 624 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size) 625 { 626 memcpy(dst, src, size); 627 } 628 629 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize) 630 { 631 return (struct scatterlist *)(iv + ivsize); 632 } 633 634 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq, 635 int nents, int ivsize, 636 struct scatterlist *src, int buflen) 637 { 638 char *iv = nkreq->src; 639 struct scatterlist *sg; 640 struct se_crypto_request *creq = &nkreq->creq; 641 642 creq->src = nitrox_creq_src_sg(iv, ivsize); 643 sg = creq->src; 644 sg_init_table(sg, nents); 645 646 /* Input format: 647 * +----+----------------+ 648 * | IV | SRC sg entries | 649 * +----+----------------+ 650 */ 651 652 /* IV */ 653 sg = create_single_sg(sg, iv, ivsize); 654 /* SRC entries */ 655 create_multi_sg(sg, src, buflen); 656 } 657 658 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq, 659 int nents) 660 { 661 int extralen = ORH_HLEN + COMP_HLEN; 662 struct se_crypto_request *creq = &nkreq->creq; 663 664 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp); 665 if (!nkreq->dst) 666 return -ENOMEM; 667 668 return 0; 669 } 670 671 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq) 672 { 673 struct se_crypto_request *creq = &nkreq->creq; 674 675 creq->orh = (u64 *)(nkreq->dst); 676 set_orh_value(creq->orh); 677 } 678 679 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq) 680 { 681 struct se_crypto_request *creq = &nkreq->creq; 682 683 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN); 684 set_comp_value(creq->comp); 685 } 686 687 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst) 688 { 689 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN); 690 } 691 692 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq, 693 int nents, int ivsize, 694 struct scatterlist *dst, int buflen) 695 { 696 struct se_crypto_request *creq = &nkreq->creq; 697 struct scatterlist *sg; 698 char *iv = nkreq->src; 699 700 creq->dst = nitrox_creq_dst_sg(nkreq->dst); 701 sg = creq->dst; 702 sg_init_table(sg, nents); 703 704 /* Output format: 705 * +-----+----+----------------+-----------------+ 706 * | ORH | IV | DST sg entries | COMPLETION Bytes| 707 * +-----+----+----------------+-----------------+ 708 */ 709 710 /* ORH */ 711 sg = create_single_sg(sg, creq->orh, ORH_HLEN); 712 /* IV */ 713 sg = create_single_sg(sg, iv, ivsize); 714 /* DST entries */ 715 sg = create_multi_sg(sg, dst, buflen); 716 /* COMPLETION Bytes */ 717 create_single_sg(sg, creq->comp, COMP_HLEN); 718 } 719 720 #endif /* __NITROX_REQ_H */ 721