1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 #ifndef __CHCR_ALGO_H__ 37 #define __CHCR_ALGO_H__ 38 39 /* Crypto key context */ 40 #define KEY_CONTEXT_CTX_LEN_S 24 41 #define KEY_CONTEXT_CTX_LEN_M 0xff 42 #define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) 43 #define KEY_CONTEXT_CTX_LEN_G(x) \ 44 (((x) >> KEY_CONTEXT_CTX_LEN_S) & KEY_CONTEXT_CTX_LEN_M) 45 46 #define KEY_CONTEXT_DUAL_CK_S 12 47 #define KEY_CONTEXT_DUAL_CK_M 0x1 48 #define KEY_CONTEXT_DUAL_CK_V(x) ((x) << KEY_CONTEXT_DUAL_CK_S) 49 #define KEY_CONTEXT_DUAL_CK_G(x) \ 50 (((x) >> KEY_CONTEXT_DUAL_CK_S) & KEY_CONTEXT_DUAL_CK_M) 51 #define KEY_CONTEXT_DUAL_CK_F KEY_CONTEXT_DUAL_CK_V(1U) 52 53 #define KEY_CONTEXT_SALT_PRESENT_S 10 54 #define KEY_CONTEXT_SALT_PRESENT_M 0x1 55 #define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) 56 #define KEY_CONTEXT_SALT_PRESENT_G(x) \ 57 (((x) >> KEY_CONTEXT_SALT_PRESENT_S) & \ 58 KEY_CONTEXT_SALT_PRESENT_M) 59 #define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) 60 61 #define KEY_CONTEXT_VALID_S 0 62 #define KEY_CONTEXT_VALID_M 0x1 63 #define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) 64 #define KEY_CONTEXT_VALID_G(x) \ 65 (((x) >> KEY_CONTEXT_VALID_S) & \ 66 KEY_CONTEXT_VALID_M) 67 #define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) 68 69 #define KEY_CONTEXT_CK_SIZE_S 6 70 #define KEY_CONTEXT_CK_SIZE_M 0xf 71 #define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) 72 #define KEY_CONTEXT_CK_SIZE_G(x) \ 73 (((x) >> KEY_CONTEXT_CK_SIZE_S) & KEY_CONTEXT_CK_SIZE_M) 74 75 #define KEY_CONTEXT_MK_SIZE_S 2 76 #define KEY_CONTEXT_MK_SIZE_M 0xf 77 #define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) 78 #define KEY_CONTEXT_MK_SIZE_G(x) \ 79 (((x) >> KEY_CONTEXT_MK_SIZE_S) & KEY_CONTEXT_MK_SIZE_M) 80 81 #define KEY_CONTEXT_OPAD_PRESENT_S 11 82 #define KEY_CONTEXT_OPAD_PRESENT_M 0x1 83 #define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) 84 #define KEY_CONTEXT_OPAD_PRESENT_G(x) \ 85 (((x) >> KEY_CONTEXT_OPAD_PRESENT_S) & \ 86 KEY_CONTEXT_OPAD_PRESENT_M) 87 #define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) 88 89 #define CHCR_HASH_MAX_DIGEST_SIZE 64 90 #define CHCR_MAX_SHA_DIGEST_SIZE 64 91 92 #define IPSEC_TRUNCATED_ICV_SIZE 12 93 #define TLS_TRUNCATED_HMAC_SIZE 10 94 #define CBCMAC_DIGEST_SIZE 16 95 #define MAX_HASH_NAME 20 96 97 #define SHA1_INIT_STATE_5X4B 5 98 #define SHA256_INIT_STATE_8X4B 8 99 #define SHA512_INIT_STATE_8X8B 8 100 #define SHA1_INIT_STATE SHA1_INIT_STATE_5X4B 101 #define SHA224_INIT_STATE SHA256_INIT_STATE_8X4B 102 #define SHA256_INIT_STATE SHA256_INIT_STATE_8X4B 103 #define SHA384_INIT_STATE SHA512_INIT_STATE_8X8B 104 #define SHA512_INIT_STATE SHA512_INIT_STATE_8X8B 105 106 #define DUMMY_BYTES 16 107 108 #define IPAD_DATA 0x36363636 109 #define OPAD_DATA 0x5c5c5c5c 110 111 #define TRANSHDR_SIZE(kctx_len)\ 112 (sizeof(struct chcr_wr) +\ 113 kctx_len) 114 #define CIPHER_TRANSHDR_SIZE(kctx_len, sge_pairs) \ 115 (TRANSHDR_SIZE((kctx_len)) + (sge_pairs) +\ 116 sizeof(struct cpl_rx_phys_dsgl)) 117 #define HASH_TRANSHDR_SIZE(kctx_len)\ 118 (TRANSHDR_SIZE(kctx_len) + DUMMY_BYTES) 119 120 121 #define FILL_SEC_CPL_OP_IVINSR(id, len, ofst) \ 122 htonl( \ 123 CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | \ 124 CPL_TX_SEC_PDU_RXCHID_V((id)) | \ 125 CPL_TX_SEC_PDU_ACKFOLLOWS_V(0) | \ 126 CPL_TX_SEC_PDU_ULPTXLPBK_V(1) | \ 127 CPL_TX_SEC_PDU_CPLLEN_V((len)) | \ 128 CPL_TX_SEC_PDU_PLACEHOLDER_V(0) | \ 129 CPL_TX_SEC_PDU_IVINSRTOFST_V((ofst))) 130 131 #define FILL_SEC_CPL_CIPHERSTOP_HI(a_start, a_stop, c_start, c_stop_hi) \ 132 htonl( \ 133 CPL_TX_SEC_PDU_AADSTART_V((a_start)) | \ 134 CPL_TX_SEC_PDU_AADSTOP_V((a_stop)) | \ 135 CPL_TX_SEC_PDU_CIPHERSTART_V((c_start)) | \ 136 CPL_TX_SEC_PDU_CIPHERSTOP_HI_V((c_stop_hi))) 137 138 #define FILL_SEC_CPL_AUTHINSERT(c_stop_lo, a_start, a_stop, a_inst) \ 139 htonl( \ 140 CPL_TX_SEC_PDU_CIPHERSTOP_LO_V((c_stop_lo)) | \ 141 CPL_TX_SEC_PDU_AUTHSTART_V((a_start)) | \ 142 CPL_TX_SEC_PDU_AUTHSTOP_V((a_stop)) | \ 143 CPL_TX_SEC_PDU_AUTHINSERT_V((a_inst))) 144 145 #define FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size) \ 146 htonl( \ 147 SCMD_SEQ_NO_CTRL_V(0) | \ 148 SCMD_STATUS_PRESENT_V(0) | \ 149 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | \ 150 SCMD_ENC_DEC_CTRL_V((ctrl)) | \ 151 SCMD_CIPH_AUTH_SEQ_CTRL_V((seq)) | \ 152 SCMD_CIPH_MODE_V((cmode)) | \ 153 SCMD_AUTH_MODE_V((amode)) | \ 154 SCMD_HMAC_CTRL_V((opad)) | \ 155 SCMD_IV_SIZE_V((size)) | \ 156 SCMD_NUM_IVS_V(0)) 157 158 #define FILL_SEC_CPL_IVGEN_HDRLEN(last, more, ctx_in, mac, ivdrop, len) htonl( \ 159 SCMD_ENB_DBGID_V(0) | \ 160 SCMD_IV_GEN_CTRL_V(0) | \ 161 SCMD_LAST_FRAG_V((last)) | \ 162 SCMD_MORE_FRAGS_V((more)) | \ 163 SCMD_TLS_COMPPDU_V(0) | \ 164 SCMD_KEY_CTX_INLINE_V((ctx_in)) | \ 165 SCMD_TLS_FRAG_ENABLE_V(0) | \ 166 SCMD_MAC_ONLY_V((mac)) | \ 167 SCMD_AADIVDROP_V((ivdrop)) | \ 168 SCMD_HDR_LEN_V((len))) 169 170 #define FILL_KEY_CTX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \ 171 htonl(KEY_CONTEXT_VALID_V(1) | \ 172 KEY_CONTEXT_CK_SIZE_V((ck_size)) | \ 173 KEY_CONTEXT_MK_SIZE_V(mk_size) | \ 174 KEY_CONTEXT_DUAL_CK_V((d_ck)) | \ 175 KEY_CONTEXT_OPAD_PRESENT_V((opad)) | \ 176 KEY_CONTEXT_SALT_PRESENT_V(1) | \ 177 KEY_CONTEXT_CTX_LEN_V((ctx_len))) 178 179 #define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \ 180 htonl( \ 181 FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \ 182 FW_CRYPTO_LOOKASIDE_WR) | \ 183 FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \ 184 FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \ 185 FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ 186 FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) 187 188 #define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \ 189 htonl( \ 190 FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ 191 FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ 192 FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ 193 FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \ 194 FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) 195 196 #define FILL_ULPTX_CMD_DEST(cid, qid) \ 197 htonl(ULPTX_CMD_V(ULP_TX_PKT) | \ 198 ULP_TXPKT_DEST_V(0) | \ 199 ULP_TXPKT_DATAMODIFY_V(0) | \ 200 ULP_TXPKT_CHANNELID_V((cid)) | \ 201 ULP_TXPKT_RO_V(1) | \ 202 ULP_TXPKT_FID_V(qid)) 203 204 #define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\ 205 _bs == SHA1_DIGEST_SIZE ? 12 : 0; }) 206 207 #define FILL_PLD_SIZE_HASH_SIZE(payload_sgl_len, sgl_lengths, total_frags) \ 208 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(payload_sgl_len ? \ 209 sgl_lengths[total_frags] : 0) |\ 210 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(0)) 211 212 #define FILL_LEN_PKD(calc_tx_flits_ofld, skb) \ 213 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP((\ 214 calc_tx_flits_ofld(skb) * 8), 16))) 215 216 #define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\ 217 ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1)) 218 219 #define MAX_NK 8 220 #define CRYPTO_MAX_IMM_TX_PKT_LEN 256 221 #define MAX_WR_SIZE 512 222 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) 223 #define MAX_DSGL_ENT 32 224 #define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2) 225 #define MIN_CIPHER_SG 1 /* IV */ 226 #define MIN_AUTH_SG 2 /*IV + AAD*/ 227 #define MIN_GCM_SG 2 /* IV + AAD*/ 228 #define MIN_DIGEST_SG 1 /*Partial Buffer*/ 229 #define MIN_CCM_SG 3 /*IV+AAD+B0*/ 230 #define SPACE_LEFT(len) \ 231 ((MAX_WR_SIZE - WR_MIN_LEN - (len))) 232 233 unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 234 48, 64, 72, 88, 235 96, 112, 120, 136, 236 144, 160, 168, 184, 237 192}; 238 unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, 239 112, 112, 128, 128, 144, 144, 160, 160, 240 192, 192, 208, 208, 224, 224, 240, 240, 241 272, 272, 288, 288, 304, 304, 320, 320}; 242 243 struct algo_param { 244 unsigned int auth_mode; 245 unsigned int mk_size; 246 unsigned int result_size; 247 }; 248 249 struct hash_wr_param { 250 unsigned int opad_needed; 251 unsigned int more; 252 unsigned int last; 253 struct algo_param alg_prm; 254 unsigned int sg_len; 255 unsigned int bfr_len; 256 u64 scmd1; 257 }; 258 259 struct cipher_wr_param { 260 struct ablkcipher_request *req; 261 struct scatterlist *srcsg; 262 char *iv; 263 int bytes; 264 short int snent; 265 unsigned short qid; 266 }; 267 enum { 268 AES_KEYLENGTH_128BIT = 128, 269 AES_KEYLENGTH_192BIT = 192, 270 AES_KEYLENGTH_256BIT = 256 271 }; 272 273 enum { 274 KEYLENGTH_3BYTES = 3, 275 KEYLENGTH_4BYTES = 4, 276 KEYLENGTH_6BYTES = 6, 277 KEYLENGTH_8BYTES = 8 278 }; 279 280 enum { 281 NUMBER_OF_ROUNDS_10 = 10, 282 NUMBER_OF_ROUNDS_12 = 12, 283 NUMBER_OF_ROUNDS_14 = 14, 284 }; 285 286 /* 287 * CCM defines values of 4, 6, 8, 10, 12, 14, and 16 octets, 288 * where they indicate the size of the integrity check value (ICV) 289 */ 290 enum { 291 ICV_4 = 4, 292 ICV_6 = 6, 293 ICV_8 = 8, 294 ICV_10 = 10, 295 ICV_12 = 12, 296 ICV_13 = 13, 297 ICV_14 = 14, 298 ICV_15 = 15, 299 ICV_16 = 16 300 }; 301 302 struct hash_op_params { 303 unsigned char mk_size; 304 unsigned char pad_align; 305 unsigned char auth_mode; 306 char hash_name[MAX_HASH_NAME]; 307 unsigned short block_size; 308 unsigned short word_size; 309 unsigned short ipad_size; 310 }; 311 312 struct phys_sge_pairs { 313 __be16 len[8]; 314 __be64 addr[8]; 315 }; 316 317 struct phys_sge_parm { 318 unsigned int nents; 319 unsigned int obsize; 320 unsigned short qid; 321 }; 322 323 struct crypto_result { 324 struct completion completion; 325 int err; 326 }; 327 328 static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { 329 SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 330 }; 331 332 static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = { 333 SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, 334 SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, 335 }; 336 337 static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = { 338 SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, 339 SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, 340 }; 341 342 static const u64 sha384_init[SHA512_DIGEST_SIZE / 8] = { 343 SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3, 344 SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7, 345 }; 346 347 static const u64 sha512_init[SHA512_DIGEST_SIZE / 8] = { 348 SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3, 349 SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7, 350 }; 351 352 static inline void copy_hash_init_values(char *key, int digestsize) 353 { 354 u8 i; 355 __be32 *dkey = (__be32 *)key; 356 u64 *ldkey = (u64 *)key; 357 __be64 *sha384 = (__be64 *)sha384_init; 358 __be64 *sha512 = (__be64 *)sha512_init; 359 360 switch (digestsize) { 361 case SHA1_DIGEST_SIZE: 362 for (i = 0; i < SHA1_INIT_STATE; i++) 363 dkey[i] = cpu_to_be32(sha1_init[i]); 364 break; 365 case SHA224_DIGEST_SIZE: 366 for (i = 0; i < SHA224_INIT_STATE; i++) 367 dkey[i] = cpu_to_be32(sha224_init[i]); 368 break; 369 case SHA256_DIGEST_SIZE: 370 for (i = 0; i < SHA256_INIT_STATE; i++) 371 dkey[i] = cpu_to_be32(sha256_init[i]); 372 break; 373 case SHA384_DIGEST_SIZE: 374 for (i = 0; i < SHA384_INIT_STATE; i++) 375 ldkey[i] = be64_to_cpu(sha384[i]); 376 break; 377 case SHA512_DIGEST_SIZE: 378 for (i = 0; i < SHA512_INIT_STATE; i++) 379 ldkey[i] = be64_to_cpu(sha512[i]); 380 break; 381 } 382 } 383 384 static const u8 sgl_lengths[20] = { 385 0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15 386 }; 387 388 /* Number of len fields(8) * size of one addr field */ 389 #define PHYSDSGL_MAX_LEN_SIZE 16 390 391 static inline u16 get_space_for_phys_dsgl(unsigned int sgl_entr) 392 { 393 /* len field size + addr field size */ 394 return ((sgl_entr >> 3) + ((sgl_entr % 8) ? 395 1 : 0)) * PHYSDSGL_MAX_LEN_SIZE + 396 (sgl_entr << 3) + ((sgl_entr % 2 ? 1 : 0) << 3); 397 } 398 399 /* The AES s-transform matrix (s-box). */ 400 static const u8 aes_sbox[256] = { 401 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 402 171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 403 156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 404 229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154, 7, 405 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110, 90, 406 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237, 32, 407 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239, 170, 408 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168, 81, 409 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 410 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 411 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 412 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92, 194, 413 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213, 78, 414 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46, 28, 166, 415 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62, 181, 102, 416 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158, 225, 248, 417 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223, 418 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 419 187, 22 420 }; 421 422 static inline u32 aes_ks_subword(const u32 w) 423 { 424 u8 bytes[4]; 425 426 *(u32 *)(&bytes[0]) = w; 427 bytes[0] = aes_sbox[bytes[0]]; 428 bytes[1] = aes_sbox[bytes[1]]; 429 bytes[2] = aes_sbox[bytes[2]]; 430 bytes[3] = aes_sbox[bytes[3]]; 431 return *(u32 *)(&bytes[0]); 432 } 433 434 static u32 round_constant[11] = { 435 0x01000000, 0x02000000, 0x04000000, 0x08000000, 436 0x10000000, 0x20000000, 0x40000000, 0x80000000, 437 0x1B000000, 0x36000000, 0x6C000000 438 }; 439 440 #endif /* __CHCR_ALGO_H__ */ 441