1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 CTR_RFC3686_NONCE_SIZE + \ 64 SHA512_DIGEST_SIZE * 2) 65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 66 #define CAAM_MAX_IV_LENGTH 16 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 5) 73 74 /* length of descriptors text */ 75 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 76 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) 77 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) 78 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) 79 80 /* Note: Nonce is counted in enckeylen */ 81 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) 82 83 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) 84 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) 85 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) 86 87 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) 88 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) 89 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) 90 91 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) 92 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 93 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 94 95 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) 96 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) 97 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) 98 99 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 100 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 101 20 * CAAM_CMD_SZ) 102 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 103 15 * CAAM_CMD_SZ) 104 105 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 106 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 107 108 #ifdef DEBUG 109 /* for print_hex_dumps with line references */ 110 #define debug(format, arg...) printk(format, arg) 111 #else 112 #define debug(format, arg...) 113 #endif 114 static struct list_head alg_list; 115 116 struct caam_alg_entry { 117 int class1_alg_type; 118 int class2_alg_type; 119 int alg_op; 120 bool rfc3686; 121 bool geniv; 122 }; 123 124 struct caam_aead_alg { 125 struct aead_alg aead; 126 struct caam_alg_entry caam; 127 bool registered; 128 }; 129 130 /* Set DK bit in class 1 operation if shared */ 131 static inline void append_dec_op1(u32 *desc, u32 type) 132 { 133 u32 *jump_cmd, *uncond_jump_cmd; 134 135 /* DK bit is valid only for AES */ 136 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 137 append_operation(desc, type | OP_ALG_AS_INITFINAL | 138 OP_ALG_DECRYPT); 139 return; 140 } 141 142 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 143 append_operation(desc, type | OP_ALG_AS_INITFINAL | 144 OP_ALG_DECRYPT); 145 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 146 set_jump_tgt_here(desc, jump_cmd); 147 append_operation(desc, type | OP_ALG_AS_INITFINAL | 148 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 149 set_jump_tgt_here(desc, uncond_jump_cmd); 150 } 151 152 /* 153 * For aead functions, read payload and write payload, 154 * both of which are specified in req->src and req->dst 155 */ 156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 157 { 158 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 159 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 160 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 161 } 162 163 /* 164 * For ablkcipher encrypt and decrypt, read from req->src and 165 * write to req->dst 166 */ 167 static inline void ablkcipher_append_src_dst(u32 *desc) 168 { 169 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 170 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 171 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 172 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 173 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 174 } 175 176 /* 177 * per-session context 178 */ 179 struct caam_ctx { 180 struct device *jrdev; 181 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 182 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 183 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 184 dma_addr_t sh_desc_enc_dma; 185 dma_addr_t sh_desc_dec_dma; 186 dma_addr_t sh_desc_givenc_dma; 187 u32 class1_alg_type; 188 u32 class2_alg_type; 189 u32 alg_op; 190 u8 key[CAAM_MAX_KEY_SIZE]; 191 dma_addr_t key_dma; 192 unsigned int enckeylen; 193 unsigned int split_key_len; 194 unsigned int split_key_pad_len; 195 unsigned int authsize; 196 }; 197 198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 199 int keys_fit_inline, bool is_rfc3686) 200 { 201 u32 *nonce; 202 unsigned int enckeylen = ctx->enckeylen; 203 204 /* 205 * RFC3686 specific: 206 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} 207 * | enckeylen = encryption key size + nonce size 208 */ 209 if (is_rfc3686) 210 enckeylen -= CTR_RFC3686_NONCE_SIZE; 211 212 if (keys_fit_inline) { 213 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 214 ctx->split_key_len, CLASS_2 | 215 KEY_DEST_MDHA_SPLIT | KEY_ENC); 216 append_key_as_imm(desc, (void *)ctx->key + 217 ctx->split_key_pad_len, enckeylen, 218 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 219 } else { 220 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 221 KEY_DEST_MDHA_SPLIT | KEY_ENC); 222 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 223 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 224 } 225 226 /* Load Counter into CONTEXT1 reg */ 227 if (is_rfc3686) { 228 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + 229 enckeylen); 230 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 231 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 232 append_move(desc, 233 MOVE_SRC_OUTFIFO | 234 MOVE_DEST_CLASS1CTX | 235 (16 << MOVE_OFFSET_SHIFT) | 236 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 237 } 238 } 239 240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 241 int keys_fit_inline, bool is_rfc3686) 242 { 243 u32 *key_jump_cmd; 244 245 /* Note: Context registers are saved. */ 246 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 247 248 /* Skip if already shared */ 249 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 250 JUMP_COND_SHRD); 251 252 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 253 254 set_jump_tgt_here(desc, key_jump_cmd); 255 } 256 257 static int aead_null_set_sh_desc(struct crypto_aead *aead) 258 { 259 struct caam_ctx *ctx = crypto_aead_ctx(aead); 260 struct device *jrdev = ctx->jrdev; 261 bool keys_fit_inline = false; 262 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; 263 u32 *desc; 264 265 /* 266 * Job Descriptor and Shared Descriptors 267 * must all fit into the 64-word Descriptor h/w Buffer 268 */ 269 if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + 270 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 271 keys_fit_inline = true; 272 273 /* aead_encrypt shared descriptor */ 274 desc = ctx->sh_desc_enc; 275 276 init_sh_desc(desc, HDR_SHARE_SERIAL); 277 278 /* Skip if already shared */ 279 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 280 JUMP_COND_SHRD); 281 if (keys_fit_inline) 282 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 283 ctx->split_key_len, CLASS_2 | 284 KEY_DEST_MDHA_SPLIT | KEY_ENC); 285 else 286 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 287 KEY_DEST_MDHA_SPLIT | KEY_ENC); 288 set_jump_tgt_here(desc, key_jump_cmd); 289 290 /* assoclen + cryptlen = seqinlen */ 291 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 292 293 /* Prepare to read and write cryptlen + assoclen bytes */ 294 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 295 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 296 297 /* 298 * MOVE_LEN opcode is not available in all SEC HW revisions, 299 * thus need to do some magic, i.e. self-patch the descriptor 300 * buffer. 301 */ 302 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 303 MOVE_DEST_MATH3 | 304 (0x6 << MOVE_LEN_SHIFT)); 305 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 306 MOVE_DEST_DESCBUF | 307 MOVE_WAITCOMP | 308 (0x8 << MOVE_LEN_SHIFT)); 309 310 /* Class 2 operation */ 311 append_operation(desc, ctx->class2_alg_type | 312 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 313 314 /* Read and write cryptlen bytes */ 315 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 316 317 set_move_tgt_here(desc, read_move_cmd); 318 set_move_tgt_here(desc, write_move_cmd); 319 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 320 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 321 MOVE_AUX_LS); 322 323 /* Write ICV */ 324 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 325 LDST_SRCDST_BYTE_CONTEXT); 326 327 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 328 desc_bytes(desc), 329 DMA_TO_DEVICE); 330 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 331 dev_err(jrdev, "unable to map shared descriptor\n"); 332 return -ENOMEM; 333 } 334 #ifdef DEBUG 335 print_hex_dump(KERN_ERR, 336 "aead null enc shdesc@"__stringify(__LINE__)": ", 337 DUMP_PREFIX_ADDRESS, 16, 4, desc, 338 desc_bytes(desc), 1); 339 #endif 340 341 /* 342 * Job Descriptor and Shared Descriptors 343 * must all fit into the 64-word Descriptor h/w Buffer 344 */ 345 keys_fit_inline = false; 346 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + 347 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 348 keys_fit_inline = true; 349 350 desc = ctx->sh_desc_dec; 351 352 /* aead_decrypt shared descriptor */ 353 init_sh_desc(desc, HDR_SHARE_SERIAL); 354 355 /* Skip if already shared */ 356 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 357 JUMP_COND_SHRD); 358 if (keys_fit_inline) 359 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 360 ctx->split_key_len, CLASS_2 | 361 KEY_DEST_MDHA_SPLIT | KEY_ENC); 362 else 363 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 364 KEY_DEST_MDHA_SPLIT | KEY_ENC); 365 set_jump_tgt_here(desc, key_jump_cmd); 366 367 /* Class 2 operation */ 368 append_operation(desc, ctx->class2_alg_type | 369 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 370 371 /* assoclen + cryptlen = seqoutlen */ 372 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 373 374 /* Prepare to read and write cryptlen + assoclen bytes */ 375 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 376 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 377 378 /* 379 * MOVE_LEN opcode is not available in all SEC HW revisions, 380 * thus need to do some magic, i.e. self-patch the descriptor 381 * buffer. 382 */ 383 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 384 MOVE_DEST_MATH2 | 385 (0x6 << MOVE_LEN_SHIFT)); 386 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 387 MOVE_DEST_DESCBUF | 388 MOVE_WAITCOMP | 389 (0x8 << MOVE_LEN_SHIFT)); 390 391 /* Read and write cryptlen bytes */ 392 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 393 394 /* 395 * Insert a NOP here, since we need at least 4 instructions between 396 * code patching the descriptor buffer and the location being patched. 397 */ 398 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 399 set_jump_tgt_here(desc, jump_cmd); 400 401 set_move_tgt_here(desc, read_move_cmd); 402 set_move_tgt_here(desc, write_move_cmd); 403 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 404 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 405 MOVE_AUX_LS); 406 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 407 408 /* Load ICV */ 409 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 410 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 411 412 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 413 desc_bytes(desc), 414 DMA_TO_DEVICE); 415 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 416 dev_err(jrdev, "unable to map shared descriptor\n"); 417 return -ENOMEM; 418 } 419 #ifdef DEBUG 420 print_hex_dump(KERN_ERR, 421 "aead null dec shdesc@"__stringify(__LINE__)": ", 422 DUMP_PREFIX_ADDRESS, 16, 4, desc, 423 desc_bytes(desc), 1); 424 #endif 425 426 return 0; 427 } 428 429 static int aead_set_sh_desc(struct crypto_aead *aead) 430 { 431 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 432 struct caam_aead_alg, aead); 433 unsigned int ivsize = crypto_aead_ivsize(aead); 434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 435 struct device *jrdev = ctx->jrdev; 436 bool keys_fit_inline; 437 u32 geniv, moveiv; 438 u32 ctx1_iv_off = 0; 439 u32 *desc; 440 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 441 OP_ALG_AAI_CTR_MOD128); 442 const bool is_rfc3686 = alg->caam.rfc3686; 443 444 /* NULL encryption / decryption */ 445 if (!ctx->enckeylen) 446 return aead_null_set_sh_desc(aead); 447 448 /* 449 * AES-CTR needs to load IV in CONTEXT1 reg 450 * at an offset of 128bits (16bytes) 451 * CONTEXT1[255:128] = IV 452 */ 453 if (ctr_mode) 454 ctx1_iv_off = 16; 455 456 /* 457 * RFC3686 specific: 458 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 459 */ 460 if (is_rfc3686) 461 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 462 463 if (alg->caam.geniv) 464 goto skip_enc; 465 466 /* 467 * Job Descriptor and Shared Descriptors 468 * must all fit into the 64-word Descriptor h/w Buffer 469 */ 470 keys_fit_inline = false; 471 if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 472 ctx->split_key_pad_len + ctx->enckeylen + 473 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 474 CAAM_DESC_BYTES_MAX) 475 keys_fit_inline = true; 476 477 /* aead_encrypt shared descriptor */ 478 desc = ctx->sh_desc_enc; 479 480 /* Note: Context registers are saved. */ 481 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 482 483 /* Class 2 operation */ 484 append_operation(desc, ctx->class2_alg_type | 485 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 486 487 /* Read and write assoclen bytes */ 488 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 489 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 490 491 /* Skip assoc data */ 492 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 493 494 /* read assoc before reading payload */ 495 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 496 FIFOLDST_VLF); 497 498 /* Load Counter into CONTEXT1 reg */ 499 if (is_rfc3686) 500 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 501 LDST_CLASS_1_CCB | 502 LDST_SRCDST_BYTE_CONTEXT | 503 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 504 LDST_OFFSET_SHIFT)); 505 506 /* Class 1 operation */ 507 append_operation(desc, ctx->class1_alg_type | 508 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 509 510 /* Read and write cryptlen bytes */ 511 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 512 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 513 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 514 515 /* Write ICV */ 516 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 517 LDST_SRCDST_BYTE_CONTEXT); 518 519 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 520 desc_bytes(desc), 521 DMA_TO_DEVICE); 522 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 523 dev_err(jrdev, "unable to map shared descriptor\n"); 524 return -ENOMEM; 525 } 526 #ifdef DEBUG 527 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 528 DUMP_PREFIX_ADDRESS, 16, 4, desc, 529 desc_bytes(desc), 1); 530 #endif 531 532 skip_enc: 533 /* 534 * Job Descriptor and Shared Descriptors 535 * must all fit into the 64-word Descriptor h/w Buffer 536 */ 537 keys_fit_inline = false; 538 if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + 539 ctx->split_key_pad_len + ctx->enckeylen + 540 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 541 CAAM_DESC_BYTES_MAX) 542 keys_fit_inline = true; 543 544 /* aead_decrypt shared descriptor */ 545 desc = ctx->sh_desc_dec; 546 547 /* Note: Context registers are saved. */ 548 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 549 550 /* Class 2 operation */ 551 append_operation(desc, ctx->class2_alg_type | 552 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 553 554 /* Read and write assoclen bytes */ 555 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 556 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 557 558 /* Skip assoc data */ 559 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 560 561 /* read assoc before reading payload */ 562 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 563 KEY_VLF); 564 565 /* Load Counter into CONTEXT1 reg */ 566 if (is_rfc3686) 567 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 568 LDST_CLASS_1_CCB | 569 LDST_SRCDST_BYTE_CONTEXT | 570 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 571 LDST_OFFSET_SHIFT)); 572 573 /* Choose operation */ 574 if (ctr_mode) 575 append_operation(desc, ctx->class1_alg_type | 576 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 577 else 578 append_dec_op1(desc, ctx->class1_alg_type); 579 580 /* Read and write cryptlen bytes */ 581 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 582 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 583 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 584 585 /* Load ICV */ 586 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 587 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 588 589 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 590 desc_bytes(desc), 591 DMA_TO_DEVICE); 592 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 593 dev_err(jrdev, "unable to map shared descriptor\n"); 594 return -ENOMEM; 595 } 596 #ifdef DEBUG 597 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 598 DUMP_PREFIX_ADDRESS, 16, 4, desc, 599 desc_bytes(desc), 1); 600 #endif 601 602 if (!alg->caam.geniv) 603 goto skip_givenc; 604 605 /* 606 * Job Descriptor and Shared Descriptors 607 * must all fit into the 64-word Descriptor h/w Buffer 608 */ 609 keys_fit_inline = false; 610 if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 611 ctx->split_key_pad_len + ctx->enckeylen + 612 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 613 CAAM_DESC_BYTES_MAX) 614 keys_fit_inline = true; 615 616 /* aead_givencrypt shared descriptor */ 617 desc = ctx->sh_desc_givenc; 618 619 /* Note: Context registers are saved. */ 620 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 621 622 if (is_rfc3686) 623 goto copy_iv; 624 625 /* Generate IV */ 626 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 627 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 628 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 629 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 630 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 631 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 632 append_move(desc, MOVE_WAITCOMP | 633 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 634 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 635 (ivsize << MOVE_LEN_SHIFT)); 636 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 637 638 copy_iv: 639 /* Copy IV to class 1 context */ 640 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 641 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 642 (ivsize << MOVE_LEN_SHIFT)); 643 644 /* Return to encryption */ 645 append_operation(desc, ctx->class2_alg_type | 646 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 647 648 /* ivsize + cryptlen = seqoutlen - authsize */ 649 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 650 651 /* Read and write assoclen bytes */ 652 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 653 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 654 655 /* Skip assoc data */ 656 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 657 658 /* read assoc before reading payload */ 659 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 660 KEY_VLF); 661 662 /* Copy iv from outfifo to class 2 fifo */ 663 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 664 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 665 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 666 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 667 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 668 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 669 670 /* Load Counter into CONTEXT1 reg */ 671 if (is_rfc3686) 672 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 673 LDST_CLASS_1_CCB | 674 LDST_SRCDST_BYTE_CONTEXT | 675 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 676 LDST_OFFSET_SHIFT)); 677 678 /* Class 1 operation */ 679 append_operation(desc, ctx->class1_alg_type | 680 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 681 682 /* Will write ivsize + cryptlen */ 683 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 684 685 /* Not need to reload iv */ 686 append_seq_fifo_load(desc, ivsize, 687 FIFOLD_CLASS_SKIP); 688 689 /* Will read cryptlen */ 690 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 691 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 692 693 /* Write ICV */ 694 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 695 LDST_SRCDST_BYTE_CONTEXT); 696 697 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 698 desc_bytes(desc), 699 DMA_TO_DEVICE); 700 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 701 dev_err(jrdev, "unable to map shared descriptor\n"); 702 return -ENOMEM; 703 } 704 #ifdef DEBUG 705 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 706 DUMP_PREFIX_ADDRESS, 16, 4, desc, 707 desc_bytes(desc), 1); 708 #endif 709 710 skip_givenc: 711 return 0; 712 } 713 714 static int aead_setauthsize(struct crypto_aead *authenc, 715 unsigned int authsize) 716 { 717 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 718 719 ctx->authsize = authsize; 720 aead_set_sh_desc(authenc); 721 722 return 0; 723 } 724 725 static int gcm_set_sh_desc(struct crypto_aead *aead) 726 { 727 struct caam_ctx *ctx = crypto_aead_ctx(aead); 728 struct device *jrdev = ctx->jrdev; 729 bool keys_fit_inline = false; 730 u32 *key_jump_cmd, *zero_payload_jump_cmd, 731 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; 732 u32 *desc; 733 734 if (!ctx->enckeylen || !ctx->authsize) 735 return 0; 736 737 /* 738 * AES GCM encrypt shared descriptor 739 * Job Descriptor and Shared Descriptor 740 * must fit into the 64-word Descriptor h/w Buffer 741 */ 742 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + 743 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 744 keys_fit_inline = true; 745 746 desc = ctx->sh_desc_enc; 747 748 init_sh_desc(desc, HDR_SHARE_SERIAL); 749 750 /* skip key loading if they are loaded due to sharing */ 751 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 752 JUMP_COND_SHRD | JUMP_COND_SELF); 753 if (keys_fit_inline) 754 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 755 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 756 else 757 append_key(desc, ctx->key_dma, ctx->enckeylen, 758 CLASS_1 | KEY_DEST_CLASS_REG); 759 set_jump_tgt_here(desc, key_jump_cmd); 760 761 /* class 1 operation */ 762 append_operation(desc, ctx->class1_alg_type | 763 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 764 765 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 766 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 767 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 768 JUMP_COND_MATH_Z); 769 770 /* if assoclen is ZERO, skip reading the assoc data */ 771 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 772 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 773 JUMP_COND_MATH_Z); 774 775 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 776 777 /* skip assoc data */ 778 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 779 780 /* cryptlen = seqinlen - assoclen */ 781 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 782 783 /* if cryptlen is ZERO jump to zero-payload commands */ 784 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 785 JUMP_COND_MATH_Z); 786 787 /* read assoc data */ 788 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 789 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 790 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 791 792 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 793 794 /* write encrypted data */ 795 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 796 797 /* read payload data */ 798 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 799 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 800 801 /* jump the zero-payload commands */ 802 append_jump(desc, JUMP_TEST_ALL | 2); 803 804 /* zero-payload commands */ 805 set_jump_tgt_here(desc, zero_payload_jump_cmd); 806 807 /* read assoc data */ 808 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 809 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 810 811 /* There is no input data */ 812 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 813 814 /* write ICV */ 815 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 816 LDST_SRCDST_BYTE_CONTEXT); 817 818 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 819 desc_bytes(desc), 820 DMA_TO_DEVICE); 821 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 822 dev_err(jrdev, "unable to map shared descriptor\n"); 823 return -ENOMEM; 824 } 825 #ifdef DEBUG 826 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", 827 DUMP_PREFIX_ADDRESS, 16, 4, desc, 828 desc_bytes(desc), 1); 829 #endif 830 831 /* 832 * Job Descriptor and Shared Descriptors 833 * must all fit into the 64-word Descriptor h/w Buffer 834 */ 835 keys_fit_inline = false; 836 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + 837 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 838 keys_fit_inline = true; 839 840 desc = ctx->sh_desc_dec; 841 842 init_sh_desc(desc, HDR_SHARE_SERIAL); 843 844 /* skip key loading if they are loaded due to sharing */ 845 key_jump_cmd = append_jump(desc, JUMP_JSL | 846 JUMP_TEST_ALL | JUMP_COND_SHRD | 847 JUMP_COND_SELF); 848 if (keys_fit_inline) 849 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 850 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 851 else 852 append_key(desc, ctx->key_dma, ctx->enckeylen, 853 CLASS_1 | KEY_DEST_CLASS_REG); 854 set_jump_tgt_here(desc, key_jump_cmd); 855 856 /* class 1 operation */ 857 append_operation(desc, ctx->class1_alg_type | 858 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 859 860 /* if assoclen is ZERO, skip reading the assoc data */ 861 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 862 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 863 JUMP_COND_MATH_Z); 864 865 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 866 867 /* skip assoc data */ 868 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 869 870 /* read assoc data */ 871 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 872 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 873 874 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 875 876 /* cryptlen = seqoutlen - assoclen */ 877 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 878 879 /* jump to zero-payload command if cryptlen is zero */ 880 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 881 JUMP_COND_MATH_Z); 882 883 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 884 885 /* store encrypted data */ 886 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 887 888 /* read payload data */ 889 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 890 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 891 892 /* zero-payload command */ 893 set_jump_tgt_here(desc, zero_payload_jump_cmd); 894 895 /* read ICV */ 896 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 897 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 898 899 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 900 desc_bytes(desc), 901 DMA_TO_DEVICE); 902 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 903 dev_err(jrdev, "unable to map shared descriptor\n"); 904 return -ENOMEM; 905 } 906 #ifdef DEBUG 907 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", 908 DUMP_PREFIX_ADDRESS, 16, 4, desc, 909 desc_bytes(desc), 1); 910 #endif 911 912 return 0; 913 } 914 915 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 916 { 917 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 918 919 ctx->authsize = authsize; 920 gcm_set_sh_desc(authenc); 921 922 return 0; 923 } 924 925 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 926 { 927 struct caam_ctx *ctx = crypto_aead_ctx(aead); 928 struct device *jrdev = ctx->jrdev; 929 bool keys_fit_inline = false; 930 u32 *key_jump_cmd; 931 u32 *desc; 932 933 if (!ctx->enckeylen || !ctx->authsize) 934 return 0; 935 936 /* 937 * RFC4106 encrypt shared descriptor 938 * Job Descriptor and Shared Descriptor 939 * must fit into the 64-word Descriptor h/w Buffer 940 */ 941 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + 942 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 943 keys_fit_inline = true; 944 945 desc = ctx->sh_desc_enc; 946 947 init_sh_desc(desc, HDR_SHARE_SERIAL); 948 949 /* Skip key loading if it is loaded due to sharing */ 950 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 951 JUMP_COND_SHRD); 952 if (keys_fit_inline) 953 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 954 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 955 else 956 append_key(desc, ctx->key_dma, ctx->enckeylen, 957 CLASS_1 | KEY_DEST_CLASS_REG); 958 set_jump_tgt_here(desc, key_jump_cmd); 959 960 /* Class 1 operation */ 961 append_operation(desc, ctx->class1_alg_type | 962 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 963 964 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 965 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 966 967 /* Read assoc data */ 968 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 969 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 970 971 /* Skip IV */ 972 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 973 974 /* Will read cryptlen bytes */ 975 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 976 977 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 978 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 979 980 /* Skip assoc data */ 981 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 982 983 /* cryptlen = seqoutlen - assoclen */ 984 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 985 986 /* Write encrypted data */ 987 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 988 989 /* Read payload data */ 990 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 991 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 992 993 /* Write ICV */ 994 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 995 LDST_SRCDST_BYTE_CONTEXT); 996 997 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 998 desc_bytes(desc), 999 DMA_TO_DEVICE); 1000 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1001 dev_err(jrdev, "unable to map shared descriptor\n"); 1002 return -ENOMEM; 1003 } 1004 #ifdef DEBUG 1005 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", 1006 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1007 desc_bytes(desc), 1); 1008 #endif 1009 1010 /* 1011 * Job Descriptor and Shared Descriptors 1012 * must all fit into the 64-word Descriptor h/w Buffer 1013 */ 1014 keys_fit_inline = false; 1015 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + 1016 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1017 keys_fit_inline = true; 1018 1019 desc = ctx->sh_desc_dec; 1020 1021 init_sh_desc(desc, HDR_SHARE_SERIAL); 1022 1023 /* Skip key loading if it is loaded due to sharing */ 1024 key_jump_cmd = append_jump(desc, JUMP_JSL | 1025 JUMP_TEST_ALL | JUMP_COND_SHRD); 1026 if (keys_fit_inline) 1027 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1028 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1029 else 1030 append_key(desc, ctx->key_dma, ctx->enckeylen, 1031 CLASS_1 | KEY_DEST_CLASS_REG); 1032 set_jump_tgt_here(desc, key_jump_cmd); 1033 1034 /* Class 1 operation */ 1035 append_operation(desc, ctx->class1_alg_type | 1036 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1037 1038 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 1039 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1040 1041 /* Read assoc data */ 1042 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1043 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1044 1045 /* Skip IV */ 1046 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1047 1048 /* Will read cryptlen bytes */ 1049 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 1050 1051 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 1052 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 1053 1054 /* Skip assoc data */ 1055 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1056 1057 /* Will write cryptlen bytes */ 1058 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1059 1060 /* Store payload data */ 1061 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1062 1063 /* Read encrypted data */ 1064 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1065 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1066 1067 /* Read ICV */ 1068 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1069 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1070 1071 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1072 desc_bytes(desc), 1073 DMA_TO_DEVICE); 1074 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1075 dev_err(jrdev, "unable to map shared descriptor\n"); 1076 return -ENOMEM; 1077 } 1078 #ifdef DEBUG 1079 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", 1080 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1081 desc_bytes(desc), 1); 1082 #endif 1083 1084 return 0; 1085 } 1086 1087 static int rfc4106_setauthsize(struct crypto_aead *authenc, 1088 unsigned int authsize) 1089 { 1090 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1091 1092 ctx->authsize = authsize; 1093 rfc4106_set_sh_desc(authenc); 1094 1095 return 0; 1096 } 1097 1098 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 1099 { 1100 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1101 struct device *jrdev = ctx->jrdev; 1102 bool keys_fit_inline = false; 1103 u32 *key_jump_cmd; 1104 u32 *read_move_cmd, *write_move_cmd; 1105 u32 *desc; 1106 1107 if (!ctx->enckeylen || !ctx->authsize) 1108 return 0; 1109 1110 /* 1111 * RFC4543 encrypt shared descriptor 1112 * Job Descriptor and Shared Descriptor 1113 * must fit into the 64-word Descriptor h/w Buffer 1114 */ 1115 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + 1116 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1117 keys_fit_inline = true; 1118 1119 desc = ctx->sh_desc_enc; 1120 1121 init_sh_desc(desc, HDR_SHARE_SERIAL); 1122 1123 /* Skip key loading if it is loaded due to sharing */ 1124 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1125 JUMP_COND_SHRD); 1126 if (keys_fit_inline) 1127 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1128 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1129 else 1130 append_key(desc, ctx->key_dma, ctx->enckeylen, 1131 CLASS_1 | KEY_DEST_CLASS_REG); 1132 set_jump_tgt_here(desc, key_jump_cmd); 1133 1134 /* Class 1 operation */ 1135 append_operation(desc, ctx->class1_alg_type | 1136 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1137 1138 /* assoclen + cryptlen = seqinlen */ 1139 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1140 1141 /* 1142 * MOVE_LEN opcode is not available in all SEC HW revisions, 1143 * thus need to do some magic, i.e. self-patch the descriptor 1144 * buffer. 1145 */ 1146 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1147 (0x6 << MOVE_LEN_SHIFT)); 1148 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1149 (0x8 << MOVE_LEN_SHIFT)); 1150 1151 /* Will read assoclen + cryptlen bytes */ 1152 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1153 1154 /* Will write assoclen + cryptlen bytes */ 1155 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1156 1157 /* Read and write assoclen + cryptlen bytes */ 1158 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1159 1160 set_move_tgt_here(desc, read_move_cmd); 1161 set_move_tgt_here(desc, write_move_cmd); 1162 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1163 /* Move payload data to OFIFO */ 1164 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1165 1166 /* Write ICV */ 1167 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1168 LDST_SRCDST_BYTE_CONTEXT); 1169 1170 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1171 desc_bytes(desc), 1172 DMA_TO_DEVICE); 1173 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1174 dev_err(jrdev, "unable to map shared descriptor\n"); 1175 return -ENOMEM; 1176 } 1177 #ifdef DEBUG 1178 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", 1179 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1180 desc_bytes(desc), 1); 1181 #endif 1182 1183 /* 1184 * Job Descriptor and Shared Descriptors 1185 * must all fit into the 64-word Descriptor h/w Buffer 1186 */ 1187 keys_fit_inline = false; 1188 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + 1189 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1190 keys_fit_inline = true; 1191 1192 desc = ctx->sh_desc_dec; 1193 1194 init_sh_desc(desc, HDR_SHARE_SERIAL); 1195 1196 /* Skip key loading if it is loaded due to sharing */ 1197 key_jump_cmd = append_jump(desc, JUMP_JSL | 1198 JUMP_TEST_ALL | JUMP_COND_SHRD); 1199 if (keys_fit_inline) 1200 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1201 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1202 else 1203 append_key(desc, ctx->key_dma, ctx->enckeylen, 1204 CLASS_1 | KEY_DEST_CLASS_REG); 1205 set_jump_tgt_here(desc, key_jump_cmd); 1206 1207 /* Class 1 operation */ 1208 append_operation(desc, ctx->class1_alg_type | 1209 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1210 1211 /* assoclen + cryptlen = seqoutlen */ 1212 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1213 1214 /* 1215 * MOVE_LEN opcode is not available in all SEC HW revisions, 1216 * thus need to do some magic, i.e. self-patch the descriptor 1217 * buffer. 1218 */ 1219 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1220 (0x6 << MOVE_LEN_SHIFT)); 1221 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1222 (0x8 << MOVE_LEN_SHIFT)); 1223 1224 /* Will read assoclen + cryptlen bytes */ 1225 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1226 1227 /* Will write assoclen + cryptlen bytes */ 1228 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1229 1230 /* Store payload data */ 1231 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1232 1233 /* In-snoop assoclen + cryptlen data */ 1234 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1235 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1236 1237 set_move_tgt_here(desc, read_move_cmd); 1238 set_move_tgt_here(desc, write_move_cmd); 1239 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1240 /* Move payload data to OFIFO */ 1241 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1242 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1243 1244 /* Read ICV */ 1245 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1246 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1247 1248 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1249 desc_bytes(desc), 1250 DMA_TO_DEVICE); 1251 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1252 dev_err(jrdev, "unable to map shared descriptor\n"); 1253 return -ENOMEM; 1254 } 1255 #ifdef DEBUG 1256 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", 1257 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1258 desc_bytes(desc), 1); 1259 #endif 1260 1261 return 0; 1262 } 1263 1264 static int rfc4543_setauthsize(struct crypto_aead *authenc, 1265 unsigned int authsize) 1266 { 1267 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1268 1269 ctx->authsize = authsize; 1270 rfc4543_set_sh_desc(authenc); 1271 1272 return 0; 1273 } 1274 1275 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 1276 u32 authkeylen) 1277 { 1278 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 1279 ctx->split_key_pad_len, key_in, authkeylen, 1280 ctx->alg_op); 1281 } 1282 1283 static int aead_setkey(struct crypto_aead *aead, 1284 const u8 *key, unsigned int keylen) 1285 { 1286 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 1287 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 1288 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1289 struct device *jrdev = ctx->jrdev; 1290 struct crypto_authenc_keys keys; 1291 int ret = 0; 1292 1293 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1294 goto badkey; 1295 1296 /* Pick class 2 key length from algorithm submask */ 1297 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1298 OP_ALG_ALGSEL_SHIFT] * 2; 1299 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 1300 1301 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) 1302 goto badkey; 1303 1304 #ifdef DEBUG 1305 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 1306 keys.authkeylen + keys.enckeylen, keys.enckeylen, 1307 keys.authkeylen); 1308 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 1309 ctx->split_key_len, ctx->split_key_pad_len); 1310 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1311 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1312 #endif 1313 1314 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); 1315 if (ret) { 1316 goto badkey; 1317 } 1318 1319 /* postpend encryption key to auth split key */ 1320 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); 1321 1322 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 1323 keys.enckeylen, DMA_TO_DEVICE); 1324 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1325 dev_err(jrdev, "unable to map key i/o memory\n"); 1326 return -ENOMEM; 1327 } 1328 #ifdef DEBUG 1329 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 1330 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 1331 ctx->split_key_pad_len + keys.enckeylen, 1); 1332 #endif 1333 1334 ctx->enckeylen = keys.enckeylen; 1335 1336 ret = aead_set_sh_desc(aead); 1337 if (ret) { 1338 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 1339 keys.enckeylen, DMA_TO_DEVICE); 1340 } 1341 1342 return ret; 1343 badkey: 1344 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 1345 return -EINVAL; 1346 } 1347 1348 static int gcm_setkey(struct crypto_aead *aead, 1349 const u8 *key, unsigned int keylen) 1350 { 1351 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1352 struct device *jrdev = ctx->jrdev; 1353 int ret = 0; 1354 1355 #ifdef DEBUG 1356 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1357 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1358 #endif 1359 1360 memcpy(ctx->key, key, keylen); 1361 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1362 DMA_TO_DEVICE); 1363 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1364 dev_err(jrdev, "unable to map key i/o memory\n"); 1365 return -ENOMEM; 1366 } 1367 ctx->enckeylen = keylen; 1368 1369 ret = gcm_set_sh_desc(aead); 1370 if (ret) { 1371 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1372 DMA_TO_DEVICE); 1373 } 1374 1375 return ret; 1376 } 1377 1378 static int rfc4106_setkey(struct crypto_aead *aead, 1379 const u8 *key, unsigned int keylen) 1380 { 1381 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1382 struct device *jrdev = ctx->jrdev; 1383 int ret = 0; 1384 1385 if (keylen < 4) 1386 return -EINVAL; 1387 1388 #ifdef DEBUG 1389 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1390 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1391 #endif 1392 1393 memcpy(ctx->key, key, keylen); 1394 1395 /* 1396 * The last four bytes of the key material are used as the salt value 1397 * in the nonce. Update the AES key length. 1398 */ 1399 ctx->enckeylen = keylen - 4; 1400 1401 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1402 DMA_TO_DEVICE); 1403 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1404 dev_err(jrdev, "unable to map key i/o memory\n"); 1405 return -ENOMEM; 1406 } 1407 1408 ret = rfc4106_set_sh_desc(aead); 1409 if (ret) { 1410 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1411 DMA_TO_DEVICE); 1412 } 1413 1414 return ret; 1415 } 1416 1417 static int rfc4543_setkey(struct crypto_aead *aead, 1418 const u8 *key, unsigned int keylen) 1419 { 1420 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1421 struct device *jrdev = ctx->jrdev; 1422 int ret = 0; 1423 1424 if (keylen < 4) 1425 return -EINVAL; 1426 1427 #ifdef DEBUG 1428 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1429 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1430 #endif 1431 1432 memcpy(ctx->key, key, keylen); 1433 1434 /* 1435 * The last four bytes of the key material are used as the salt value 1436 * in the nonce. Update the AES key length. 1437 */ 1438 ctx->enckeylen = keylen - 4; 1439 1440 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1441 DMA_TO_DEVICE); 1442 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1443 dev_err(jrdev, "unable to map key i/o memory\n"); 1444 return -ENOMEM; 1445 } 1446 1447 ret = rfc4543_set_sh_desc(aead); 1448 if (ret) { 1449 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1450 DMA_TO_DEVICE); 1451 } 1452 1453 return ret; 1454 } 1455 1456 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1457 const u8 *key, unsigned int keylen) 1458 { 1459 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1460 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; 1461 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 1462 const char *alg_name = crypto_tfm_alg_name(tfm); 1463 struct device *jrdev = ctx->jrdev; 1464 int ret = 0; 1465 u32 *key_jump_cmd; 1466 u32 *desc; 1467 u32 *nonce; 1468 u32 geniv; 1469 u32 ctx1_iv_off = 0; 1470 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 1471 OP_ALG_AAI_CTR_MOD128); 1472 const bool is_rfc3686 = (ctr_mode && 1473 (strstr(alg_name, "rfc3686") != NULL)); 1474 1475 #ifdef DEBUG 1476 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1477 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1478 #endif 1479 /* 1480 * AES-CTR needs to load IV in CONTEXT1 reg 1481 * at an offset of 128bits (16bytes) 1482 * CONTEXT1[255:128] = IV 1483 */ 1484 if (ctr_mode) 1485 ctx1_iv_off = 16; 1486 1487 /* 1488 * RFC3686 specific: 1489 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1490 * | *key = {KEY, NONCE} 1491 */ 1492 if (is_rfc3686) { 1493 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1494 keylen -= CTR_RFC3686_NONCE_SIZE; 1495 } 1496 1497 memcpy(ctx->key, key, keylen); 1498 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1499 DMA_TO_DEVICE); 1500 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1501 dev_err(jrdev, "unable to map key i/o memory\n"); 1502 return -ENOMEM; 1503 } 1504 ctx->enckeylen = keylen; 1505 1506 /* ablkcipher_encrypt shared descriptor */ 1507 desc = ctx->sh_desc_enc; 1508 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1509 /* Skip if already shared */ 1510 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1511 JUMP_COND_SHRD); 1512 1513 /* Load class1 key only */ 1514 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1515 ctx->enckeylen, CLASS_1 | 1516 KEY_DEST_CLASS_REG); 1517 1518 /* Load nonce into CONTEXT1 reg */ 1519 if (is_rfc3686) { 1520 nonce = (u32 *)(key + keylen); 1521 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1522 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1523 append_move(desc, MOVE_WAITCOMP | 1524 MOVE_SRC_OUTFIFO | 1525 MOVE_DEST_CLASS1CTX | 1526 (16 << MOVE_OFFSET_SHIFT) | 1527 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1528 } 1529 1530 set_jump_tgt_here(desc, key_jump_cmd); 1531 1532 /* Load iv */ 1533 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1534 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1535 1536 /* Load counter into CONTEXT1 reg */ 1537 if (is_rfc3686) 1538 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1539 LDST_CLASS_1_CCB | 1540 LDST_SRCDST_BYTE_CONTEXT | 1541 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1542 LDST_OFFSET_SHIFT)); 1543 1544 /* Load operation */ 1545 append_operation(desc, ctx->class1_alg_type | 1546 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1547 1548 /* Perform operation */ 1549 ablkcipher_append_src_dst(desc); 1550 1551 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1552 desc_bytes(desc), 1553 DMA_TO_DEVICE); 1554 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1555 dev_err(jrdev, "unable to map shared descriptor\n"); 1556 return -ENOMEM; 1557 } 1558 #ifdef DEBUG 1559 print_hex_dump(KERN_ERR, 1560 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 1561 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1562 desc_bytes(desc), 1); 1563 #endif 1564 /* ablkcipher_decrypt shared descriptor */ 1565 desc = ctx->sh_desc_dec; 1566 1567 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1568 /* Skip if already shared */ 1569 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1570 JUMP_COND_SHRD); 1571 1572 /* Load class1 key only */ 1573 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1574 ctx->enckeylen, CLASS_1 | 1575 KEY_DEST_CLASS_REG); 1576 1577 /* Load nonce into CONTEXT1 reg */ 1578 if (is_rfc3686) { 1579 nonce = (u32 *)(key + keylen); 1580 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1581 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1582 append_move(desc, MOVE_WAITCOMP | 1583 MOVE_SRC_OUTFIFO | 1584 MOVE_DEST_CLASS1CTX | 1585 (16 << MOVE_OFFSET_SHIFT) | 1586 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1587 } 1588 1589 set_jump_tgt_here(desc, key_jump_cmd); 1590 1591 /* load IV */ 1592 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1593 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1594 1595 /* Load counter into CONTEXT1 reg */ 1596 if (is_rfc3686) 1597 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1598 LDST_CLASS_1_CCB | 1599 LDST_SRCDST_BYTE_CONTEXT | 1600 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1601 LDST_OFFSET_SHIFT)); 1602 1603 /* Choose operation */ 1604 if (ctr_mode) 1605 append_operation(desc, ctx->class1_alg_type | 1606 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 1607 else 1608 append_dec_op1(desc, ctx->class1_alg_type); 1609 1610 /* Perform operation */ 1611 ablkcipher_append_src_dst(desc); 1612 1613 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1614 desc_bytes(desc), 1615 DMA_TO_DEVICE); 1616 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1617 dev_err(jrdev, "unable to map shared descriptor\n"); 1618 return -ENOMEM; 1619 } 1620 1621 #ifdef DEBUG 1622 print_hex_dump(KERN_ERR, 1623 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 1624 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1625 desc_bytes(desc), 1); 1626 #endif 1627 /* ablkcipher_givencrypt shared descriptor */ 1628 desc = ctx->sh_desc_givenc; 1629 1630 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1631 /* Skip if already shared */ 1632 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1633 JUMP_COND_SHRD); 1634 1635 /* Load class1 key only */ 1636 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1637 ctx->enckeylen, CLASS_1 | 1638 KEY_DEST_CLASS_REG); 1639 1640 /* Load Nonce into CONTEXT1 reg */ 1641 if (is_rfc3686) { 1642 nonce = (u32 *)(key + keylen); 1643 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1644 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1645 append_move(desc, MOVE_WAITCOMP | 1646 MOVE_SRC_OUTFIFO | 1647 MOVE_DEST_CLASS1CTX | 1648 (16 << MOVE_OFFSET_SHIFT) | 1649 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1650 } 1651 set_jump_tgt_here(desc, key_jump_cmd); 1652 1653 /* Generate IV */ 1654 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1655 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 1656 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); 1657 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1658 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1659 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1660 append_move(desc, MOVE_WAITCOMP | 1661 MOVE_SRC_INFIFO | 1662 MOVE_DEST_CLASS1CTX | 1663 (crt->ivsize << MOVE_LEN_SHIFT) | 1664 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1665 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1666 1667 /* Copy generated IV to memory */ 1668 append_seq_store(desc, crt->ivsize, 1669 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1670 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1671 1672 /* Load Counter into CONTEXT1 reg */ 1673 if (is_rfc3686) 1674 append_load_imm_u32(desc, (u32)1, LDST_IMM | 1675 LDST_CLASS_1_CCB | 1676 LDST_SRCDST_BYTE_CONTEXT | 1677 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1678 LDST_OFFSET_SHIFT)); 1679 1680 if (ctx1_iv_off) 1681 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1682 (1 << JUMP_OFFSET_SHIFT)); 1683 1684 /* Load operation */ 1685 append_operation(desc, ctx->class1_alg_type | 1686 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1687 1688 /* Perform operation */ 1689 ablkcipher_append_src_dst(desc); 1690 1691 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 1692 desc_bytes(desc), 1693 DMA_TO_DEVICE); 1694 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 1695 dev_err(jrdev, "unable to map shared descriptor\n"); 1696 return -ENOMEM; 1697 } 1698 #ifdef DEBUG 1699 print_hex_dump(KERN_ERR, 1700 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1701 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1702 desc_bytes(desc), 1); 1703 #endif 1704 1705 return ret; 1706 } 1707 1708 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1709 const u8 *key, unsigned int keylen) 1710 { 1711 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1712 struct device *jrdev = ctx->jrdev; 1713 u32 *key_jump_cmd, *desc; 1714 __be64 sector_size = cpu_to_be64(512); 1715 1716 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 1717 crypto_ablkcipher_set_flags(ablkcipher, 1718 CRYPTO_TFM_RES_BAD_KEY_LEN); 1719 dev_err(jrdev, "key size mismatch\n"); 1720 return -EINVAL; 1721 } 1722 1723 memcpy(ctx->key, key, keylen); 1724 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); 1725 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1726 dev_err(jrdev, "unable to map key i/o memory\n"); 1727 return -ENOMEM; 1728 } 1729 ctx->enckeylen = keylen; 1730 1731 /* xts_ablkcipher_encrypt shared descriptor */ 1732 desc = ctx->sh_desc_enc; 1733 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1734 /* Skip if already shared */ 1735 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1736 JUMP_COND_SHRD); 1737 1738 /* Load class1 keys only */ 1739 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1740 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1741 1742 /* Load sector size with index 40 bytes (0x28) */ 1743 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | 1744 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); 1745 append_data(desc, (void *)§or_size, 8); 1746 1747 set_jump_tgt_here(desc, key_jump_cmd); 1748 1749 /* 1750 * create sequence for loading the sector index 1751 * Upper 8B of IV - will be used as sector index 1752 * Lower 8B of IV - will be discarded 1753 */ 1754 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 1755 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); 1756 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1757 1758 /* Load operation */ 1759 append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | 1760 OP_ALG_ENCRYPT); 1761 1762 /* Perform operation */ 1763 ablkcipher_append_src_dst(desc); 1764 1765 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 1766 DMA_TO_DEVICE); 1767 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1768 dev_err(jrdev, "unable to map shared descriptor\n"); 1769 return -ENOMEM; 1770 } 1771 #ifdef DEBUG 1772 print_hex_dump(KERN_ERR, 1773 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", 1774 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1775 #endif 1776 1777 /* xts_ablkcipher_decrypt shared descriptor */ 1778 desc = ctx->sh_desc_dec; 1779 1780 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1781 /* Skip if already shared */ 1782 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1783 JUMP_COND_SHRD); 1784 1785 /* Load class1 key only */ 1786 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1787 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1788 1789 /* Load sector size with index 40 bytes (0x28) */ 1790 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | 1791 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); 1792 append_data(desc, (void *)§or_size, 8); 1793 1794 set_jump_tgt_here(desc, key_jump_cmd); 1795 1796 /* 1797 * create sequence for loading the sector index 1798 * Upper 8B of IV - will be used as sector index 1799 * Lower 8B of IV - will be discarded 1800 */ 1801 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 1802 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); 1803 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1804 1805 /* Load operation */ 1806 append_dec_op1(desc, ctx->class1_alg_type); 1807 1808 /* Perform operation */ 1809 ablkcipher_append_src_dst(desc); 1810 1811 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 1812 DMA_TO_DEVICE); 1813 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1814 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma, 1815 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 1816 dev_err(jrdev, "unable to map shared descriptor\n"); 1817 return -ENOMEM; 1818 } 1819 #ifdef DEBUG 1820 print_hex_dump(KERN_ERR, 1821 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", 1822 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1823 #endif 1824 1825 return 0; 1826 } 1827 1828 /* 1829 * aead_edesc - s/w-extended aead descriptor 1830 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 1831 * @src_nents: number of segments in input scatterlist 1832 * @dst_nents: number of segments in output scatterlist 1833 * @iv_dma: dma address of iv for checking continuity and link table 1834 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1835 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1836 * @sec4_sg_dma: bus physical mapped address of h/w link table 1837 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1838 */ 1839 struct aead_edesc { 1840 int assoc_nents; 1841 int src_nents; 1842 int dst_nents; 1843 dma_addr_t iv_dma; 1844 int sec4_sg_bytes; 1845 dma_addr_t sec4_sg_dma; 1846 struct sec4_sg_entry *sec4_sg; 1847 u32 hw_desc[]; 1848 }; 1849 1850 /* 1851 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 1852 * @src_nents: number of segments in input scatterlist 1853 * @dst_nents: number of segments in output scatterlist 1854 * @iv_dma: dma address of iv for checking continuity and link table 1855 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1856 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1857 * @sec4_sg_dma: bus physical mapped address of h/w link table 1858 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1859 */ 1860 struct ablkcipher_edesc { 1861 int src_nents; 1862 int dst_nents; 1863 dma_addr_t iv_dma; 1864 int sec4_sg_bytes; 1865 dma_addr_t sec4_sg_dma; 1866 struct sec4_sg_entry *sec4_sg; 1867 u32 hw_desc[0]; 1868 }; 1869 1870 static void caam_unmap(struct device *dev, struct scatterlist *src, 1871 struct scatterlist *dst, int src_nents, 1872 int dst_nents, 1873 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 1874 int sec4_sg_bytes) 1875 { 1876 if (dst != src) { 1877 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); 1878 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); 1879 } else { 1880 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); 1881 } 1882 1883 if (iv_dma) 1884 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 1885 if (sec4_sg_bytes) 1886 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 1887 DMA_TO_DEVICE); 1888 } 1889 1890 static void aead_unmap(struct device *dev, 1891 struct aead_edesc *edesc, 1892 struct aead_request *req) 1893 { 1894 caam_unmap(dev, req->src, req->dst, 1895 edesc->src_nents, edesc->dst_nents, 0, 0, 1896 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1897 } 1898 1899 static void ablkcipher_unmap(struct device *dev, 1900 struct ablkcipher_edesc *edesc, 1901 struct ablkcipher_request *req) 1902 { 1903 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1904 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1905 1906 caam_unmap(dev, req->src, req->dst, 1907 edesc->src_nents, edesc->dst_nents, 1908 edesc->iv_dma, ivsize, 1909 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1910 } 1911 1912 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1913 void *context) 1914 { 1915 struct aead_request *req = context; 1916 struct aead_edesc *edesc; 1917 1918 #ifdef DEBUG 1919 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1920 #endif 1921 1922 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1923 1924 if (err) 1925 caam_jr_strstatus(jrdev, err); 1926 1927 aead_unmap(jrdev, edesc, req); 1928 1929 kfree(edesc); 1930 1931 aead_request_complete(req, err); 1932 } 1933 1934 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1935 void *context) 1936 { 1937 struct aead_request *req = context; 1938 struct aead_edesc *edesc; 1939 1940 #ifdef DEBUG 1941 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1942 #endif 1943 1944 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1945 1946 if (err) 1947 caam_jr_strstatus(jrdev, err); 1948 1949 aead_unmap(jrdev, edesc, req); 1950 1951 /* 1952 * verify hw auth check passed else return -EBADMSG 1953 */ 1954 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 1955 err = -EBADMSG; 1956 1957 kfree(edesc); 1958 1959 aead_request_complete(req, err); 1960 } 1961 1962 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1963 void *context) 1964 { 1965 struct ablkcipher_request *req = context; 1966 struct ablkcipher_edesc *edesc; 1967 #ifdef DEBUG 1968 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1969 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1970 1971 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1972 #endif 1973 1974 edesc = (struct ablkcipher_edesc *)((char *)desc - 1975 offsetof(struct ablkcipher_edesc, hw_desc)); 1976 1977 if (err) 1978 caam_jr_strstatus(jrdev, err); 1979 1980 #ifdef DEBUG 1981 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1982 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1983 edesc->src_nents > 1 ? 100 : ivsize, 1); 1984 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1985 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1986 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1987 #endif 1988 1989 ablkcipher_unmap(jrdev, edesc, req); 1990 kfree(edesc); 1991 1992 ablkcipher_request_complete(req, err); 1993 } 1994 1995 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1996 void *context) 1997 { 1998 struct ablkcipher_request *req = context; 1999 struct ablkcipher_edesc *edesc; 2000 #ifdef DEBUG 2001 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2002 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2003 2004 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 2005 #endif 2006 2007 edesc = (struct ablkcipher_edesc *)((char *)desc - 2008 offsetof(struct ablkcipher_edesc, hw_desc)); 2009 if (err) 2010 caam_jr_strstatus(jrdev, err); 2011 2012 #ifdef DEBUG 2013 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 2014 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2015 ivsize, 1); 2016 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 2017 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2018 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 2019 #endif 2020 2021 ablkcipher_unmap(jrdev, edesc, req); 2022 kfree(edesc); 2023 2024 ablkcipher_request_complete(req, err); 2025 } 2026 2027 /* 2028 * Fill in aead job descriptor 2029 */ 2030 static void init_aead_job(struct aead_request *req, 2031 struct aead_edesc *edesc, 2032 bool all_contig, bool encrypt) 2033 { 2034 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2035 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2036 int authsize = ctx->authsize; 2037 u32 *desc = edesc->hw_desc; 2038 u32 out_options, in_options; 2039 dma_addr_t dst_dma, src_dma; 2040 int len, sec4_sg_index = 0; 2041 dma_addr_t ptr; 2042 u32 *sh_desc; 2043 2044 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 2045 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 2046 2047 len = desc_len(sh_desc); 2048 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2049 2050 if (all_contig) { 2051 src_dma = sg_dma_address(req->src); 2052 in_options = 0; 2053 } else { 2054 src_dma = edesc->sec4_sg_dma; 2055 sec4_sg_index += edesc->src_nents; 2056 in_options = LDST_SGF; 2057 } 2058 2059 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 2060 in_options); 2061 2062 dst_dma = src_dma; 2063 out_options = in_options; 2064 2065 if (unlikely(req->src != req->dst)) { 2066 if (!edesc->dst_nents) { 2067 dst_dma = sg_dma_address(req->dst); 2068 } else { 2069 dst_dma = edesc->sec4_sg_dma + 2070 sec4_sg_index * 2071 sizeof(struct sec4_sg_entry); 2072 out_options = LDST_SGF; 2073 } 2074 } 2075 2076 if (encrypt) 2077 append_seq_out_ptr(desc, dst_dma, 2078 req->assoclen + req->cryptlen + authsize, 2079 out_options); 2080 else 2081 append_seq_out_ptr(desc, dst_dma, 2082 req->assoclen + req->cryptlen - authsize, 2083 out_options); 2084 2085 /* REG3 = assoclen */ 2086 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 2087 } 2088 2089 static void init_gcm_job(struct aead_request *req, 2090 struct aead_edesc *edesc, 2091 bool all_contig, bool encrypt) 2092 { 2093 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2094 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2095 unsigned int ivsize = crypto_aead_ivsize(aead); 2096 u32 *desc = edesc->hw_desc; 2097 bool generic_gcm = (ivsize == 12); 2098 unsigned int last; 2099 2100 init_aead_job(req, edesc, all_contig, encrypt); 2101 2102 /* BUG This should not be specific to generic GCM. */ 2103 last = 0; 2104 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 2105 last = FIFOLD_TYPE_LAST1; 2106 2107 /* Read GCM IV */ 2108 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 2109 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); 2110 /* Append Salt */ 2111 if (!generic_gcm) 2112 append_data(desc, ctx->key + ctx->enckeylen, 4); 2113 /* Append IV */ 2114 append_data(desc, req->iv, ivsize); 2115 /* End of blank commands */ 2116 } 2117 2118 static void init_authenc_job(struct aead_request *req, 2119 struct aead_edesc *edesc, 2120 bool all_contig, bool encrypt) 2121 { 2122 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2123 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 2124 struct caam_aead_alg, aead); 2125 unsigned int ivsize = crypto_aead_ivsize(aead); 2126 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2127 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 2128 OP_ALG_AAI_CTR_MOD128); 2129 const bool is_rfc3686 = alg->caam.rfc3686; 2130 u32 *desc = edesc->hw_desc; 2131 u32 ivoffset = 0; 2132 2133 /* 2134 * AES-CTR needs to load IV in CONTEXT1 reg 2135 * at an offset of 128bits (16bytes) 2136 * CONTEXT1[255:128] = IV 2137 */ 2138 if (ctr_mode) 2139 ivoffset = 16; 2140 2141 /* 2142 * RFC3686 specific: 2143 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 2144 */ 2145 if (is_rfc3686) 2146 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 2147 2148 init_aead_job(req, edesc, all_contig, encrypt); 2149 2150 if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt))) 2151 append_load_as_imm(desc, req->iv, ivsize, 2152 LDST_CLASS_1_CCB | 2153 LDST_SRCDST_BYTE_CONTEXT | 2154 (ivoffset << LDST_OFFSET_SHIFT)); 2155 } 2156 2157 /* 2158 * Fill in ablkcipher job descriptor 2159 */ 2160 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 2161 struct ablkcipher_edesc *edesc, 2162 struct ablkcipher_request *req, 2163 bool iv_contig) 2164 { 2165 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2166 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2167 u32 *desc = edesc->hw_desc; 2168 u32 out_options = 0, in_options; 2169 dma_addr_t dst_dma, src_dma; 2170 int len, sec4_sg_index = 0; 2171 2172 #ifdef DEBUG 2173 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2174 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2175 ivsize, 1); 2176 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2177 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2178 edesc->src_nents ? 100 : req->nbytes, 1); 2179 #endif 2180 2181 len = desc_len(sh_desc); 2182 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2183 2184 if (iv_contig) { 2185 src_dma = edesc->iv_dma; 2186 in_options = 0; 2187 } else { 2188 src_dma = edesc->sec4_sg_dma; 2189 sec4_sg_index += edesc->src_nents + 1; 2190 in_options = LDST_SGF; 2191 } 2192 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 2193 2194 if (likely(req->src == req->dst)) { 2195 if (!edesc->src_nents && iv_contig) { 2196 dst_dma = sg_dma_address(req->src); 2197 } else { 2198 dst_dma = edesc->sec4_sg_dma + 2199 sizeof(struct sec4_sg_entry); 2200 out_options = LDST_SGF; 2201 } 2202 } else { 2203 if (!edesc->dst_nents) { 2204 dst_dma = sg_dma_address(req->dst); 2205 } else { 2206 dst_dma = edesc->sec4_sg_dma + 2207 sec4_sg_index * sizeof(struct sec4_sg_entry); 2208 out_options = LDST_SGF; 2209 } 2210 } 2211 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 2212 } 2213 2214 /* 2215 * Fill in ablkcipher givencrypt job descriptor 2216 */ 2217 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 2218 struct ablkcipher_edesc *edesc, 2219 struct ablkcipher_request *req, 2220 bool iv_contig) 2221 { 2222 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2223 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2224 u32 *desc = edesc->hw_desc; 2225 u32 out_options, in_options; 2226 dma_addr_t dst_dma, src_dma; 2227 int len, sec4_sg_index = 0; 2228 2229 #ifdef DEBUG 2230 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 2231 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2232 ivsize, 1); 2233 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", 2234 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2235 edesc->src_nents ? 100 : req->nbytes, 1); 2236 #endif 2237 2238 len = desc_len(sh_desc); 2239 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2240 2241 if (!edesc->src_nents) { 2242 src_dma = sg_dma_address(req->src); 2243 in_options = 0; 2244 } else { 2245 src_dma = edesc->sec4_sg_dma; 2246 sec4_sg_index += edesc->src_nents; 2247 in_options = LDST_SGF; 2248 } 2249 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 2250 2251 if (iv_contig) { 2252 dst_dma = edesc->iv_dma; 2253 out_options = 0; 2254 } else { 2255 dst_dma = edesc->sec4_sg_dma + 2256 sec4_sg_index * sizeof(struct sec4_sg_entry); 2257 out_options = LDST_SGF; 2258 } 2259 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 2260 } 2261 2262 /* 2263 * allocate and map the aead extended descriptor 2264 */ 2265 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 2266 int desc_bytes, bool *all_contig_ptr, 2267 bool encrypt) 2268 { 2269 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2270 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2271 struct device *jrdev = ctx->jrdev; 2272 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2273 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2274 int src_nents, dst_nents = 0; 2275 struct aead_edesc *edesc; 2276 int sgc; 2277 bool all_contig = true; 2278 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2279 unsigned int authsize = ctx->authsize; 2280 2281 if (unlikely(req->dst != req->src)) { 2282 src_nents = sg_count(req->src, req->assoclen + req->cryptlen); 2283 dst_nents = sg_count(req->dst, 2284 req->assoclen + req->cryptlen + 2285 (encrypt ? authsize : (-authsize))); 2286 } else { 2287 src_nents = sg_count(req->src, 2288 req->assoclen + req->cryptlen + 2289 (encrypt ? authsize : 0)); 2290 } 2291 2292 /* Check if data are contiguous. */ 2293 all_contig = !src_nents; 2294 if (!all_contig) { 2295 src_nents = src_nents ? : 1; 2296 sec4_sg_len = src_nents; 2297 } 2298 2299 sec4_sg_len += dst_nents; 2300 2301 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2302 2303 /* allocate space for base edesc and hw desc commands, link tables */ 2304 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2305 GFP_DMA | flags); 2306 if (!edesc) { 2307 dev_err(jrdev, "could not allocate extended descriptor\n"); 2308 return ERR_PTR(-ENOMEM); 2309 } 2310 2311 if (likely(req->src == req->dst)) { 2312 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2313 DMA_BIDIRECTIONAL); 2314 if (unlikely(!sgc)) { 2315 dev_err(jrdev, "unable to map source\n"); 2316 kfree(edesc); 2317 return ERR_PTR(-ENOMEM); 2318 } 2319 } else { 2320 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2321 DMA_TO_DEVICE); 2322 if (unlikely(!sgc)) { 2323 dev_err(jrdev, "unable to map source\n"); 2324 kfree(edesc); 2325 return ERR_PTR(-ENOMEM); 2326 } 2327 2328 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2329 DMA_FROM_DEVICE); 2330 if (unlikely(!sgc)) { 2331 dev_err(jrdev, "unable to map destination\n"); 2332 dma_unmap_sg(jrdev, req->src, src_nents ? : 1, 2333 DMA_TO_DEVICE); 2334 kfree(edesc); 2335 return ERR_PTR(-ENOMEM); 2336 } 2337 } 2338 2339 edesc->src_nents = src_nents; 2340 edesc->dst_nents = dst_nents; 2341 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2342 desc_bytes; 2343 *all_contig_ptr = all_contig; 2344 2345 sec4_sg_index = 0; 2346 if (!all_contig) { 2347 sg_to_sec4_sg_last(req->src, src_nents, 2348 edesc->sec4_sg + sec4_sg_index, 0); 2349 sec4_sg_index += src_nents; 2350 } 2351 if (dst_nents) { 2352 sg_to_sec4_sg_last(req->dst, dst_nents, 2353 edesc->sec4_sg + sec4_sg_index, 0); 2354 } 2355 2356 if (!sec4_sg_bytes) 2357 return edesc; 2358 2359 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2360 sec4_sg_bytes, DMA_TO_DEVICE); 2361 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2362 dev_err(jrdev, "unable to map S/G table\n"); 2363 aead_unmap(jrdev, edesc, req); 2364 kfree(edesc); 2365 return ERR_PTR(-ENOMEM); 2366 } 2367 2368 edesc->sec4_sg_bytes = sec4_sg_bytes; 2369 2370 return edesc; 2371 } 2372 2373 static int gcm_encrypt(struct aead_request *req) 2374 { 2375 struct aead_edesc *edesc; 2376 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2377 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2378 struct device *jrdev = ctx->jrdev; 2379 bool all_contig; 2380 u32 *desc; 2381 int ret = 0; 2382 2383 /* allocate extended descriptor */ 2384 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 2385 if (IS_ERR(edesc)) 2386 return PTR_ERR(edesc); 2387 2388 /* Create and submit job descriptor */ 2389 init_gcm_job(req, edesc, all_contig, true); 2390 #ifdef DEBUG 2391 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2392 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2393 desc_bytes(edesc->hw_desc), 1); 2394 #endif 2395 2396 desc = edesc->hw_desc; 2397 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2398 if (!ret) { 2399 ret = -EINPROGRESS; 2400 } else { 2401 aead_unmap(jrdev, edesc, req); 2402 kfree(edesc); 2403 } 2404 2405 return ret; 2406 } 2407 2408 static int ipsec_gcm_encrypt(struct aead_request *req) 2409 { 2410 if (req->assoclen < 8) 2411 return -EINVAL; 2412 2413 return gcm_encrypt(req); 2414 } 2415 2416 static int aead_encrypt(struct aead_request *req) 2417 { 2418 struct aead_edesc *edesc; 2419 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2420 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2421 struct device *jrdev = ctx->jrdev; 2422 bool all_contig; 2423 u32 *desc; 2424 int ret = 0; 2425 2426 /* allocate extended descriptor */ 2427 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2428 &all_contig, true); 2429 if (IS_ERR(edesc)) 2430 return PTR_ERR(edesc); 2431 2432 /* Create and submit job descriptor */ 2433 init_authenc_job(req, edesc, all_contig, true); 2434 #ifdef DEBUG 2435 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2436 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2437 desc_bytes(edesc->hw_desc), 1); 2438 #endif 2439 2440 desc = edesc->hw_desc; 2441 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2442 if (!ret) { 2443 ret = -EINPROGRESS; 2444 } else { 2445 aead_unmap(jrdev, edesc, req); 2446 kfree(edesc); 2447 } 2448 2449 return ret; 2450 } 2451 2452 static int gcm_decrypt(struct aead_request *req) 2453 { 2454 struct aead_edesc *edesc; 2455 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2456 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2457 struct device *jrdev = ctx->jrdev; 2458 bool all_contig; 2459 u32 *desc; 2460 int ret = 0; 2461 2462 /* allocate extended descriptor */ 2463 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 2464 if (IS_ERR(edesc)) 2465 return PTR_ERR(edesc); 2466 2467 /* Create and submit job descriptor*/ 2468 init_gcm_job(req, edesc, all_contig, false); 2469 #ifdef DEBUG 2470 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2471 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2472 desc_bytes(edesc->hw_desc), 1); 2473 #endif 2474 2475 desc = edesc->hw_desc; 2476 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2477 if (!ret) { 2478 ret = -EINPROGRESS; 2479 } else { 2480 aead_unmap(jrdev, edesc, req); 2481 kfree(edesc); 2482 } 2483 2484 return ret; 2485 } 2486 2487 static int ipsec_gcm_decrypt(struct aead_request *req) 2488 { 2489 if (req->assoclen < 8) 2490 return -EINVAL; 2491 2492 return gcm_decrypt(req); 2493 } 2494 2495 static int aead_decrypt(struct aead_request *req) 2496 { 2497 struct aead_edesc *edesc; 2498 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2499 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2500 struct device *jrdev = ctx->jrdev; 2501 bool all_contig; 2502 u32 *desc; 2503 int ret = 0; 2504 2505 /* allocate extended descriptor */ 2506 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2507 &all_contig, false); 2508 if (IS_ERR(edesc)) 2509 return PTR_ERR(edesc); 2510 2511 #ifdef DEBUG 2512 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 2513 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2514 req->assoclen + req->cryptlen, 1); 2515 #endif 2516 2517 /* Create and submit job descriptor*/ 2518 init_authenc_job(req, edesc, all_contig, false); 2519 #ifdef DEBUG 2520 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2521 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2522 desc_bytes(edesc->hw_desc), 1); 2523 #endif 2524 2525 desc = edesc->hw_desc; 2526 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2527 if (!ret) { 2528 ret = -EINPROGRESS; 2529 } else { 2530 aead_unmap(jrdev, edesc, req); 2531 kfree(edesc); 2532 } 2533 2534 return ret; 2535 } 2536 2537 static int aead_givdecrypt(struct aead_request *req) 2538 { 2539 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2540 unsigned int ivsize = crypto_aead_ivsize(aead); 2541 2542 if (req->cryptlen < ivsize) 2543 return -EINVAL; 2544 2545 req->cryptlen -= ivsize; 2546 req->assoclen += ivsize; 2547 2548 return aead_decrypt(req); 2549 } 2550 2551 /* 2552 * allocate and map the ablkcipher extended descriptor for ablkcipher 2553 */ 2554 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 2555 *req, int desc_bytes, 2556 bool *iv_contig_out) 2557 { 2558 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2559 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2560 struct device *jrdev = ctx->jrdev; 2561 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2562 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2563 GFP_KERNEL : GFP_ATOMIC; 2564 int src_nents, dst_nents = 0, sec4_sg_bytes; 2565 struct ablkcipher_edesc *edesc; 2566 dma_addr_t iv_dma = 0; 2567 bool iv_contig = false; 2568 int sgc; 2569 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2570 int sec4_sg_index; 2571 2572 src_nents = sg_count(req->src, req->nbytes); 2573 2574 if (req->dst != req->src) 2575 dst_nents = sg_count(req->dst, req->nbytes); 2576 2577 if (likely(req->src == req->dst)) { 2578 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2579 DMA_BIDIRECTIONAL); 2580 } else { 2581 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2582 DMA_TO_DEVICE); 2583 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2584 DMA_FROM_DEVICE); 2585 } 2586 2587 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 2588 if (dma_mapping_error(jrdev, iv_dma)) { 2589 dev_err(jrdev, "unable to map IV\n"); 2590 return ERR_PTR(-ENOMEM); 2591 } 2592 2593 /* 2594 * Check if iv can be contiguous with source and destination. 2595 * If so, include it. If not, create scatterlist. 2596 */ 2597 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 2598 iv_contig = true; 2599 else 2600 src_nents = src_nents ? : 1; 2601 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2602 sizeof(struct sec4_sg_entry); 2603 2604 /* allocate space for base edesc and hw desc commands, link tables */ 2605 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2606 GFP_DMA | flags); 2607 if (!edesc) { 2608 dev_err(jrdev, "could not allocate extended descriptor\n"); 2609 return ERR_PTR(-ENOMEM); 2610 } 2611 2612 edesc->src_nents = src_nents; 2613 edesc->dst_nents = dst_nents; 2614 edesc->sec4_sg_bytes = sec4_sg_bytes; 2615 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2616 desc_bytes; 2617 2618 sec4_sg_index = 0; 2619 if (!iv_contig) { 2620 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 2621 sg_to_sec4_sg_last(req->src, src_nents, 2622 edesc->sec4_sg + 1, 0); 2623 sec4_sg_index += 1 + src_nents; 2624 } 2625 2626 if (dst_nents) { 2627 sg_to_sec4_sg_last(req->dst, dst_nents, 2628 edesc->sec4_sg + sec4_sg_index, 0); 2629 } 2630 2631 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2632 sec4_sg_bytes, DMA_TO_DEVICE); 2633 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2634 dev_err(jrdev, "unable to map S/G table\n"); 2635 return ERR_PTR(-ENOMEM); 2636 } 2637 2638 edesc->iv_dma = iv_dma; 2639 2640 #ifdef DEBUG 2641 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 2642 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2643 sec4_sg_bytes, 1); 2644 #endif 2645 2646 *iv_contig_out = iv_contig; 2647 return edesc; 2648 } 2649 2650 static int ablkcipher_encrypt(struct ablkcipher_request *req) 2651 { 2652 struct ablkcipher_edesc *edesc; 2653 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2654 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2655 struct device *jrdev = ctx->jrdev; 2656 bool iv_contig; 2657 u32 *desc; 2658 int ret = 0; 2659 2660 /* allocate extended descriptor */ 2661 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2662 CAAM_CMD_SZ, &iv_contig); 2663 if (IS_ERR(edesc)) 2664 return PTR_ERR(edesc); 2665 2666 /* Create and submit job descriptor*/ 2667 init_ablkcipher_job(ctx->sh_desc_enc, 2668 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 2669 #ifdef DEBUG 2670 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2671 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2672 desc_bytes(edesc->hw_desc), 1); 2673 #endif 2674 desc = edesc->hw_desc; 2675 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2676 2677 if (!ret) { 2678 ret = -EINPROGRESS; 2679 } else { 2680 ablkcipher_unmap(jrdev, edesc, req); 2681 kfree(edesc); 2682 } 2683 2684 return ret; 2685 } 2686 2687 static int ablkcipher_decrypt(struct ablkcipher_request *req) 2688 { 2689 struct ablkcipher_edesc *edesc; 2690 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2691 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2692 struct device *jrdev = ctx->jrdev; 2693 bool iv_contig; 2694 u32 *desc; 2695 int ret = 0; 2696 2697 /* allocate extended descriptor */ 2698 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2699 CAAM_CMD_SZ, &iv_contig); 2700 if (IS_ERR(edesc)) 2701 return PTR_ERR(edesc); 2702 2703 /* Create and submit job descriptor*/ 2704 init_ablkcipher_job(ctx->sh_desc_dec, 2705 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 2706 desc = edesc->hw_desc; 2707 #ifdef DEBUG 2708 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2709 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2710 desc_bytes(edesc->hw_desc), 1); 2711 #endif 2712 2713 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 2714 if (!ret) { 2715 ret = -EINPROGRESS; 2716 } else { 2717 ablkcipher_unmap(jrdev, edesc, req); 2718 kfree(edesc); 2719 } 2720 2721 return ret; 2722 } 2723 2724 /* 2725 * allocate and map the ablkcipher extended descriptor 2726 * for ablkcipher givencrypt 2727 */ 2728 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 2729 struct skcipher_givcrypt_request *greq, 2730 int desc_bytes, 2731 bool *iv_contig_out) 2732 { 2733 struct ablkcipher_request *req = &greq->creq; 2734 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2735 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2736 struct device *jrdev = ctx->jrdev; 2737 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2738 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2739 GFP_KERNEL : GFP_ATOMIC; 2740 int src_nents, dst_nents = 0, sec4_sg_bytes; 2741 struct ablkcipher_edesc *edesc; 2742 dma_addr_t iv_dma = 0; 2743 bool iv_contig = false; 2744 int sgc; 2745 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2746 int sec4_sg_index; 2747 2748 src_nents = sg_count(req->src, req->nbytes); 2749 2750 if (unlikely(req->dst != req->src)) 2751 dst_nents = sg_count(req->dst, req->nbytes); 2752 2753 if (likely(req->src == req->dst)) { 2754 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2755 DMA_BIDIRECTIONAL); 2756 } else { 2757 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2758 DMA_TO_DEVICE); 2759 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2760 DMA_FROM_DEVICE); 2761 } 2762 2763 /* 2764 * Check if iv can be contiguous with source and destination. 2765 * If so, include it. If not, create scatterlist. 2766 */ 2767 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 2768 if (dma_mapping_error(jrdev, iv_dma)) { 2769 dev_err(jrdev, "unable to map IV\n"); 2770 return ERR_PTR(-ENOMEM); 2771 } 2772 2773 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) 2774 iv_contig = true; 2775 else 2776 dst_nents = dst_nents ? : 1; 2777 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2778 sizeof(struct sec4_sg_entry); 2779 2780 /* allocate space for base edesc and hw desc commands, link tables */ 2781 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2782 GFP_DMA | flags); 2783 if (!edesc) { 2784 dev_err(jrdev, "could not allocate extended descriptor\n"); 2785 return ERR_PTR(-ENOMEM); 2786 } 2787 2788 edesc->src_nents = src_nents; 2789 edesc->dst_nents = dst_nents; 2790 edesc->sec4_sg_bytes = sec4_sg_bytes; 2791 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2792 desc_bytes; 2793 2794 sec4_sg_index = 0; 2795 if (src_nents) { 2796 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 2797 sec4_sg_index += src_nents; 2798 } 2799 2800 if (!iv_contig) { 2801 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2802 iv_dma, ivsize, 0); 2803 sec4_sg_index += 1; 2804 sg_to_sec4_sg_last(req->dst, dst_nents, 2805 edesc->sec4_sg + sec4_sg_index, 0); 2806 } 2807 2808 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2809 sec4_sg_bytes, DMA_TO_DEVICE); 2810 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2811 dev_err(jrdev, "unable to map S/G table\n"); 2812 return ERR_PTR(-ENOMEM); 2813 } 2814 edesc->iv_dma = iv_dma; 2815 2816 #ifdef DEBUG 2817 print_hex_dump(KERN_ERR, 2818 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 2819 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2820 sec4_sg_bytes, 1); 2821 #endif 2822 2823 *iv_contig_out = iv_contig; 2824 return edesc; 2825 } 2826 2827 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 2828 { 2829 struct ablkcipher_request *req = &creq->creq; 2830 struct ablkcipher_edesc *edesc; 2831 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2832 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2833 struct device *jrdev = ctx->jrdev; 2834 bool iv_contig; 2835 u32 *desc; 2836 int ret = 0; 2837 2838 /* allocate extended descriptor */ 2839 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 2840 CAAM_CMD_SZ, &iv_contig); 2841 if (IS_ERR(edesc)) 2842 return PTR_ERR(edesc); 2843 2844 /* Create and submit job descriptor*/ 2845 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 2846 edesc, req, iv_contig); 2847 #ifdef DEBUG 2848 print_hex_dump(KERN_ERR, 2849 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 2850 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2851 desc_bytes(edesc->hw_desc), 1); 2852 #endif 2853 desc = edesc->hw_desc; 2854 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2855 2856 if (!ret) { 2857 ret = -EINPROGRESS; 2858 } else { 2859 ablkcipher_unmap(jrdev, edesc, req); 2860 kfree(edesc); 2861 } 2862 2863 return ret; 2864 } 2865 2866 #define template_aead template_u.aead 2867 #define template_ablkcipher template_u.ablkcipher 2868 struct caam_alg_template { 2869 char name[CRYPTO_MAX_ALG_NAME]; 2870 char driver_name[CRYPTO_MAX_ALG_NAME]; 2871 unsigned int blocksize; 2872 u32 type; 2873 union { 2874 struct ablkcipher_alg ablkcipher; 2875 } template_u; 2876 u32 class1_alg_type; 2877 u32 class2_alg_type; 2878 u32 alg_op; 2879 }; 2880 2881 static struct caam_alg_template driver_algs[] = { 2882 /* ablkcipher descriptor */ 2883 { 2884 .name = "cbc(aes)", 2885 .driver_name = "cbc-aes-caam", 2886 .blocksize = AES_BLOCK_SIZE, 2887 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2888 .template_ablkcipher = { 2889 .setkey = ablkcipher_setkey, 2890 .encrypt = ablkcipher_encrypt, 2891 .decrypt = ablkcipher_decrypt, 2892 .givencrypt = ablkcipher_givencrypt, 2893 .geniv = "<built-in>", 2894 .min_keysize = AES_MIN_KEY_SIZE, 2895 .max_keysize = AES_MAX_KEY_SIZE, 2896 .ivsize = AES_BLOCK_SIZE, 2897 }, 2898 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2899 }, 2900 { 2901 .name = "cbc(des3_ede)", 2902 .driver_name = "cbc-3des-caam", 2903 .blocksize = DES3_EDE_BLOCK_SIZE, 2904 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2905 .template_ablkcipher = { 2906 .setkey = ablkcipher_setkey, 2907 .encrypt = ablkcipher_encrypt, 2908 .decrypt = ablkcipher_decrypt, 2909 .givencrypt = ablkcipher_givencrypt, 2910 .geniv = "<built-in>", 2911 .min_keysize = DES3_EDE_KEY_SIZE, 2912 .max_keysize = DES3_EDE_KEY_SIZE, 2913 .ivsize = DES3_EDE_BLOCK_SIZE, 2914 }, 2915 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2916 }, 2917 { 2918 .name = "cbc(des)", 2919 .driver_name = "cbc-des-caam", 2920 .blocksize = DES_BLOCK_SIZE, 2921 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2922 .template_ablkcipher = { 2923 .setkey = ablkcipher_setkey, 2924 .encrypt = ablkcipher_encrypt, 2925 .decrypt = ablkcipher_decrypt, 2926 .givencrypt = ablkcipher_givencrypt, 2927 .geniv = "<built-in>", 2928 .min_keysize = DES_KEY_SIZE, 2929 .max_keysize = DES_KEY_SIZE, 2930 .ivsize = DES_BLOCK_SIZE, 2931 }, 2932 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2933 }, 2934 { 2935 .name = "ctr(aes)", 2936 .driver_name = "ctr-aes-caam", 2937 .blocksize = 1, 2938 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2939 .template_ablkcipher = { 2940 .setkey = ablkcipher_setkey, 2941 .encrypt = ablkcipher_encrypt, 2942 .decrypt = ablkcipher_decrypt, 2943 .geniv = "chainiv", 2944 .min_keysize = AES_MIN_KEY_SIZE, 2945 .max_keysize = AES_MAX_KEY_SIZE, 2946 .ivsize = AES_BLOCK_SIZE, 2947 }, 2948 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2949 }, 2950 { 2951 .name = "rfc3686(ctr(aes))", 2952 .driver_name = "rfc3686-ctr-aes-caam", 2953 .blocksize = 1, 2954 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2955 .template_ablkcipher = { 2956 .setkey = ablkcipher_setkey, 2957 .encrypt = ablkcipher_encrypt, 2958 .decrypt = ablkcipher_decrypt, 2959 .givencrypt = ablkcipher_givencrypt, 2960 .geniv = "<built-in>", 2961 .min_keysize = AES_MIN_KEY_SIZE + 2962 CTR_RFC3686_NONCE_SIZE, 2963 .max_keysize = AES_MAX_KEY_SIZE + 2964 CTR_RFC3686_NONCE_SIZE, 2965 .ivsize = CTR_RFC3686_IV_SIZE, 2966 }, 2967 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2968 }, 2969 { 2970 .name = "xts(aes)", 2971 .driver_name = "xts-aes-caam", 2972 .blocksize = AES_BLOCK_SIZE, 2973 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2974 .template_ablkcipher = { 2975 .setkey = xts_ablkcipher_setkey, 2976 .encrypt = ablkcipher_encrypt, 2977 .decrypt = ablkcipher_decrypt, 2978 .geniv = "eseqiv", 2979 .min_keysize = 2 * AES_MIN_KEY_SIZE, 2980 .max_keysize = 2 * AES_MAX_KEY_SIZE, 2981 .ivsize = AES_BLOCK_SIZE, 2982 }, 2983 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 2984 }, 2985 }; 2986 2987 static struct caam_aead_alg driver_aeads[] = { 2988 { 2989 .aead = { 2990 .base = { 2991 .cra_name = "rfc4106(gcm(aes))", 2992 .cra_driver_name = "rfc4106-gcm-aes-caam", 2993 .cra_blocksize = 1, 2994 }, 2995 .setkey = rfc4106_setkey, 2996 .setauthsize = rfc4106_setauthsize, 2997 .encrypt = ipsec_gcm_encrypt, 2998 .decrypt = ipsec_gcm_decrypt, 2999 .ivsize = 8, 3000 .maxauthsize = AES_BLOCK_SIZE, 3001 }, 3002 .caam = { 3003 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3004 }, 3005 }, 3006 { 3007 .aead = { 3008 .base = { 3009 .cra_name = "rfc4543(gcm(aes))", 3010 .cra_driver_name = "rfc4543-gcm-aes-caam", 3011 .cra_blocksize = 1, 3012 }, 3013 .setkey = rfc4543_setkey, 3014 .setauthsize = rfc4543_setauthsize, 3015 .encrypt = ipsec_gcm_encrypt, 3016 .decrypt = ipsec_gcm_decrypt, 3017 .ivsize = 8, 3018 .maxauthsize = AES_BLOCK_SIZE, 3019 }, 3020 .caam = { 3021 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3022 }, 3023 }, 3024 /* Galois Counter Mode */ 3025 { 3026 .aead = { 3027 .base = { 3028 .cra_name = "gcm(aes)", 3029 .cra_driver_name = "gcm-aes-caam", 3030 .cra_blocksize = 1, 3031 }, 3032 .setkey = gcm_setkey, 3033 .setauthsize = gcm_setauthsize, 3034 .encrypt = gcm_encrypt, 3035 .decrypt = gcm_decrypt, 3036 .ivsize = 12, 3037 .maxauthsize = AES_BLOCK_SIZE, 3038 }, 3039 .caam = { 3040 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3041 }, 3042 }, 3043 /* single-pass ipsec_esp descriptor */ 3044 { 3045 .aead = { 3046 .base = { 3047 .cra_name = "authenc(hmac(md5)," 3048 "ecb(cipher_null))", 3049 .cra_driver_name = "authenc-hmac-md5-" 3050 "ecb-cipher_null-caam", 3051 .cra_blocksize = NULL_BLOCK_SIZE, 3052 }, 3053 .setkey = aead_setkey, 3054 .setauthsize = aead_setauthsize, 3055 .encrypt = aead_encrypt, 3056 .decrypt = aead_decrypt, 3057 .ivsize = NULL_IV_SIZE, 3058 .maxauthsize = MD5_DIGEST_SIZE, 3059 }, 3060 .caam = { 3061 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3062 OP_ALG_AAI_HMAC_PRECOMP, 3063 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3064 }, 3065 }, 3066 { 3067 .aead = { 3068 .base = { 3069 .cra_name = "authenc(hmac(sha1)," 3070 "ecb(cipher_null))", 3071 .cra_driver_name = "authenc-hmac-sha1-" 3072 "ecb-cipher_null-caam", 3073 .cra_blocksize = NULL_BLOCK_SIZE, 3074 }, 3075 .setkey = aead_setkey, 3076 .setauthsize = aead_setauthsize, 3077 .encrypt = aead_encrypt, 3078 .decrypt = aead_decrypt, 3079 .ivsize = NULL_IV_SIZE, 3080 .maxauthsize = SHA1_DIGEST_SIZE, 3081 }, 3082 .caam = { 3083 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3084 OP_ALG_AAI_HMAC_PRECOMP, 3085 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3086 }, 3087 }, 3088 { 3089 .aead = { 3090 .base = { 3091 .cra_name = "authenc(hmac(sha224)," 3092 "ecb(cipher_null))", 3093 .cra_driver_name = "authenc-hmac-sha224-" 3094 "ecb-cipher_null-caam", 3095 .cra_blocksize = NULL_BLOCK_SIZE, 3096 }, 3097 .setkey = aead_setkey, 3098 .setauthsize = aead_setauthsize, 3099 .encrypt = aead_encrypt, 3100 .decrypt = aead_decrypt, 3101 .ivsize = NULL_IV_SIZE, 3102 .maxauthsize = SHA224_DIGEST_SIZE, 3103 }, 3104 .caam = { 3105 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3106 OP_ALG_AAI_HMAC_PRECOMP, 3107 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3108 }, 3109 }, 3110 { 3111 .aead = { 3112 .base = { 3113 .cra_name = "authenc(hmac(sha256)," 3114 "ecb(cipher_null))", 3115 .cra_driver_name = "authenc-hmac-sha256-" 3116 "ecb-cipher_null-caam", 3117 .cra_blocksize = NULL_BLOCK_SIZE, 3118 }, 3119 .setkey = aead_setkey, 3120 .setauthsize = aead_setauthsize, 3121 .encrypt = aead_encrypt, 3122 .decrypt = aead_decrypt, 3123 .ivsize = NULL_IV_SIZE, 3124 .maxauthsize = SHA256_DIGEST_SIZE, 3125 }, 3126 .caam = { 3127 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3128 OP_ALG_AAI_HMAC_PRECOMP, 3129 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3130 }, 3131 }, 3132 { 3133 .aead = { 3134 .base = { 3135 .cra_name = "authenc(hmac(sha384)," 3136 "ecb(cipher_null))", 3137 .cra_driver_name = "authenc-hmac-sha384-" 3138 "ecb-cipher_null-caam", 3139 .cra_blocksize = NULL_BLOCK_SIZE, 3140 }, 3141 .setkey = aead_setkey, 3142 .setauthsize = aead_setauthsize, 3143 .encrypt = aead_encrypt, 3144 .decrypt = aead_decrypt, 3145 .ivsize = NULL_IV_SIZE, 3146 .maxauthsize = SHA384_DIGEST_SIZE, 3147 }, 3148 .caam = { 3149 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3150 OP_ALG_AAI_HMAC_PRECOMP, 3151 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3152 }, 3153 }, 3154 { 3155 .aead = { 3156 .base = { 3157 .cra_name = "authenc(hmac(sha512)," 3158 "ecb(cipher_null))", 3159 .cra_driver_name = "authenc-hmac-sha512-" 3160 "ecb-cipher_null-caam", 3161 .cra_blocksize = NULL_BLOCK_SIZE, 3162 }, 3163 .setkey = aead_setkey, 3164 .setauthsize = aead_setauthsize, 3165 .encrypt = aead_encrypt, 3166 .decrypt = aead_decrypt, 3167 .ivsize = NULL_IV_SIZE, 3168 .maxauthsize = SHA512_DIGEST_SIZE, 3169 }, 3170 .caam = { 3171 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3172 OP_ALG_AAI_HMAC_PRECOMP, 3173 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3174 }, 3175 }, 3176 { 3177 .aead = { 3178 .base = { 3179 .cra_name = "authenc(hmac(md5),cbc(aes))", 3180 .cra_driver_name = "authenc-hmac-md5-" 3181 "cbc-aes-caam", 3182 .cra_blocksize = AES_BLOCK_SIZE, 3183 }, 3184 .setkey = aead_setkey, 3185 .setauthsize = aead_setauthsize, 3186 .encrypt = aead_encrypt, 3187 .decrypt = aead_decrypt, 3188 .ivsize = AES_BLOCK_SIZE, 3189 .maxauthsize = MD5_DIGEST_SIZE, 3190 }, 3191 .caam = { 3192 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3193 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3194 OP_ALG_AAI_HMAC_PRECOMP, 3195 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3196 }, 3197 }, 3198 { 3199 .aead = { 3200 .base = { 3201 .cra_name = "echainiv(authenc(hmac(md5)," 3202 "cbc(aes)))", 3203 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3204 "cbc-aes-caam", 3205 .cra_blocksize = AES_BLOCK_SIZE, 3206 }, 3207 .setkey = aead_setkey, 3208 .setauthsize = aead_setauthsize, 3209 .encrypt = aead_encrypt, 3210 .decrypt = aead_givdecrypt, 3211 .ivsize = AES_BLOCK_SIZE, 3212 .maxauthsize = MD5_DIGEST_SIZE, 3213 }, 3214 .caam = { 3215 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3216 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3217 OP_ALG_AAI_HMAC_PRECOMP, 3218 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3219 .geniv = true, 3220 }, 3221 }, 3222 { 3223 .aead = { 3224 .base = { 3225 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3226 .cra_driver_name = "authenc-hmac-sha1-" 3227 "cbc-aes-caam", 3228 .cra_blocksize = AES_BLOCK_SIZE, 3229 }, 3230 .setkey = aead_setkey, 3231 .setauthsize = aead_setauthsize, 3232 .encrypt = aead_encrypt, 3233 .decrypt = aead_decrypt, 3234 .ivsize = AES_BLOCK_SIZE, 3235 .maxauthsize = SHA1_DIGEST_SIZE, 3236 }, 3237 .caam = { 3238 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3239 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3240 OP_ALG_AAI_HMAC_PRECOMP, 3241 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3242 }, 3243 }, 3244 { 3245 .aead = { 3246 .base = { 3247 .cra_name = "echainiv(authenc(hmac(sha1)," 3248 "cbc(aes)))", 3249 .cra_driver_name = "echainiv-authenc-" 3250 "hmac-sha1-cbc-aes-caam", 3251 .cra_blocksize = AES_BLOCK_SIZE, 3252 }, 3253 .setkey = aead_setkey, 3254 .setauthsize = aead_setauthsize, 3255 .encrypt = aead_encrypt, 3256 .decrypt = aead_givdecrypt, 3257 .ivsize = AES_BLOCK_SIZE, 3258 .maxauthsize = SHA1_DIGEST_SIZE, 3259 }, 3260 .caam = { 3261 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3262 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3263 OP_ALG_AAI_HMAC_PRECOMP, 3264 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3265 .geniv = true, 3266 }, 3267 }, 3268 { 3269 .aead = { 3270 .base = { 3271 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3272 .cra_driver_name = "authenc-hmac-sha224-" 3273 "cbc-aes-caam", 3274 .cra_blocksize = AES_BLOCK_SIZE, 3275 }, 3276 .setkey = aead_setkey, 3277 .setauthsize = aead_setauthsize, 3278 .encrypt = aead_encrypt, 3279 .decrypt = aead_decrypt, 3280 .ivsize = AES_BLOCK_SIZE, 3281 .maxauthsize = SHA224_DIGEST_SIZE, 3282 }, 3283 .caam = { 3284 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3285 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3286 OP_ALG_AAI_HMAC_PRECOMP, 3287 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3288 }, 3289 }, 3290 { 3291 .aead = { 3292 .base = { 3293 .cra_name = "echainiv(authenc(hmac(sha224)," 3294 "cbc(aes)))", 3295 .cra_driver_name = "echainiv-authenc-" 3296 "hmac-sha224-cbc-aes-caam", 3297 .cra_blocksize = AES_BLOCK_SIZE, 3298 }, 3299 .setkey = aead_setkey, 3300 .setauthsize = aead_setauthsize, 3301 .encrypt = aead_encrypt, 3302 .decrypt = aead_givdecrypt, 3303 .ivsize = AES_BLOCK_SIZE, 3304 .maxauthsize = SHA224_DIGEST_SIZE, 3305 }, 3306 .caam = { 3307 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3308 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3309 OP_ALG_AAI_HMAC_PRECOMP, 3310 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3311 .geniv = true, 3312 }, 3313 }, 3314 { 3315 .aead = { 3316 .base = { 3317 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3318 .cra_driver_name = "authenc-hmac-sha256-" 3319 "cbc-aes-caam", 3320 .cra_blocksize = AES_BLOCK_SIZE, 3321 }, 3322 .setkey = aead_setkey, 3323 .setauthsize = aead_setauthsize, 3324 .encrypt = aead_encrypt, 3325 .decrypt = aead_decrypt, 3326 .ivsize = AES_BLOCK_SIZE, 3327 .maxauthsize = SHA256_DIGEST_SIZE, 3328 }, 3329 .caam = { 3330 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3331 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3332 OP_ALG_AAI_HMAC_PRECOMP, 3333 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3334 }, 3335 }, 3336 { 3337 .aead = { 3338 .base = { 3339 .cra_name = "echainiv(authenc(hmac(sha256)," 3340 "cbc(aes)))", 3341 .cra_driver_name = "echainiv-authenc-" 3342 "hmac-sha256-cbc-aes-caam", 3343 .cra_blocksize = AES_BLOCK_SIZE, 3344 }, 3345 .setkey = aead_setkey, 3346 .setauthsize = aead_setauthsize, 3347 .encrypt = aead_encrypt, 3348 .decrypt = aead_givdecrypt, 3349 .ivsize = AES_BLOCK_SIZE, 3350 .maxauthsize = SHA256_DIGEST_SIZE, 3351 }, 3352 .caam = { 3353 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3354 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3355 OP_ALG_AAI_HMAC_PRECOMP, 3356 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3357 .geniv = true, 3358 }, 3359 }, 3360 { 3361 .aead = { 3362 .base = { 3363 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3364 .cra_driver_name = "authenc-hmac-sha384-" 3365 "cbc-aes-caam", 3366 .cra_blocksize = AES_BLOCK_SIZE, 3367 }, 3368 .setkey = aead_setkey, 3369 .setauthsize = aead_setauthsize, 3370 .encrypt = aead_encrypt, 3371 .decrypt = aead_decrypt, 3372 .ivsize = AES_BLOCK_SIZE, 3373 .maxauthsize = SHA384_DIGEST_SIZE, 3374 }, 3375 .caam = { 3376 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3377 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3378 OP_ALG_AAI_HMAC_PRECOMP, 3379 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3380 }, 3381 }, 3382 { 3383 .aead = { 3384 .base = { 3385 .cra_name = "echainiv(authenc(hmac(sha384)," 3386 "cbc(aes)))", 3387 .cra_driver_name = "echainiv-authenc-" 3388 "hmac-sha384-cbc-aes-caam", 3389 .cra_blocksize = AES_BLOCK_SIZE, 3390 }, 3391 .setkey = aead_setkey, 3392 .setauthsize = aead_setauthsize, 3393 .encrypt = aead_encrypt, 3394 .decrypt = aead_givdecrypt, 3395 .ivsize = AES_BLOCK_SIZE, 3396 .maxauthsize = SHA384_DIGEST_SIZE, 3397 }, 3398 .caam = { 3399 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3400 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3401 OP_ALG_AAI_HMAC_PRECOMP, 3402 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3403 .geniv = true, 3404 }, 3405 }, 3406 { 3407 .aead = { 3408 .base = { 3409 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3410 .cra_driver_name = "authenc-hmac-sha512-" 3411 "cbc-aes-caam", 3412 .cra_blocksize = AES_BLOCK_SIZE, 3413 }, 3414 .setkey = aead_setkey, 3415 .setauthsize = aead_setauthsize, 3416 .encrypt = aead_encrypt, 3417 .decrypt = aead_decrypt, 3418 .ivsize = AES_BLOCK_SIZE, 3419 .maxauthsize = SHA512_DIGEST_SIZE, 3420 }, 3421 .caam = { 3422 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3423 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3424 OP_ALG_AAI_HMAC_PRECOMP, 3425 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3426 }, 3427 }, 3428 { 3429 .aead = { 3430 .base = { 3431 .cra_name = "echainiv(authenc(hmac(sha512)," 3432 "cbc(aes)))", 3433 .cra_driver_name = "echainiv-authenc-" 3434 "hmac-sha512-cbc-aes-caam", 3435 .cra_blocksize = AES_BLOCK_SIZE, 3436 }, 3437 .setkey = aead_setkey, 3438 .setauthsize = aead_setauthsize, 3439 .encrypt = aead_encrypt, 3440 .decrypt = aead_givdecrypt, 3441 .ivsize = AES_BLOCK_SIZE, 3442 .maxauthsize = SHA512_DIGEST_SIZE, 3443 }, 3444 .caam = { 3445 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3446 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3447 OP_ALG_AAI_HMAC_PRECOMP, 3448 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3449 .geniv = true, 3450 }, 3451 }, 3452 { 3453 .aead = { 3454 .base = { 3455 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 3456 .cra_driver_name = "authenc-hmac-md5-" 3457 "cbc-des3_ede-caam", 3458 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3459 }, 3460 .setkey = aead_setkey, 3461 .setauthsize = aead_setauthsize, 3462 .encrypt = aead_encrypt, 3463 .decrypt = aead_decrypt, 3464 .ivsize = DES3_EDE_BLOCK_SIZE, 3465 .maxauthsize = MD5_DIGEST_SIZE, 3466 }, 3467 .caam = { 3468 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3469 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3470 OP_ALG_AAI_HMAC_PRECOMP, 3471 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3472 } 3473 }, 3474 { 3475 .aead = { 3476 .base = { 3477 .cra_name = "echainiv(authenc(hmac(md5)," 3478 "cbc(des3_ede)))", 3479 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3480 "cbc-des3_ede-caam", 3481 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3482 }, 3483 .setkey = aead_setkey, 3484 .setauthsize = aead_setauthsize, 3485 .encrypt = aead_encrypt, 3486 .decrypt = aead_givdecrypt, 3487 .ivsize = DES3_EDE_BLOCK_SIZE, 3488 .maxauthsize = MD5_DIGEST_SIZE, 3489 }, 3490 .caam = { 3491 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3492 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3493 OP_ALG_AAI_HMAC_PRECOMP, 3494 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3495 .geniv = true, 3496 } 3497 }, 3498 { 3499 .aead = { 3500 .base = { 3501 .cra_name = "authenc(hmac(sha1)," 3502 "cbc(des3_ede))", 3503 .cra_driver_name = "authenc-hmac-sha1-" 3504 "cbc-des3_ede-caam", 3505 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3506 }, 3507 .setkey = aead_setkey, 3508 .setauthsize = aead_setauthsize, 3509 .encrypt = aead_encrypt, 3510 .decrypt = aead_decrypt, 3511 .ivsize = DES3_EDE_BLOCK_SIZE, 3512 .maxauthsize = SHA1_DIGEST_SIZE, 3513 }, 3514 .caam = { 3515 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3516 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3517 OP_ALG_AAI_HMAC_PRECOMP, 3518 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3519 }, 3520 }, 3521 { 3522 .aead = { 3523 .base = { 3524 .cra_name = "echainiv(authenc(hmac(sha1)," 3525 "cbc(des3_ede)))", 3526 .cra_driver_name = "echainiv-authenc-" 3527 "hmac-sha1-" 3528 "cbc-des3_ede-caam", 3529 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3530 }, 3531 .setkey = aead_setkey, 3532 .setauthsize = aead_setauthsize, 3533 .encrypt = aead_encrypt, 3534 .decrypt = aead_givdecrypt, 3535 .ivsize = DES3_EDE_BLOCK_SIZE, 3536 .maxauthsize = SHA1_DIGEST_SIZE, 3537 }, 3538 .caam = { 3539 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3540 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3541 OP_ALG_AAI_HMAC_PRECOMP, 3542 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3543 .geniv = true, 3544 }, 3545 }, 3546 { 3547 .aead = { 3548 .base = { 3549 .cra_name = "authenc(hmac(sha224)," 3550 "cbc(des3_ede))", 3551 .cra_driver_name = "authenc-hmac-sha224-" 3552 "cbc-des3_ede-caam", 3553 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3554 }, 3555 .setkey = aead_setkey, 3556 .setauthsize = aead_setauthsize, 3557 .encrypt = aead_encrypt, 3558 .decrypt = aead_decrypt, 3559 .ivsize = DES3_EDE_BLOCK_SIZE, 3560 .maxauthsize = SHA224_DIGEST_SIZE, 3561 }, 3562 .caam = { 3563 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3564 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3565 OP_ALG_AAI_HMAC_PRECOMP, 3566 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3567 }, 3568 }, 3569 { 3570 .aead = { 3571 .base = { 3572 .cra_name = "echainiv(authenc(hmac(sha224)," 3573 "cbc(des3_ede)))", 3574 .cra_driver_name = "echainiv-authenc-" 3575 "hmac-sha224-" 3576 "cbc-des3_ede-caam", 3577 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3578 }, 3579 .setkey = aead_setkey, 3580 .setauthsize = aead_setauthsize, 3581 .encrypt = aead_encrypt, 3582 .decrypt = aead_givdecrypt, 3583 .ivsize = DES3_EDE_BLOCK_SIZE, 3584 .maxauthsize = SHA224_DIGEST_SIZE, 3585 }, 3586 .caam = { 3587 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3588 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3589 OP_ALG_AAI_HMAC_PRECOMP, 3590 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3591 .geniv = true, 3592 }, 3593 }, 3594 { 3595 .aead = { 3596 .base = { 3597 .cra_name = "authenc(hmac(sha256)," 3598 "cbc(des3_ede))", 3599 .cra_driver_name = "authenc-hmac-sha256-" 3600 "cbc-des3_ede-caam", 3601 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3602 }, 3603 .setkey = aead_setkey, 3604 .setauthsize = aead_setauthsize, 3605 .encrypt = aead_encrypt, 3606 .decrypt = aead_decrypt, 3607 .ivsize = DES3_EDE_BLOCK_SIZE, 3608 .maxauthsize = SHA256_DIGEST_SIZE, 3609 }, 3610 .caam = { 3611 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3612 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3613 OP_ALG_AAI_HMAC_PRECOMP, 3614 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3615 }, 3616 }, 3617 { 3618 .aead = { 3619 .base = { 3620 .cra_name = "echainiv(authenc(hmac(sha256)," 3621 "cbc(des3_ede)))", 3622 .cra_driver_name = "echainiv-authenc-" 3623 "hmac-sha256-" 3624 "cbc-des3_ede-caam", 3625 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3626 }, 3627 .setkey = aead_setkey, 3628 .setauthsize = aead_setauthsize, 3629 .encrypt = aead_encrypt, 3630 .decrypt = aead_givdecrypt, 3631 .ivsize = DES3_EDE_BLOCK_SIZE, 3632 .maxauthsize = SHA256_DIGEST_SIZE, 3633 }, 3634 .caam = { 3635 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3636 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3637 OP_ALG_AAI_HMAC_PRECOMP, 3638 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3639 .geniv = true, 3640 }, 3641 }, 3642 { 3643 .aead = { 3644 .base = { 3645 .cra_name = "authenc(hmac(sha384)," 3646 "cbc(des3_ede))", 3647 .cra_driver_name = "authenc-hmac-sha384-" 3648 "cbc-des3_ede-caam", 3649 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3650 }, 3651 .setkey = aead_setkey, 3652 .setauthsize = aead_setauthsize, 3653 .encrypt = aead_encrypt, 3654 .decrypt = aead_decrypt, 3655 .ivsize = DES3_EDE_BLOCK_SIZE, 3656 .maxauthsize = SHA384_DIGEST_SIZE, 3657 }, 3658 .caam = { 3659 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3660 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3661 OP_ALG_AAI_HMAC_PRECOMP, 3662 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3663 }, 3664 }, 3665 { 3666 .aead = { 3667 .base = { 3668 .cra_name = "echainiv(authenc(hmac(sha384)," 3669 "cbc(des3_ede)))", 3670 .cra_driver_name = "echainiv-authenc-" 3671 "hmac-sha384-" 3672 "cbc-des3_ede-caam", 3673 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3674 }, 3675 .setkey = aead_setkey, 3676 .setauthsize = aead_setauthsize, 3677 .encrypt = aead_encrypt, 3678 .decrypt = aead_givdecrypt, 3679 .ivsize = DES3_EDE_BLOCK_SIZE, 3680 .maxauthsize = SHA384_DIGEST_SIZE, 3681 }, 3682 .caam = { 3683 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3684 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3685 OP_ALG_AAI_HMAC_PRECOMP, 3686 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3687 .geniv = true, 3688 }, 3689 }, 3690 { 3691 .aead = { 3692 .base = { 3693 .cra_name = "authenc(hmac(sha512)," 3694 "cbc(des3_ede))", 3695 .cra_driver_name = "authenc-hmac-sha512-" 3696 "cbc-des3_ede-caam", 3697 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3698 }, 3699 .setkey = aead_setkey, 3700 .setauthsize = aead_setauthsize, 3701 .encrypt = aead_encrypt, 3702 .decrypt = aead_decrypt, 3703 .ivsize = DES3_EDE_BLOCK_SIZE, 3704 .maxauthsize = SHA512_DIGEST_SIZE, 3705 }, 3706 .caam = { 3707 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3708 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3709 OP_ALG_AAI_HMAC_PRECOMP, 3710 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3711 }, 3712 }, 3713 { 3714 .aead = { 3715 .base = { 3716 .cra_name = "echainiv(authenc(hmac(sha512)," 3717 "cbc(des3_ede)))", 3718 .cra_driver_name = "echainiv-authenc-" 3719 "hmac-sha512-" 3720 "cbc-des3_ede-caam", 3721 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3722 }, 3723 .setkey = aead_setkey, 3724 .setauthsize = aead_setauthsize, 3725 .encrypt = aead_encrypt, 3726 .decrypt = aead_givdecrypt, 3727 .ivsize = DES3_EDE_BLOCK_SIZE, 3728 .maxauthsize = SHA512_DIGEST_SIZE, 3729 }, 3730 .caam = { 3731 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3732 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3733 OP_ALG_AAI_HMAC_PRECOMP, 3734 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3735 .geniv = true, 3736 }, 3737 }, 3738 { 3739 .aead = { 3740 .base = { 3741 .cra_name = "authenc(hmac(md5),cbc(des))", 3742 .cra_driver_name = "authenc-hmac-md5-" 3743 "cbc-des-caam", 3744 .cra_blocksize = DES_BLOCK_SIZE, 3745 }, 3746 .setkey = aead_setkey, 3747 .setauthsize = aead_setauthsize, 3748 .encrypt = aead_encrypt, 3749 .decrypt = aead_decrypt, 3750 .ivsize = DES_BLOCK_SIZE, 3751 .maxauthsize = MD5_DIGEST_SIZE, 3752 }, 3753 .caam = { 3754 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3755 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3756 OP_ALG_AAI_HMAC_PRECOMP, 3757 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3758 }, 3759 }, 3760 { 3761 .aead = { 3762 .base = { 3763 .cra_name = "echainiv(authenc(hmac(md5)," 3764 "cbc(des)))", 3765 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3766 "cbc-des-caam", 3767 .cra_blocksize = DES_BLOCK_SIZE, 3768 }, 3769 .setkey = aead_setkey, 3770 .setauthsize = aead_setauthsize, 3771 .encrypt = aead_encrypt, 3772 .decrypt = aead_givdecrypt, 3773 .ivsize = DES_BLOCK_SIZE, 3774 .maxauthsize = MD5_DIGEST_SIZE, 3775 }, 3776 .caam = { 3777 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3778 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3779 OP_ALG_AAI_HMAC_PRECOMP, 3780 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3781 .geniv = true, 3782 }, 3783 }, 3784 { 3785 .aead = { 3786 .base = { 3787 .cra_name = "authenc(hmac(sha1),cbc(des))", 3788 .cra_driver_name = "authenc-hmac-sha1-" 3789 "cbc-des-caam", 3790 .cra_blocksize = DES_BLOCK_SIZE, 3791 }, 3792 .setkey = aead_setkey, 3793 .setauthsize = aead_setauthsize, 3794 .encrypt = aead_encrypt, 3795 .decrypt = aead_decrypt, 3796 .ivsize = DES_BLOCK_SIZE, 3797 .maxauthsize = SHA1_DIGEST_SIZE, 3798 }, 3799 .caam = { 3800 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3801 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3802 OP_ALG_AAI_HMAC_PRECOMP, 3803 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3804 }, 3805 }, 3806 { 3807 .aead = { 3808 .base = { 3809 .cra_name = "echainiv(authenc(hmac(sha1)," 3810 "cbc(des)))", 3811 .cra_driver_name = "echainiv-authenc-" 3812 "hmac-sha1-cbc-des-caam", 3813 .cra_blocksize = DES_BLOCK_SIZE, 3814 }, 3815 .setkey = aead_setkey, 3816 .setauthsize = aead_setauthsize, 3817 .encrypt = aead_encrypt, 3818 .decrypt = aead_givdecrypt, 3819 .ivsize = DES_BLOCK_SIZE, 3820 .maxauthsize = SHA1_DIGEST_SIZE, 3821 }, 3822 .caam = { 3823 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3824 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3825 OP_ALG_AAI_HMAC_PRECOMP, 3826 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3827 .geniv = true, 3828 }, 3829 }, 3830 { 3831 .aead = { 3832 .base = { 3833 .cra_name = "authenc(hmac(sha224),cbc(des))", 3834 .cra_driver_name = "authenc-hmac-sha224-" 3835 "cbc-des-caam", 3836 .cra_blocksize = DES_BLOCK_SIZE, 3837 }, 3838 .setkey = aead_setkey, 3839 .setauthsize = aead_setauthsize, 3840 .encrypt = aead_encrypt, 3841 .decrypt = aead_decrypt, 3842 .ivsize = DES_BLOCK_SIZE, 3843 .maxauthsize = SHA224_DIGEST_SIZE, 3844 }, 3845 .caam = { 3846 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3847 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3848 OP_ALG_AAI_HMAC_PRECOMP, 3849 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3850 }, 3851 }, 3852 { 3853 .aead = { 3854 .base = { 3855 .cra_name = "echainiv(authenc(hmac(sha224)," 3856 "cbc(des)))", 3857 .cra_driver_name = "echainiv-authenc-" 3858 "hmac-sha224-cbc-des-caam", 3859 .cra_blocksize = DES_BLOCK_SIZE, 3860 }, 3861 .setkey = aead_setkey, 3862 .setauthsize = aead_setauthsize, 3863 .encrypt = aead_encrypt, 3864 .decrypt = aead_givdecrypt, 3865 .ivsize = DES_BLOCK_SIZE, 3866 .maxauthsize = SHA224_DIGEST_SIZE, 3867 }, 3868 .caam = { 3869 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3870 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3871 OP_ALG_AAI_HMAC_PRECOMP, 3872 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3873 .geniv = true, 3874 }, 3875 }, 3876 { 3877 .aead = { 3878 .base = { 3879 .cra_name = "authenc(hmac(sha256),cbc(des))", 3880 .cra_driver_name = "authenc-hmac-sha256-" 3881 "cbc-des-caam", 3882 .cra_blocksize = DES_BLOCK_SIZE, 3883 }, 3884 .setkey = aead_setkey, 3885 .setauthsize = aead_setauthsize, 3886 .encrypt = aead_encrypt, 3887 .decrypt = aead_decrypt, 3888 .ivsize = DES_BLOCK_SIZE, 3889 .maxauthsize = SHA256_DIGEST_SIZE, 3890 }, 3891 .caam = { 3892 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3893 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3894 OP_ALG_AAI_HMAC_PRECOMP, 3895 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3896 }, 3897 }, 3898 { 3899 .aead = { 3900 .base = { 3901 .cra_name = "echainiv(authenc(hmac(sha256)," 3902 "cbc(des)))", 3903 .cra_driver_name = "echainiv-authenc-" 3904 "hmac-sha256-cbc-des-caam", 3905 .cra_blocksize = DES_BLOCK_SIZE, 3906 }, 3907 .setkey = aead_setkey, 3908 .setauthsize = aead_setauthsize, 3909 .encrypt = aead_encrypt, 3910 .decrypt = aead_givdecrypt, 3911 .ivsize = DES_BLOCK_SIZE, 3912 .maxauthsize = SHA256_DIGEST_SIZE, 3913 }, 3914 .caam = { 3915 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3916 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3917 OP_ALG_AAI_HMAC_PRECOMP, 3918 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3919 .geniv = true, 3920 }, 3921 }, 3922 { 3923 .aead = { 3924 .base = { 3925 .cra_name = "authenc(hmac(sha384),cbc(des))", 3926 .cra_driver_name = "authenc-hmac-sha384-" 3927 "cbc-des-caam", 3928 .cra_blocksize = DES_BLOCK_SIZE, 3929 }, 3930 .setkey = aead_setkey, 3931 .setauthsize = aead_setauthsize, 3932 .encrypt = aead_encrypt, 3933 .decrypt = aead_decrypt, 3934 .ivsize = DES_BLOCK_SIZE, 3935 .maxauthsize = SHA384_DIGEST_SIZE, 3936 }, 3937 .caam = { 3938 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3939 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3940 OP_ALG_AAI_HMAC_PRECOMP, 3941 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3942 }, 3943 }, 3944 { 3945 .aead = { 3946 .base = { 3947 .cra_name = "echainiv(authenc(hmac(sha384)," 3948 "cbc(des)))", 3949 .cra_driver_name = "echainiv-authenc-" 3950 "hmac-sha384-cbc-des-caam", 3951 .cra_blocksize = DES_BLOCK_SIZE, 3952 }, 3953 .setkey = aead_setkey, 3954 .setauthsize = aead_setauthsize, 3955 .encrypt = aead_encrypt, 3956 .decrypt = aead_givdecrypt, 3957 .ivsize = DES_BLOCK_SIZE, 3958 .maxauthsize = SHA384_DIGEST_SIZE, 3959 }, 3960 .caam = { 3961 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3962 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3963 OP_ALG_AAI_HMAC_PRECOMP, 3964 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3965 .geniv = true, 3966 }, 3967 }, 3968 { 3969 .aead = { 3970 .base = { 3971 .cra_name = "authenc(hmac(sha512),cbc(des))", 3972 .cra_driver_name = "authenc-hmac-sha512-" 3973 "cbc-des-caam", 3974 .cra_blocksize = DES_BLOCK_SIZE, 3975 }, 3976 .setkey = aead_setkey, 3977 .setauthsize = aead_setauthsize, 3978 .encrypt = aead_encrypt, 3979 .decrypt = aead_decrypt, 3980 .ivsize = DES_BLOCK_SIZE, 3981 .maxauthsize = SHA512_DIGEST_SIZE, 3982 }, 3983 .caam = { 3984 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3985 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3986 OP_ALG_AAI_HMAC_PRECOMP, 3987 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3988 }, 3989 }, 3990 { 3991 .aead = { 3992 .base = { 3993 .cra_name = "echainiv(authenc(hmac(sha512)," 3994 "cbc(des)))", 3995 .cra_driver_name = "echainiv-authenc-" 3996 "hmac-sha512-cbc-des-caam", 3997 .cra_blocksize = DES_BLOCK_SIZE, 3998 }, 3999 .setkey = aead_setkey, 4000 .setauthsize = aead_setauthsize, 4001 .encrypt = aead_encrypt, 4002 .decrypt = aead_givdecrypt, 4003 .ivsize = DES_BLOCK_SIZE, 4004 .maxauthsize = SHA512_DIGEST_SIZE, 4005 }, 4006 .caam = { 4007 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 4008 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4009 OP_ALG_AAI_HMAC_PRECOMP, 4010 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4011 .geniv = true, 4012 }, 4013 }, 4014 { 4015 .aead = { 4016 .base = { 4017 .cra_name = "authenc(hmac(md5)," 4018 "rfc3686(ctr(aes)))", 4019 .cra_driver_name = "authenc-hmac-md5-" 4020 "rfc3686-ctr-aes-caam", 4021 .cra_blocksize = 1, 4022 }, 4023 .setkey = aead_setkey, 4024 .setauthsize = aead_setauthsize, 4025 .encrypt = aead_encrypt, 4026 .decrypt = aead_decrypt, 4027 .ivsize = CTR_RFC3686_IV_SIZE, 4028 .maxauthsize = MD5_DIGEST_SIZE, 4029 }, 4030 .caam = { 4031 .class1_alg_type = OP_ALG_ALGSEL_AES | 4032 OP_ALG_AAI_CTR_MOD128, 4033 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 4034 OP_ALG_AAI_HMAC_PRECOMP, 4035 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 4036 .rfc3686 = true, 4037 }, 4038 }, 4039 { 4040 .aead = { 4041 .base = { 4042 .cra_name = "seqiv(authenc(" 4043 "hmac(md5),rfc3686(ctr(aes))))", 4044 .cra_driver_name = "seqiv-authenc-hmac-md5-" 4045 "rfc3686-ctr-aes-caam", 4046 .cra_blocksize = 1, 4047 }, 4048 .setkey = aead_setkey, 4049 .setauthsize = aead_setauthsize, 4050 .encrypt = aead_encrypt, 4051 .decrypt = aead_givdecrypt, 4052 .ivsize = CTR_RFC3686_IV_SIZE, 4053 .maxauthsize = MD5_DIGEST_SIZE, 4054 }, 4055 .caam = { 4056 .class1_alg_type = OP_ALG_ALGSEL_AES | 4057 OP_ALG_AAI_CTR_MOD128, 4058 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 4059 OP_ALG_AAI_HMAC_PRECOMP, 4060 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 4061 .rfc3686 = true, 4062 .geniv = true, 4063 }, 4064 }, 4065 { 4066 .aead = { 4067 .base = { 4068 .cra_name = "authenc(hmac(sha1)," 4069 "rfc3686(ctr(aes)))", 4070 .cra_driver_name = "authenc-hmac-sha1-" 4071 "rfc3686-ctr-aes-caam", 4072 .cra_blocksize = 1, 4073 }, 4074 .setkey = aead_setkey, 4075 .setauthsize = aead_setauthsize, 4076 .encrypt = aead_encrypt, 4077 .decrypt = aead_decrypt, 4078 .ivsize = CTR_RFC3686_IV_SIZE, 4079 .maxauthsize = SHA1_DIGEST_SIZE, 4080 }, 4081 .caam = { 4082 .class1_alg_type = OP_ALG_ALGSEL_AES | 4083 OP_ALG_AAI_CTR_MOD128, 4084 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4085 OP_ALG_AAI_HMAC_PRECOMP, 4086 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4087 .rfc3686 = true, 4088 }, 4089 }, 4090 { 4091 .aead = { 4092 .base = { 4093 .cra_name = "seqiv(authenc(" 4094 "hmac(sha1),rfc3686(ctr(aes))))", 4095 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 4096 "rfc3686-ctr-aes-caam", 4097 .cra_blocksize = 1, 4098 }, 4099 .setkey = aead_setkey, 4100 .setauthsize = aead_setauthsize, 4101 .encrypt = aead_encrypt, 4102 .decrypt = aead_givdecrypt, 4103 .ivsize = CTR_RFC3686_IV_SIZE, 4104 .maxauthsize = SHA1_DIGEST_SIZE, 4105 }, 4106 .caam = { 4107 .class1_alg_type = OP_ALG_ALGSEL_AES | 4108 OP_ALG_AAI_CTR_MOD128, 4109 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4110 OP_ALG_AAI_HMAC_PRECOMP, 4111 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4112 .rfc3686 = true, 4113 .geniv = true, 4114 }, 4115 }, 4116 { 4117 .aead = { 4118 .base = { 4119 .cra_name = "authenc(hmac(sha224)," 4120 "rfc3686(ctr(aes)))", 4121 .cra_driver_name = "authenc-hmac-sha224-" 4122 "rfc3686-ctr-aes-caam", 4123 .cra_blocksize = 1, 4124 }, 4125 .setkey = aead_setkey, 4126 .setauthsize = aead_setauthsize, 4127 .encrypt = aead_encrypt, 4128 .decrypt = aead_decrypt, 4129 .ivsize = CTR_RFC3686_IV_SIZE, 4130 .maxauthsize = SHA224_DIGEST_SIZE, 4131 }, 4132 .caam = { 4133 .class1_alg_type = OP_ALG_ALGSEL_AES | 4134 OP_ALG_AAI_CTR_MOD128, 4135 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4136 OP_ALG_AAI_HMAC_PRECOMP, 4137 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4138 .rfc3686 = true, 4139 }, 4140 }, 4141 { 4142 .aead = { 4143 .base = { 4144 .cra_name = "seqiv(authenc(" 4145 "hmac(sha224),rfc3686(ctr(aes))))", 4146 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 4147 "rfc3686-ctr-aes-caam", 4148 .cra_blocksize = 1, 4149 }, 4150 .setkey = aead_setkey, 4151 .setauthsize = aead_setauthsize, 4152 .encrypt = aead_encrypt, 4153 .decrypt = aead_givdecrypt, 4154 .ivsize = CTR_RFC3686_IV_SIZE, 4155 .maxauthsize = SHA224_DIGEST_SIZE, 4156 }, 4157 .caam = { 4158 .class1_alg_type = OP_ALG_ALGSEL_AES | 4159 OP_ALG_AAI_CTR_MOD128, 4160 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4161 OP_ALG_AAI_HMAC_PRECOMP, 4162 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4163 .rfc3686 = true, 4164 .geniv = true, 4165 }, 4166 }, 4167 { 4168 .aead = { 4169 .base = { 4170 .cra_name = "authenc(hmac(sha256)," 4171 "rfc3686(ctr(aes)))", 4172 .cra_driver_name = "authenc-hmac-sha256-" 4173 "rfc3686-ctr-aes-caam", 4174 .cra_blocksize = 1, 4175 }, 4176 .setkey = aead_setkey, 4177 .setauthsize = aead_setauthsize, 4178 .encrypt = aead_encrypt, 4179 .decrypt = aead_decrypt, 4180 .ivsize = CTR_RFC3686_IV_SIZE, 4181 .maxauthsize = SHA256_DIGEST_SIZE, 4182 }, 4183 .caam = { 4184 .class1_alg_type = OP_ALG_ALGSEL_AES | 4185 OP_ALG_AAI_CTR_MOD128, 4186 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4187 OP_ALG_AAI_HMAC_PRECOMP, 4188 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4189 .rfc3686 = true, 4190 }, 4191 }, 4192 { 4193 .aead = { 4194 .base = { 4195 .cra_name = "seqiv(authenc(hmac(sha256)," 4196 "rfc3686(ctr(aes))))", 4197 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 4198 "rfc3686-ctr-aes-caam", 4199 .cra_blocksize = 1, 4200 }, 4201 .setkey = aead_setkey, 4202 .setauthsize = aead_setauthsize, 4203 .encrypt = aead_encrypt, 4204 .decrypt = aead_givdecrypt, 4205 .ivsize = CTR_RFC3686_IV_SIZE, 4206 .maxauthsize = SHA256_DIGEST_SIZE, 4207 }, 4208 .caam = { 4209 .class1_alg_type = OP_ALG_ALGSEL_AES | 4210 OP_ALG_AAI_CTR_MOD128, 4211 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4212 OP_ALG_AAI_HMAC_PRECOMP, 4213 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4214 .rfc3686 = true, 4215 .geniv = true, 4216 }, 4217 }, 4218 { 4219 .aead = { 4220 .base = { 4221 .cra_name = "authenc(hmac(sha384)," 4222 "rfc3686(ctr(aes)))", 4223 .cra_driver_name = "authenc-hmac-sha384-" 4224 "rfc3686-ctr-aes-caam", 4225 .cra_blocksize = 1, 4226 }, 4227 .setkey = aead_setkey, 4228 .setauthsize = aead_setauthsize, 4229 .encrypt = aead_encrypt, 4230 .decrypt = aead_decrypt, 4231 .ivsize = CTR_RFC3686_IV_SIZE, 4232 .maxauthsize = SHA384_DIGEST_SIZE, 4233 }, 4234 .caam = { 4235 .class1_alg_type = OP_ALG_ALGSEL_AES | 4236 OP_ALG_AAI_CTR_MOD128, 4237 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4238 OP_ALG_AAI_HMAC_PRECOMP, 4239 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4240 .rfc3686 = true, 4241 }, 4242 }, 4243 { 4244 .aead = { 4245 .base = { 4246 .cra_name = "seqiv(authenc(hmac(sha384)," 4247 "rfc3686(ctr(aes))))", 4248 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 4249 "rfc3686-ctr-aes-caam", 4250 .cra_blocksize = 1, 4251 }, 4252 .setkey = aead_setkey, 4253 .setauthsize = aead_setauthsize, 4254 .encrypt = aead_encrypt, 4255 .decrypt = aead_givdecrypt, 4256 .ivsize = CTR_RFC3686_IV_SIZE, 4257 .maxauthsize = SHA384_DIGEST_SIZE, 4258 }, 4259 .caam = { 4260 .class1_alg_type = OP_ALG_ALGSEL_AES | 4261 OP_ALG_AAI_CTR_MOD128, 4262 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4263 OP_ALG_AAI_HMAC_PRECOMP, 4264 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4265 .rfc3686 = true, 4266 .geniv = true, 4267 }, 4268 }, 4269 { 4270 .aead = { 4271 .base = { 4272 .cra_name = "authenc(hmac(sha512)," 4273 "rfc3686(ctr(aes)))", 4274 .cra_driver_name = "authenc-hmac-sha512-" 4275 "rfc3686-ctr-aes-caam", 4276 .cra_blocksize = 1, 4277 }, 4278 .setkey = aead_setkey, 4279 .setauthsize = aead_setauthsize, 4280 .encrypt = aead_encrypt, 4281 .decrypt = aead_decrypt, 4282 .ivsize = CTR_RFC3686_IV_SIZE, 4283 .maxauthsize = SHA512_DIGEST_SIZE, 4284 }, 4285 .caam = { 4286 .class1_alg_type = OP_ALG_ALGSEL_AES | 4287 OP_ALG_AAI_CTR_MOD128, 4288 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4289 OP_ALG_AAI_HMAC_PRECOMP, 4290 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4291 .rfc3686 = true, 4292 }, 4293 }, 4294 { 4295 .aead = { 4296 .base = { 4297 .cra_name = "seqiv(authenc(hmac(sha512)," 4298 "rfc3686(ctr(aes))))", 4299 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 4300 "rfc3686-ctr-aes-caam", 4301 .cra_blocksize = 1, 4302 }, 4303 .setkey = aead_setkey, 4304 .setauthsize = aead_setauthsize, 4305 .encrypt = aead_encrypt, 4306 .decrypt = aead_givdecrypt, 4307 .ivsize = CTR_RFC3686_IV_SIZE, 4308 .maxauthsize = SHA512_DIGEST_SIZE, 4309 }, 4310 .caam = { 4311 .class1_alg_type = OP_ALG_ALGSEL_AES | 4312 OP_ALG_AAI_CTR_MOD128, 4313 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4314 OP_ALG_AAI_HMAC_PRECOMP, 4315 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4316 .rfc3686 = true, 4317 .geniv = true, 4318 }, 4319 }, 4320 }; 4321 4322 struct caam_crypto_alg { 4323 struct crypto_alg crypto_alg; 4324 struct list_head entry; 4325 struct caam_alg_entry caam; 4326 }; 4327 4328 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 4329 { 4330 ctx->jrdev = caam_jr_alloc(); 4331 if (IS_ERR(ctx->jrdev)) { 4332 pr_err("Job Ring Device allocation for transform failed\n"); 4333 return PTR_ERR(ctx->jrdev); 4334 } 4335 4336 /* copy descriptor header template value */ 4337 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 4338 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 4339 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; 4340 4341 return 0; 4342 } 4343 4344 static int caam_cra_init(struct crypto_tfm *tfm) 4345 { 4346 struct crypto_alg *alg = tfm->__crt_alg; 4347 struct caam_crypto_alg *caam_alg = 4348 container_of(alg, struct caam_crypto_alg, crypto_alg); 4349 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 4350 4351 return caam_init_common(ctx, &caam_alg->caam); 4352 } 4353 4354 static int caam_aead_init(struct crypto_aead *tfm) 4355 { 4356 struct aead_alg *alg = crypto_aead_alg(tfm); 4357 struct caam_aead_alg *caam_alg = 4358 container_of(alg, struct caam_aead_alg, aead); 4359 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 4360 4361 return caam_init_common(ctx, &caam_alg->caam); 4362 } 4363 4364 static void caam_exit_common(struct caam_ctx *ctx) 4365 { 4366 if (ctx->sh_desc_enc_dma && 4367 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 4368 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 4369 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 4370 if (ctx->sh_desc_dec_dma && 4371 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 4372 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 4373 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 4374 if (ctx->sh_desc_givenc_dma && 4375 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 4376 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 4377 desc_bytes(ctx->sh_desc_givenc), 4378 DMA_TO_DEVICE); 4379 if (ctx->key_dma && 4380 !dma_mapping_error(ctx->jrdev, ctx->key_dma)) 4381 dma_unmap_single(ctx->jrdev, ctx->key_dma, 4382 ctx->enckeylen + ctx->split_key_pad_len, 4383 DMA_TO_DEVICE); 4384 4385 caam_jr_free(ctx->jrdev); 4386 } 4387 4388 static void caam_cra_exit(struct crypto_tfm *tfm) 4389 { 4390 caam_exit_common(crypto_tfm_ctx(tfm)); 4391 } 4392 4393 static void caam_aead_exit(struct crypto_aead *tfm) 4394 { 4395 caam_exit_common(crypto_aead_ctx(tfm)); 4396 } 4397 4398 static void __exit caam_algapi_exit(void) 4399 { 4400 4401 struct caam_crypto_alg *t_alg, *n; 4402 int i; 4403 4404 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4405 struct caam_aead_alg *t_alg = driver_aeads + i; 4406 4407 if (t_alg->registered) 4408 crypto_unregister_aead(&t_alg->aead); 4409 } 4410 4411 if (!alg_list.next) 4412 return; 4413 4414 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 4415 crypto_unregister_alg(&t_alg->crypto_alg); 4416 list_del(&t_alg->entry); 4417 kfree(t_alg); 4418 } 4419 } 4420 4421 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 4422 *template) 4423 { 4424 struct caam_crypto_alg *t_alg; 4425 struct crypto_alg *alg; 4426 4427 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4428 if (!t_alg) { 4429 pr_err("failed to allocate t_alg\n"); 4430 return ERR_PTR(-ENOMEM); 4431 } 4432 4433 alg = &t_alg->crypto_alg; 4434 4435 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 4436 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4437 template->driver_name); 4438 alg->cra_module = THIS_MODULE; 4439 alg->cra_init = caam_cra_init; 4440 alg->cra_exit = caam_cra_exit; 4441 alg->cra_priority = CAAM_CRA_PRIORITY; 4442 alg->cra_blocksize = template->blocksize; 4443 alg->cra_alignmask = 0; 4444 alg->cra_ctxsize = sizeof(struct caam_ctx); 4445 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 4446 template->type; 4447 switch (template->type) { 4448 case CRYPTO_ALG_TYPE_GIVCIPHER: 4449 alg->cra_type = &crypto_givcipher_type; 4450 alg->cra_ablkcipher = template->template_ablkcipher; 4451 break; 4452 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4453 alg->cra_type = &crypto_ablkcipher_type; 4454 alg->cra_ablkcipher = template->template_ablkcipher; 4455 break; 4456 } 4457 4458 t_alg->caam.class1_alg_type = template->class1_alg_type; 4459 t_alg->caam.class2_alg_type = template->class2_alg_type; 4460 t_alg->caam.alg_op = template->alg_op; 4461 4462 return t_alg; 4463 } 4464 4465 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 4466 { 4467 struct aead_alg *alg = &t_alg->aead; 4468 4469 alg->base.cra_module = THIS_MODULE; 4470 alg->base.cra_priority = CAAM_CRA_PRIORITY; 4471 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 4472 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 4473 4474 alg->init = caam_aead_init; 4475 alg->exit = caam_aead_exit; 4476 } 4477 4478 static int __init caam_algapi_init(void) 4479 { 4480 struct device_node *dev_node; 4481 struct platform_device *pdev; 4482 struct device *ctrldev; 4483 struct caam_drv_private *priv; 4484 int i = 0, err = 0; 4485 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 4486 unsigned int md_limit = SHA512_DIGEST_SIZE; 4487 bool registered = false; 4488 4489 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 4490 if (!dev_node) { 4491 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 4492 if (!dev_node) 4493 return -ENODEV; 4494 } 4495 4496 pdev = of_find_device_by_node(dev_node); 4497 if (!pdev) { 4498 of_node_put(dev_node); 4499 return -ENODEV; 4500 } 4501 4502 ctrldev = &pdev->dev; 4503 priv = dev_get_drvdata(ctrldev); 4504 of_node_put(dev_node); 4505 4506 /* 4507 * If priv is NULL, it's probably because the caam driver wasn't 4508 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 4509 */ 4510 if (!priv) 4511 return -ENODEV; 4512 4513 4514 INIT_LIST_HEAD(&alg_list); 4515 4516 /* 4517 * Register crypto algorithms the device supports. 4518 * First, detect presence and attributes of DES, AES, and MD blocks. 4519 */ 4520 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 4521 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 4522 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 4523 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 4524 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 4525 4526 /* If MD is present, limit digest size based on LP256 */ 4527 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 4528 md_limit = SHA256_DIGEST_SIZE; 4529 4530 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4531 struct caam_crypto_alg *t_alg; 4532 struct caam_alg_template *alg = driver_algs + i; 4533 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 4534 4535 /* Skip DES algorithms if not supported by device */ 4536 if (!des_inst && 4537 ((alg_sel == OP_ALG_ALGSEL_3DES) || 4538 (alg_sel == OP_ALG_ALGSEL_DES))) 4539 continue; 4540 4541 /* Skip AES algorithms if not supported by device */ 4542 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 4543 continue; 4544 4545 t_alg = caam_alg_alloc(alg); 4546 if (IS_ERR(t_alg)) { 4547 err = PTR_ERR(t_alg); 4548 pr_warn("%s alg allocation failed\n", alg->driver_name); 4549 continue; 4550 } 4551 4552 err = crypto_register_alg(&t_alg->crypto_alg); 4553 if (err) { 4554 pr_warn("%s alg registration failed\n", 4555 t_alg->crypto_alg.cra_driver_name); 4556 kfree(t_alg); 4557 continue; 4558 } 4559 4560 list_add_tail(&t_alg->entry, &alg_list); 4561 registered = true; 4562 } 4563 4564 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4565 struct caam_aead_alg *t_alg = driver_aeads + i; 4566 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 4567 OP_ALG_ALGSEL_MASK; 4568 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 4569 OP_ALG_ALGSEL_MASK; 4570 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 4571 4572 /* Skip DES algorithms if not supported by device */ 4573 if (!des_inst && 4574 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 4575 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 4576 continue; 4577 4578 /* Skip AES algorithms if not supported by device */ 4579 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 4580 continue; 4581 4582 /* 4583 * Check support for AES algorithms not available 4584 * on LP devices. 4585 */ 4586 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 4587 if (alg_aai == OP_ALG_AAI_GCM) 4588 continue; 4589 4590 /* 4591 * Skip algorithms requiring message digests 4592 * if MD or MD size is not supported by device. 4593 */ 4594 if (c2_alg_sel && 4595 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 4596 continue; 4597 4598 caam_aead_alg_init(t_alg); 4599 4600 err = crypto_register_aead(&t_alg->aead); 4601 if (err) { 4602 pr_warn("%s alg registration failed\n", 4603 t_alg->aead.base.cra_driver_name); 4604 continue; 4605 } 4606 4607 t_alg->registered = true; 4608 registered = true; 4609 } 4610 4611 if (registered) 4612 pr_info("caam algorithms registered in /proc/crypto\n"); 4613 4614 return err; 4615 } 4616 4617 module_init(caam_algapi_init); 4618 module_exit(caam_algapi_exit); 4619 4620 MODULE_LICENSE("GPL"); 4621 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 4622 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 4623