1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/internal/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/des.h> 10 #include <linux/rtnetlink.h> 11 #include "cc_driver.h" 12 #include "cc_buffer_mgr.h" 13 #include "cc_aead.h" 14 #include "cc_request_mgr.h" 15 #include "cc_hash.h" 16 #include "cc_sram_mgr.h" 17 18 #define template_aead template_u.aead 19 20 #define MAX_AEAD_SETKEY_SEQ 12 21 #define MAX_AEAD_PROCESS_SEQ 23 22 23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 25 26 #define AES_CCM_RFC4309_NONCE_SIZE 3 27 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 28 29 /* Value of each ICV_CMP byte (of 8) in case of success */ 30 #define ICV_VERIF_OK 0x01 31 32 struct cc_aead_handle { 33 cc_sram_addr_t sram_workspace_addr; 34 struct list_head aead_list; 35 }; 36 37 struct cc_hmac_s { 38 u8 *padded_authkey; 39 u8 *ipad_opad; /* IPAD, OPAD*/ 40 dma_addr_t padded_authkey_dma_addr; 41 dma_addr_t ipad_opad_dma_addr; 42 }; 43 44 struct cc_xcbc_s { 45 u8 *xcbc_keys; /* K1,K2,K3 */ 46 dma_addr_t xcbc_keys_dma_addr; 47 }; 48 49 struct cc_aead_ctx { 50 struct cc_drvdata *drvdata; 51 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 52 u8 *enckey; 53 dma_addr_t enckey_dma_addr; 54 union { 55 struct cc_hmac_s hmac; 56 struct cc_xcbc_s xcbc; 57 } auth_state; 58 unsigned int enc_keylen; 59 unsigned int auth_keylen; 60 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 61 unsigned int hash_len; 62 enum drv_cipher_mode cipher_mode; 63 enum cc_flow_mode flow_mode; 64 enum drv_hash_mode auth_mode; 65 }; 66 67 static inline bool valid_assoclen(struct aead_request *req) 68 { 69 return ((req->assoclen == 16) || (req->assoclen == 20)); 70 } 71 72 static void cc_aead_exit(struct crypto_aead *tfm) 73 { 74 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 75 struct device *dev = drvdata_to_dev(ctx->drvdata); 76 77 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 78 crypto_tfm_alg_name(&tfm->base)); 79 80 /* Unmap enckey buffer */ 81 if (ctx->enckey) { 82 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 83 ctx->enckey_dma_addr); 84 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 85 &ctx->enckey_dma_addr); 86 ctx->enckey_dma_addr = 0; 87 ctx->enckey = NULL; 88 } 89 90 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 91 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 92 93 if (xcbc->xcbc_keys) { 94 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 95 xcbc->xcbc_keys, 96 xcbc->xcbc_keys_dma_addr); 97 } 98 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 99 &xcbc->xcbc_keys_dma_addr); 100 xcbc->xcbc_keys_dma_addr = 0; 101 xcbc->xcbc_keys = NULL; 102 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 103 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 104 105 if (hmac->ipad_opad) { 106 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 107 hmac->ipad_opad, 108 hmac->ipad_opad_dma_addr); 109 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 110 &hmac->ipad_opad_dma_addr); 111 hmac->ipad_opad_dma_addr = 0; 112 hmac->ipad_opad = NULL; 113 } 114 if (hmac->padded_authkey) { 115 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 116 hmac->padded_authkey, 117 hmac->padded_authkey_dma_addr); 118 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 119 &hmac->padded_authkey_dma_addr); 120 hmac->padded_authkey_dma_addr = 0; 121 hmac->padded_authkey = NULL; 122 } 123 } 124 } 125 126 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 127 { 128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 129 130 return cc_get_default_hash_len(ctx->drvdata); 131 } 132 133 static int cc_aead_init(struct crypto_aead *tfm) 134 { 135 struct aead_alg *alg = crypto_aead_alg(tfm); 136 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 137 struct cc_crypto_alg *cc_alg = 138 container_of(alg, struct cc_crypto_alg, aead_alg); 139 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 140 141 dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 142 crypto_tfm_alg_name(&tfm->base)); 143 144 /* Initialize modes in instance */ 145 ctx->cipher_mode = cc_alg->cipher_mode; 146 ctx->flow_mode = cc_alg->flow_mode; 147 ctx->auth_mode = cc_alg->auth_mode; 148 ctx->drvdata = cc_alg->drvdata; 149 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx)); 150 151 /* Allocate key buffer, cache line aligned */ 152 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 153 &ctx->enckey_dma_addr, GFP_KERNEL); 154 if (!ctx->enckey) { 155 dev_err(dev, "Failed allocating key buffer\n"); 156 goto init_failed; 157 } 158 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 159 ctx->enckey); 160 161 /* Set default authlen value */ 162 163 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 164 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 165 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 166 167 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 168 /* (and temporary for user key - up to 256b) */ 169 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 170 &xcbc->xcbc_keys_dma_addr, 171 GFP_KERNEL); 172 if (!xcbc->xcbc_keys) { 173 dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 174 goto init_failed; 175 } 176 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 177 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 178 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 179 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 180 181 /* Allocate dma-coherent buffer for IPAD + OPAD */ 182 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 183 &hmac->ipad_opad_dma_addr, 184 GFP_KERNEL); 185 186 if (!hmac->ipad_opad) { 187 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 188 goto init_failed; 189 } 190 191 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 192 hmac->ipad_opad); 193 194 hmac->padded_authkey = dma_alloc_coherent(dev, 195 MAX_HMAC_BLOCK_SIZE, 196 pkey_dma, 197 GFP_KERNEL); 198 199 if (!hmac->padded_authkey) { 200 dev_err(dev, "failed to allocate padded_authkey\n"); 201 goto init_failed; 202 } 203 } else { 204 ctx->auth_state.hmac.ipad_opad = NULL; 205 ctx->auth_state.hmac.padded_authkey = NULL; 206 } 207 ctx->hash_len = cc_get_aead_hash_len(tfm); 208 209 return 0; 210 211 init_failed: 212 cc_aead_exit(tfm); 213 return -ENOMEM; 214 } 215 216 static void cc_aead_complete(struct device *dev, void *cc_req, int err) 217 { 218 struct aead_request *areq = (struct aead_request *)cc_req; 219 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 220 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 221 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 222 223 cc_unmap_aead_request(dev, areq); 224 225 /* Restore ordinary iv pointer */ 226 areq->iv = areq_ctx->backup_iv; 227 228 if (err) 229 goto done; 230 231 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 232 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 233 ctx->authsize) != 0) { 234 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 235 ctx->authsize, ctx->cipher_mode); 236 /* In case of payload authentication failure, MUST NOT 237 * revealed the decrypted message --> zero its memory. 238 */ 239 cc_zero_sgl(areq->dst, areq_ctx->cryptlen); 240 err = -EBADMSG; 241 } 242 } else { /*ENCRYPT*/ 243 if (areq_ctx->is_icv_fragmented) { 244 u32 skip = areq->cryptlen + areq_ctx->dst_offset; 245 246 cc_copy_sg_portion(dev, areq_ctx->mac_buf, 247 areq_ctx->dst_sgl, skip, 248 (skip + ctx->authsize), 249 CC_SG_FROM_BUF); 250 } 251 252 /* If an IV was generated, copy it back to the user provided 253 * buffer. 254 */ 255 if (areq_ctx->backup_giv) { 256 if (ctx->cipher_mode == DRV_CIPHER_CTR) 257 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv + 258 CTR_RFC3686_NONCE_SIZE, 259 CTR_RFC3686_IV_SIZE); 260 else if (ctx->cipher_mode == DRV_CIPHER_CCM) 261 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv + 262 CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE); 263 } 264 } 265 done: 266 aead_request_complete(areq, err); 267 } 268 269 static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 270 struct cc_aead_ctx *ctx) 271 { 272 /* Load the AES key */ 273 hw_desc_init(&desc[0]); 274 /* We are using for the source/user key the same buffer 275 * as for the output keys, * because after this key loading it 276 * is not needed anymore 277 */ 278 set_din_type(&desc[0], DMA_DLLI, 279 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 280 NS_BIT); 281 set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 282 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 283 set_key_size_aes(&desc[0], ctx->auth_keylen); 284 set_flow_mode(&desc[0], S_DIN_to_AES); 285 set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 286 287 hw_desc_init(&desc[1]); 288 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 289 set_flow_mode(&desc[1], DIN_AES_DOUT); 290 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 291 AES_KEYSIZE_128, NS_BIT, 0); 292 293 hw_desc_init(&desc[2]); 294 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 295 set_flow_mode(&desc[2], DIN_AES_DOUT); 296 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 297 + AES_KEYSIZE_128), 298 AES_KEYSIZE_128, NS_BIT, 0); 299 300 hw_desc_init(&desc[3]); 301 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 302 set_flow_mode(&desc[3], DIN_AES_DOUT); 303 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 304 + 2 * AES_KEYSIZE_128), 305 AES_KEYSIZE_128, NS_BIT, 0); 306 307 return 4; 308 } 309 310 static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx) 311 { 312 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 313 unsigned int digest_ofs = 0; 314 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 315 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 316 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 317 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 318 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 319 320 unsigned int idx = 0; 321 int i; 322 323 /* calc derived HMAC key */ 324 for (i = 0; i < 2; i++) { 325 /* Load hash initial state */ 326 hw_desc_init(&desc[idx]); 327 set_cipher_mode(&desc[idx], hash_mode); 328 set_din_sram(&desc[idx], 329 cc_larval_digest_addr(ctx->drvdata, 330 ctx->auth_mode), 331 digest_size); 332 set_flow_mode(&desc[idx], S_DIN_to_HASH); 333 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 334 idx++; 335 336 /* Load the hash current length*/ 337 hw_desc_init(&desc[idx]); 338 set_cipher_mode(&desc[idx], hash_mode); 339 set_din_const(&desc[idx], 0, ctx->hash_len); 340 set_flow_mode(&desc[idx], S_DIN_to_HASH); 341 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 342 idx++; 343 344 /* Prepare ipad key */ 345 hw_desc_init(&desc[idx]); 346 set_xor_val(&desc[idx], hmac_pad_const[i]); 347 set_cipher_mode(&desc[idx], hash_mode); 348 set_flow_mode(&desc[idx], S_DIN_to_HASH); 349 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 350 idx++; 351 352 /* Perform HASH update */ 353 hw_desc_init(&desc[idx]); 354 set_din_type(&desc[idx], DMA_DLLI, 355 hmac->padded_authkey_dma_addr, 356 SHA256_BLOCK_SIZE, NS_BIT); 357 set_cipher_mode(&desc[idx], hash_mode); 358 set_xor_active(&desc[idx]); 359 set_flow_mode(&desc[idx], DIN_HASH); 360 idx++; 361 362 /* Get the digset */ 363 hw_desc_init(&desc[idx]); 364 set_cipher_mode(&desc[idx], hash_mode); 365 set_dout_dlli(&desc[idx], 366 (hmac->ipad_opad_dma_addr + digest_ofs), 367 digest_size, NS_BIT, 0); 368 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 369 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 370 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 371 idx++; 372 373 digest_ofs += digest_size; 374 } 375 376 return idx; 377 } 378 379 static int validate_keys_sizes(struct cc_aead_ctx *ctx) 380 { 381 struct device *dev = drvdata_to_dev(ctx->drvdata); 382 383 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 384 ctx->enc_keylen, ctx->auth_keylen); 385 386 switch (ctx->auth_mode) { 387 case DRV_HASH_SHA1: 388 case DRV_HASH_SHA256: 389 break; 390 case DRV_HASH_XCBC_MAC: 391 if (ctx->auth_keylen != AES_KEYSIZE_128 && 392 ctx->auth_keylen != AES_KEYSIZE_192 && 393 ctx->auth_keylen != AES_KEYSIZE_256) 394 return -ENOTSUPP; 395 break; 396 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 397 if (ctx->auth_keylen > 0) 398 return -EINVAL; 399 break; 400 default: 401 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 402 return -EINVAL; 403 } 404 /* Check cipher key size */ 405 if (ctx->flow_mode == S_DIN_to_DES) { 406 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 407 dev_err(dev, "Invalid cipher(3DES) key size: %u\n", 408 ctx->enc_keylen); 409 return -EINVAL; 410 } 411 } else { /* Default assumed to be AES ciphers */ 412 if (ctx->enc_keylen != AES_KEYSIZE_128 && 413 ctx->enc_keylen != AES_KEYSIZE_192 && 414 ctx->enc_keylen != AES_KEYSIZE_256) { 415 dev_err(dev, "Invalid cipher(AES) key size: %u\n", 416 ctx->enc_keylen); 417 return -EINVAL; 418 } 419 } 420 421 return 0; /* All tests of keys sizes passed */ 422 } 423 424 /* This function prepers the user key so it can pass to the hmac processing 425 * (copy to intenral buffer or hash in case of key longer than block 426 */ 427 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key, 428 unsigned int keylen) 429 { 430 dma_addr_t key_dma_addr = 0; 431 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 432 struct device *dev = drvdata_to_dev(ctx->drvdata); 433 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode); 434 struct cc_crypto_req cc_req = {}; 435 unsigned int blocksize; 436 unsigned int digestsize; 437 unsigned int hashmode; 438 unsigned int idx = 0; 439 int rc = 0; 440 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 441 dma_addr_t padded_authkey_dma_addr = 442 ctx->auth_state.hmac.padded_authkey_dma_addr; 443 444 switch (ctx->auth_mode) { /* auth_key required and >0 */ 445 case DRV_HASH_SHA1: 446 blocksize = SHA1_BLOCK_SIZE; 447 digestsize = SHA1_DIGEST_SIZE; 448 hashmode = DRV_HASH_HW_SHA1; 449 break; 450 case DRV_HASH_SHA256: 451 default: 452 blocksize = SHA256_BLOCK_SIZE; 453 digestsize = SHA256_DIGEST_SIZE; 454 hashmode = DRV_HASH_HW_SHA256; 455 } 456 457 if (keylen != 0) { 458 key_dma_addr = dma_map_single(dev, (void *)key, keylen, 459 DMA_TO_DEVICE); 460 if (dma_mapping_error(dev, key_dma_addr)) { 461 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 462 key, keylen); 463 return -ENOMEM; 464 } 465 if (keylen > blocksize) { 466 /* Load hash initial state */ 467 hw_desc_init(&desc[idx]); 468 set_cipher_mode(&desc[idx], hashmode); 469 set_din_sram(&desc[idx], larval_addr, digestsize); 470 set_flow_mode(&desc[idx], S_DIN_to_HASH); 471 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 472 idx++; 473 474 /* Load the hash current length*/ 475 hw_desc_init(&desc[idx]); 476 set_cipher_mode(&desc[idx], hashmode); 477 set_din_const(&desc[idx], 0, ctx->hash_len); 478 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 479 set_flow_mode(&desc[idx], S_DIN_to_HASH); 480 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 481 idx++; 482 483 hw_desc_init(&desc[idx]); 484 set_din_type(&desc[idx], DMA_DLLI, 485 key_dma_addr, keylen, NS_BIT); 486 set_flow_mode(&desc[idx], DIN_HASH); 487 idx++; 488 489 /* Get hashed key */ 490 hw_desc_init(&desc[idx]); 491 set_cipher_mode(&desc[idx], hashmode); 492 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 493 digestsize, NS_BIT, 0); 494 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 495 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 496 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 497 set_cipher_config0(&desc[idx], 498 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 499 idx++; 500 501 hw_desc_init(&desc[idx]); 502 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 503 set_flow_mode(&desc[idx], BYPASS); 504 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 505 digestsize), (blocksize - digestsize), 506 NS_BIT, 0); 507 idx++; 508 } else { 509 hw_desc_init(&desc[idx]); 510 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 511 keylen, NS_BIT); 512 set_flow_mode(&desc[idx], BYPASS); 513 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 514 keylen, NS_BIT, 0); 515 idx++; 516 517 if ((blocksize - keylen) != 0) { 518 hw_desc_init(&desc[idx]); 519 set_din_const(&desc[idx], 0, 520 (blocksize - keylen)); 521 set_flow_mode(&desc[idx], BYPASS); 522 set_dout_dlli(&desc[idx], 523 (padded_authkey_dma_addr + 524 keylen), 525 (blocksize - keylen), NS_BIT, 0); 526 idx++; 527 } 528 } 529 } else { 530 hw_desc_init(&desc[idx]); 531 set_din_const(&desc[idx], 0, (blocksize - keylen)); 532 set_flow_mode(&desc[idx], BYPASS); 533 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 534 blocksize, NS_BIT, 0); 535 idx++; 536 } 537 538 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 539 if (rc) 540 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 541 542 if (key_dma_addr) 543 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 544 545 return rc; 546 } 547 548 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 549 unsigned int keylen) 550 { 551 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 552 struct rtattr *rta = (struct rtattr *)key; 553 struct cc_crypto_req cc_req = {}; 554 struct crypto_authenc_key_param *param; 555 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 556 int rc = -EINVAL; 557 unsigned int seq_len = 0; 558 struct device *dev = drvdata_to_dev(ctx->drvdata); 559 560 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 561 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 562 563 /* STAT_PHASE_0: Init and sanity checks */ 564 565 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 566 if (!RTA_OK(rta, keylen)) 567 goto badkey; 568 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 569 goto badkey; 570 if (RTA_PAYLOAD(rta) < sizeof(*param)) 571 goto badkey; 572 param = RTA_DATA(rta); 573 ctx->enc_keylen = be32_to_cpu(param->enckeylen); 574 key += RTA_ALIGN(rta->rta_len); 575 keylen -= RTA_ALIGN(rta->rta_len); 576 if (keylen < ctx->enc_keylen) 577 goto badkey; 578 ctx->auth_keylen = keylen - ctx->enc_keylen; 579 580 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 581 /* the nonce is stored in bytes at end of key */ 582 if (ctx->enc_keylen < 583 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 584 goto badkey; 585 /* Copy nonce from last 4 bytes in CTR key to 586 * first 4 bytes in CTR IV 587 */ 588 memcpy(ctx->ctr_nonce, key + ctx->auth_keylen + 589 ctx->enc_keylen - CTR_RFC3686_NONCE_SIZE, 590 CTR_RFC3686_NONCE_SIZE); 591 /* Set CTR key size */ 592 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 593 } 594 } else { /* non-authenc - has just one key */ 595 ctx->enc_keylen = keylen; 596 ctx->auth_keylen = 0; 597 } 598 599 rc = validate_keys_sizes(ctx); 600 if (rc) 601 goto badkey; 602 603 /* STAT_PHASE_1: Copy key to ctx */ 604 605 /* Get key material */ 606 memcpy(ctx->enckey, key + ctx->auth_keylen, ctx->enc_keylen); 607 if (ctx->enc_keylen == 24) 608 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 609 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 610 memcpy(ctx->auth_state.xcbc.xcbc_keys, key, ctx->auth_keylen); 611 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 612 rc = cc_get_plain_hmac_key(tfm, key, ctx->auth_keylen); 613 if (rc) 614 goto badkey; 615 } 616 617 /* STAT_PHASE_2: Create sequence */ 618 619 switch (ctx->auth_mode) { 620 case DRV_HASH_SHA1: 621 case DRV_HASH_SHA256: 622 seq_len = hmac_setkey(desc, ctx); 623 break; 624 case DRV_HASH_XCBC_MAC: 625 seq_len = xcbc_setkey(desc, ctx); 626 break; 627 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 628 break; /* No auth. key setup */ 629 default: 630 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 631 rc = -ENOTSUPP; 632 goto badkey; 633 } 634 635 /* STAT_PHASE_3: Submit sequence to HW */ 636 637 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 638 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 639 if (rc) { 640 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 641 goto setkey_error; 642 } 643 } 644 645 /* Update STAT_PHASE_3 */ 646 return rc; 647 648 badkey: 649 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 650 651 setkey_error: 652 return rc; 653 } 654 655 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 656 unsigned int keylen) 657 { 658 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 659 660 if (keylen < 3) 661 return -EINVAL; 662 663 keylen -= 3; 664 memcpy(ctx->ctr_nonce, key + keylen, 3); 665 666 return cc_aead_setkey(tfm, key, keylen); 667 } 668 669 static int cc_aead_setauthsize(struct crypto_aead *authenc, 670 unsigned int authsize) 671 { 672 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 673 struct device *dev = drvdata_to_dev(ctx->drvdata); 674 675 /* Unsupported auth. sizes */ 676 if (authsize == 0 || 677 authsize > crypto_aead_maxauthsize(authenc)) { 678 return -ENOTSUPP; 679 } 680 681 ctx->authsize = authsize; 682 dev_dbg(dev, "authlen=%d\n", ctx->authsize); 683 684 return 0; 685 } 686 687 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 688 unsigned int authsize) 689 { 690 switch (authsize) { 691 case 8: 692 case 12: 693 case 16: 694 break; 695 default: 696 return -EINVAL; 697 } 698 699 return cc_aead_setauthsize(authenc, authsize); 700 } 701 702 static int cc_ccm_setauthsize(struct crypto_aead *authenc, 703 unsigned int authsize) 704 { 705 switch (authsize) { 706 case 4: 707 case 6: 708 case 8: 709 case 10: 710 case 12: 711 case 14: 712 case 16: 713 break; 714 default: 715 return -EINVAL; 716 } 717 718 return cc_aead_setauthsize(authenc, authsize); 719 } 720 721 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 722 struct cc_hw_desc desc[], unsigned int *seq_size) 723 { 724 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 725 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 726 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 727 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 728 unsigned int idx = *seq_size; 729 struct device *dev = drvdata_to_dev(ctx->drvdata); 730 731 switch (assoc_dma_type) { 732 case CC_DMA_BUF_DLLI: 733 dev_dbg(dev, "ASSOC buffer type DLLI\n"); 734 hw_desc_init(&desc[idx]); 735 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 736 areq->assoclen, NS_BIT); 737 set_flow_mode(&desc[idx], flow_mode); 738 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 739 areq_ctx->cryptlen > 0) 740 set_din_not_last_indication(&desc[idx]); 741 break; 742 case CC_DMA_BUF_MLLI: 743 dev_dbg(dev, "ASSOC buffer type MLLI\n"); 744 hw_desc_init(&desc[idx]); 745 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 746 areq_ctx->assoc.mlli_nents, NS_BIT); 747 set_flow_mode(&desc[idx], flow_mode); 748 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 749 areq_ctx->cryptlen > 0) 750 set_din_not_last_indication(&desc[idx]); 751 break; 752 case CC_DMA_BUF_NULL: 753 default: 754 dev_err(dev, "Invalid ASSOC buffer type\n"); 755 } 756 757 *seq_size = (++idx); 758 } 759 760 static void cc_proc_authen_desc(struct aead_request *areq, 761 unsigned int flow_mode, 762 struct cc_hw_desc desc[], 763 unsigned int *seq_size, int direct) 764 { 765 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 766 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 767 unsigned int idx = *seq_size; 768 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 769 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 770 struct device *dev = drvdata_to_dev(ctx->drvdata); 771 772 switch (data_dma_type) { 773 case CC_DMA_BUF_DLLI: 774 { 775 struct scatterlist *cipher = 776 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 777 areq_ctx->dst_sgl : areq_ctx->src_sgl; 778 779 unsigned int offset = 780 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 781 areq_ctx->dst_offset : areq_ctx->src_offset; 782 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 783 hw_desc_init(&desc[idx]); 784 set_din_type(&desc[idx], DMA_DLLI, 785 (sg_dma_address(cipher) + offset), 786 areq_ctx->cryptlen, NS_BIT); 787 set_flow_mode(&desc[idx], flow_mode); 788 break; 789 } 790 case CC_DMA_BUF_MLLI: 791 { 792 /* DOUBLE-PASS flow (as default) 793 * assoc. + iv + data -compact in one table 794 * if assoclen is ZERO only IV perform 795 */ 796 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr; 797 u32 mlli_nents = areq_ctx->assoc.mlli_nents; 798 799 if (areq_ctx->is_single_pass) { 800 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 801 mlli_addr = areq_ctx->dst.sram_addr; 802 mlli_nents = areq_ctx->dst.mlli_nents; 803 } else { 804 mlli_addr = areq_ctx->src.sram_addr; 805 mlli_nents = areq_ctx->src.mlli_nents; 806 } 807 } 808 809 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 810 hw_desc_init(&desc[idx]); 811 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 812 NS_BIT); 813 set_flow_mode(&desc[idx], flow_mode); 814 break; 815 } 816 case CC_DMA_BUF_NULL: 817 default: 818 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 819 } 820 821 *seq_size = (++idx); 822 } 823 824 static void cc_proc_cipher_desc(struct aead_request *areq, 825 unsigned int flow_mode, 826 struct cc_hw_desc desc[], 827 unsigned int *seq_size) 828 { 829 unsigned int idx = *seq_size; 830 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 831 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 832 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 833 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 834 struct device *dev = drvdata_to_dev(ctx->drvdata); 835 836 if (areq_ctx->cryptlen == 0) 837 return; /*null processing*/ 838 839 switch (data_dma_type) { 840 case CC_DMA_BUF_DLLI: 841 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 842 hw_desc_init(&desc[idx]); 843 set_din_type(&desc[idx], DMA_DLLI, 844 (sg_dma_address(areq_ctx->src_sgl) + 845 areq_ctx->src_offset), areq_ctx->cryptlen, 846 NS_BIT); 847 set_dout_dlli(&desc[idx], 848 (sg_dma_address(areq_ctx->dst_sgl) + 849 areq_ctx->dst_offset), 850 areq_ctx->cryptlen, NS_BIT, 0); 851 set_flow_mode(&desc[idx], flow_mode); 852 break; 853 case CC_DMA_BUF_MLLI: 854 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 855 hw_desc_init(&desc[idx]); 856 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 857 areq_ctx->src.mlli_nents, NS_BIT); 858 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 859 areq_ctx->dst.mlli_nents, NS_BIT, 0); 860 set_flow_mode(&desc[idx], flow_mode); 861 break; 862 case CC_DMA_BUF_NULL: 863 default: 864 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 865 } 866 867 *seq_size = (++idx); 868 } 869 870 static void cc_proc_digest_desc(struct aead_request *req, 871 struct cc_hw_desc desc[], 872 unsigned int *seq_size) 873 { 874 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 875 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 876 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 877 unsigned int idx = *seq_size; 878 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 879 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 880 int direct = req_ctx->gen_ctx.op_type; 881 882 /* Get final ICV result */ 883 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 884 hw_desc_init(&desc[idx]); 885 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 886 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 887 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 888 NS_BIT, 1); 889 set_queue_last_ind(ctx->drvdata, &desc[idx]); 890 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 891 set_aes_not_hash_mode(&desc[idx]); 892 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 893 } else { 894 set_cipher_config0(&desc[idx], 895 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 896 set_cipher_mode(&desc[idx], hash_mode); 897 } 898 } else { /*Decrypt*/ 899 /* Get ICV out from hardware */ 900 hw_desc_init(&desc[idx]); 901 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 902 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 903 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 904 ctx->authsize, NS_BIT, 1); 905 set_queue_last_ind(ctx->drvdata, &desc[idx]); 906 set_cipher_config0(&desc[idx], 907 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 908 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 909 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 910 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 911 set_aes_not_hash_mode(&desc[idx]); 912 } else { 913 set_cipher_mode(&desc[idx], hash_mode); 914 } 915 } 916 917 *seq_size = (++idx); 918 } 919 920 static void cc_set_cipher_desc(struct aead_request *req, 921 struct cc_hw_desc desc[], 922 unsigned int *seq_size) 923 { 924 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 925 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 926 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 927 unsigned int hw_iv_size = req_ctx->hw_iv_size; 928 unsigned int idx = *seq_size; 929 int direct = req_ctx->gen_ctx.op_type; 930 931 /* Setup cipher state */ 932 hw_desc_init(&desc[idx]); 933 set_cipher_config0(&desc[idx], direct); 934 set_flow_mode(&desc[idx], ctx->flow_mode); 935 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 936 hw_iv_size, NS_BIT); 937 if (ctx->cipher_mode == DRV_CIPHER_CTR) 938 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 939 else 940 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 941 set_cipher_mode(&desc[idx], ctx->cipher_mode); 942 idx++; 943 944 /* Setup enc. key */ 945 hw_desc_init(&desc[idx]); 946 set_cipher_config0(&desc[idx], direct); 947 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 948 set_flow_mode(&desc[idx], ctx->flow_mode); 949 if (ctx->flow_mode == S_DIN_to_AES) { 950 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 951 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 952 ctx->enc_keylen), NS_BIT); 953 set_key_size_aes(&desc[idx], ctx->enc_keylen); 954 } else { 955 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 956 ctx->enc_keylen, NS_BIT); 957 set_key_size_des(&desc[idx], ctx->enc_keylen); 958 } 959 set_cipher_mode(&desc[idx], ctx->cipher_mode); 960 idx++; 961 962 *seq_size = idx; 963 } 964 965 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 966 unsigned int *seq_size, unsigned int data_flow_mode) 967 { 968 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 969 int direct = req_ctx->gen_ctx.op_type; 970 unsigned int idx = *seq_size; 971 972 if (req_ctx->cryptlen == 0) 973 return; /*null processing*/ 974 975 cc_set_cipher_desc(req, desc, &idx); 976 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 977 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 978 /* We must wait for DMA to write all cipher */ 979 hw_desc_init(&desc[idx]); 980 set_din_no_dma(&desc[idx], 0, 0xfffff0); 981 set_dout_no_dma(&desc[idx], 0, 0, 1); 982 idx++; 983 } 984 985 *seq_size = idx; 986 } 987 988 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 989 unsigned int *seq_size) 990 { 991 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 992 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 993 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 994 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 995 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 996 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 997 unsigned int idx = *seq_size; 998 999 /* Loading hash ipad xor key state */ 1000 hw_desc_init(&desc[idx]); 1001 set_cipher_mode(&desc[idx], hash_mode); 1002 set_din_type(&desc[idx], DMA_DLLI, 1003 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1004 NS_BIT); 1005 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1006 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1007 idx++; 1008 1009 /* Load init. digest len (64 bytes) */ 1010 hw_desc_init(&desc[idx]); 1011 set_cipher_mode(&desc[idx], hash_mode); 1012 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1013 ctx->hash_len); 1014 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1015 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1016 idx++; 1017 1018 *seq_size = idx; 1019 } 1020 1021 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1022 unsigned int *seq_size) 1023 { 1024 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1025 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1026 unsigned int idx = *seq_size; 1027 1028 /* Loading MAC state */ 1029 hw_desc_init(&desc[idx]); 1030 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1031 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1032 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1033 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1034 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1035 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1036 set_aes_not_hash_mode(&desc[idx]); 1037 idx++; 1038 1039 /* Setup XCBC MAC K1 */ 1040 hw_desc_init(&desc[idx]); 1041 set_din_type(&desc[idx], DMA_DLLI, 1042 ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1043 AES_KEYSIZE_128, NS_BIT); 1044 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1045 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1046 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1047 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1048 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1049 set_aes_not_hash_mode(&desc[idx]); 1050 idx++; 1051 1052 /* Setup XCBC MAC K2 */ 1053 hw_desc_init(&desc[idx]); 1054 set_din_type(&desc[idx], DMA_DLLI, 1055 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1056 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1057 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1058 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1059 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1060 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1061 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1062 set_aes_not_hash_mode(&desc[idx]); 1063 idx++; 1064 1065 /* Setup XCBC MAC K3 */ 1066 hw_desc_init(&desc[idx]); 1067 set_din_type(&desc[idx], DMA_DLLI, 1068 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1069 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1070 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1071 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1072 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1073 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1074 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1075 set_aes_not_hash_mode(&desc[idx]); 1076 idx++; 1077 1078 *seq_size = idx; 1079 } 1080 1081 static void cc_proc_header_desc(struct aead_request *req, 1082 struct cc_hw_desc desc[], 1083 unsigned int *seq_size) 1084 { 1085 unsigned int idx = *seq_size; 1086 /* Hash associated data */ 1087 if (req->assoclen > 0) 1088 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1089 1090 /* Hash IV */ 1091 *seq_size = idx; 1092 } 1093 1094 static void cc_proc_scheme_desc(struct aead_request *req, 1095 struct cc_hw_desc desc[], 1096 unsigned int *seq_size) 1097 { 1098 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1099 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1100 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1101 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1102 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1103 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1104 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1105 unsigned int idx = *seq_size; 1106 1107 hw_desc_init(&desc[idx]); 1108 set_cipher_mode(&desc[idx], hash_mode); 1109 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1110 ctx->hash_len); 1111 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1112 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1113 set_cipher_do(&desc[idx], DO_PAD); 1114 idx++; 1115 1116 /* Get final ICV result */ 1117 hw_desc_init(&desc[idx]); 1118 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1119 digest_size); 1120 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1121 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1122 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1123 set_cipher_mode(&desc[idx], hash_mode); 1124 idx++; 1125 1126 /* Loading hash opad xor key state */ 1127 hw_desc_init(&desc[idx]); 1128 set_cipher_mode(&desc[idx], hash_mode); 1129 set_din_type(&desc[idx], DMA_DLLI, 1130 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1131 digest_size, NS_BIT); 1132 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1133 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1134 idx++; 1135 1136 /* Load init. digest len (64 bytes) */ 1137 hw_desc_init(&desc[idx]); 1138 set_cipher_mode(&desc[idx], hash_mode); 1139 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1140 ctx->hash_len); 1141 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1142 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1143 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1144 idx++; 1145 1146 /* Perform HASH update */ 1147 hw_desc_init(&desc[idx]); 1148 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1149 digest_size); 1150 set_flow_mode(&desc[idx], DIN_HASH); 1151 idx++; 1152 1153 *seq_size = idx; 1154 } 1155 1156 static void cc_mlli_to_sram(struct aead_request *req, 1157 struct cc_hw_desc desc[], unsigned int *seq_size) 1158 { 1159 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1160 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1161 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1162 struct device *dev = drvdata_to_dev(ctx->drvdata); 1163 1164 if (req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1165 req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1166 !req_ctx->is_single_pass) { 1167 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1168 (unsigned int)ctx->drvdata->mlli_sram_addr, 1169 req_ctx->mlli_params.mlli_len); 1170 /* Copy MLLI table host-to-sram */ 1171 hw_desc_init(&desc[*seq_size]); 1172 set_din_type(&desc[*seq_size], DMA_DLLI, 1173 req_ctx->mlli_params.mlli_dma_addr, 1174 req_ctx->mlli_params.mlli_len, NS_BIT); 1175 set_dout_sram(&desc[*seq_size], 1176 ctx->drvdata->mlli_sram_addr, 1177 req_ctx->mlli_params.mlli_len); 1178 set_flow_mode(&desc[*seq_size], BYPASS); 1179 (*seq_size)++; 1180 } 1181 } 1182 1183 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1184 enum cc_flow_mode setup_flow_mode, 1185 bool is_single_pass) 1186 { 1187 enum cc_flow_mode data_flow_mode; 1188 1189 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1190 if (setup_flow_mode == S_DIN_to_AES) 1191 data_flow_mode = is_single_pass ? 1192 AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1193 else 1194 data_flow_mode = is_single_pass ? 1195 DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1196 } else { /* Decrypt */ 1197 if (setup_flow_mode == S_DIN_to_AES) 1198 data_flow_mode = is_single_pass ? 1199 AES_and_HASH : DIN_AES_DOUT; 1200 else 1201 data_flow_mode = is_single_pass ? 1202 DES_and_HASH : DIN_DES_DOUT; 1203 } 1204 1205 return data_flow_mode; 1206 } 1207 1208 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1209 unsigned int *seq_size) 1210 { 1211 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1212 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1213 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1214 int direct = req_ctx->gen_ctx.op_type; 1215 unsigned int data_flow_mode = 1216 cc_get_data_flow(direct, ctx->flow_mode, 1217 req_ctx->is_single_pass); 1218 1219 if (req_ctx->is_single_pass) { 1220 /** 1221 * Single-pass flow 1222 */ 1223 cc_set_hmac_desc(req, desc, seq_size); 1224 cc_set_cipher_desc(req, desc, seq_size); 1225 cc_proc_header_desc(req, desc, seq_size); 1226 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1227 cc_proc_scheme_desc(req, desc, seq_size); 1228 cc_proc_digest_desc(req, desc, seq_size); 1229 return; 1230 } 1231 1232 /** 1233 * Double-pass flow 1234 * Fallback for unsupported single-pass modes, 1235 * i.e. using assoc. data of non-word-multiple 1236 */ 1237 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1238 /* encrypt first.. */ 1239 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1240 /* authenc after..*/ 1241 cc_set_hmac_desc(req, desc, seq_size); 1242 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1243 cc_proc_scheme_desc(req, desc, seq_size); 1244 cc_proc_digest_desc(req, desc, seq_size); 1245 1246 } else { /*DECRYPT*/ 1247 /* authenc first..*/ 1248 cc_set_hmac_desc(req, desc, seq_size); 1249 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1250 cc_proc_scheme_desc(req, desc, seq_size); 1251 /* decrypt after.. */ 1252 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1253 /* read the digest result with setting the completion bit 1254 * must be after the cipher operation 1255 */ 1256 cc_proc_digest_desc(req, desc, seq_size); 1257 } 1258 } 1259 1260 static void 1261 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1262 unsigned int *seq_size) 1263 { 1264 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1265 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1266 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1267 int direct = req_ctx->gen_ctx.op_type; 1268 unsigned int data_flow_mode = 1269 cc_get_data_flow(direct, ctx->flow_mode, 1270 req_ctx->is_single_pass); 1271 1272 if (req_ctx->is_single_pass) { 1273 /** 1274 * Single-pass flow 1275 */ 1276 cc_set_xcbc_desc(req, desc, seq_size); 1277 cc_set_cipher_desc(req, desc, seq_size); 1278 cc_proc_header_desc(req, desc, seq_size); 1279 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1280 cc_proc_digest_desc(req, desc, seq_size); 1281 return; 1282 } 1283 1284 /** 1285 * Double-pass flow 1286 * Fallback for unsupported single-pass modes, 1287 * i.e. using assoc. data of non-word-multiple 1288 */ 1289 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1290 /* encrypt first.. */ 1291 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1292 /* authenc after.. */ 1293 cc_set_xcbc_desc(req, desc, seq_size); 1294 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1295 cc_proc_digest_desc(req, desc, seq_size); 1296 } else { /*DECRYPT*/ 1297 /* authenc first.. */ 1298 cc_set_xcbc_desc(req, desc, seq_size); 1299 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1300 /* decrypt after..*/ 1301 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1302 /* read the digest result with setting the completion bit 1303 * must be after the cipher operation 1304 */ 1305 cc_proc_digest_desc(req, desc, seq_size); 1306 } 1307 } 1308 1309 static int validate_data_size(struct cc_aead_ctx *ctx, 1310 enum drv_crypto_direction direct, 1311 struct aead_request *req) 1312 { 1313 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1314 struct device *dev = drvdata_to_dev(ctx->drvdata); 1315 unsigned int assoclen = req->assoclen; 1316 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1317 (req->cryptlen - ctx->authsize) : req->cryptlen; 1318 1319 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1320 req->cryptlen < ctx->authsize) 1321 goto data_size_err; 1322 1323 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1324 1325 switch (ctx->flow_mode) { 1326 case S_DIN_to_AES: 1327 if (ctx->cipher_mode == DRV_CIPHER_CBC && 1328 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1329 goto data_size_err; 1330 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1331 break; 1332 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1333 if (areq_ctx->plaintext_authenticate_only) 1334 areq_ctx->is_single_pass = false; 1335 break; 1336 } 1337 1338 if (!IS_ALIGNED(assoclen, sizeof(u32))) 1339 areq_ctx->is_single_pass = false; 1340 1341 if (ctx->cipher_mode == DRV_CIPHER_CTR && 1342 !IS_ALIGNED(cipherlen, sizeof(u32))) 1343 areq_ctx->is_single_pass = false; 1344 1345 break; 1346 case S_DIN_to_DES: 1347 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1348 goto data_size_err; 1349 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1350 areq_ctx->is_single_pass = false; 1351 break; 1352 default: 1353 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1354 goto data_size_err; 1355 } 1356 1357 return 0; 1358 1359 data_size_err: 1360 return -EINVAL; 1361 } 1362 1363 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1364 { 1365 unsigned int len = 0; 1366 1367 if (header_size == 0) 1368 return 0; 1369 1370 if (header_size < ((1UL << 16) - (1UL << 8))) { 1371 len = 2; 1372 1373 pa0_buff[0] = (header_size >> 8) & 0xFF; 1374 pa0_buff[1] = header_size & 0xFF; 1375 } else { 1376 len = 6; 1377 1378 pa0_buff[0] = 0xFF; 1379 pa0_buff[1] = 0xFE; 1380 pa0_buff[2] = (header_size >> 24) & 0xFF; 1381 pa0_buff[3] = (header_size >> 16) & 0xFF; 1382 pa0_buff[4] = (header_size >> 8) & 0xFF; 1383 pa0_buff[5] = header_size & 0xFF; 1384 } 1385 1386 return len; 1387 } 1388 1389 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1390 { 1391 __be32 data; 1392 1393 memset(block, 0, csize); 1394 block += csize; 1395 1396 if (csize >= 4) 1397 csize = 4; 1398 else if (msglen > (1 << (8 * csize))) 1399 return -EOVERFLOW; 1400 1401 data = cpu_to_be32(msglen); 1402 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1403 1404 return 0; 1405 } 1406 1407 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1408 unsigned int *seq_size) 1409 { 1410 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1411 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1412 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1413 unsigned int idx = *seq_size; 1414 unsigned int cipher_flow_mode; 1415 dma_addr_t mac_result; 1416 1417 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1418 cipher_flow_mode = AES_to_HASH_and_DOUT; 1419 mac_result = req_ctx->mac_buf_dma_addr; 1420 } else { /* Encrypt */ 1421 cipher_flow_mode = AES_and_HASH; 1422 mac_result = req_ctx->icv_dma_addr; 1423 } 1424 1425 /* load key */ 1426 hw_desc_init(&desc[idx]); 1427 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1428 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1429 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1430 ctx->enc_keylen), NS_BIT); 1431 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1432 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1433 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1434 set_flow_mode(&desc[idx], S_DIN_to_AES); 1435 idx++; 1436 1437 /* load ctr state */ 1438 hw_desc_init(&desc[idx]); 1439 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1440 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1441 set_din_type(&desc[idx], DMA_DLLI, 1442 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1443 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1444 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1445 set_flow_mode(&desc[idx], S_DIN_to_AES); 1446 idx++; 1447 1448 /* load MAC key */ 1449 hw_desc_init(&desc[idx]); 1450 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1451 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1452 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1453 ctx->enc_keylen), NS_BIT); 1454 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1455 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1456 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1457 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1458 set_aes_not_hash_mode(&desc[idx]); 1459 idx++; 1460 1461 /* load MAC state */ 1462 hw_desc_init(&desc[idx]); 1463 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1464 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1465 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1466 AES_BLOCK_SIZE, NS_BIT); 1467 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1468 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1469 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1470 set_aes_not_hash_mode(&desc[idx]); 1471 idx++; 1472 1473 /* process assoc data */ 1474 if (req->assoclen > 0) { 1475 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1476 } else { 1477 hw_desc_init(&desc[idx]); 1478 set_din_type(&desc[idx], DMA_DLLI, 1479 sg_dma_address(&req_ctx->ccm_adata_sg), 1480 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1481 set_flow_mode(&desc[idx], DIN_HASH); 1482 idx++; 1483 } 1484 1485 /* process the cipher */ 1486 if (req_ctx->cryptlen) 1487 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1488 1489 /* Read temporal MAC */ 1490 hw_desc_init(&desc[idx]); 1491 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1492 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1493 NS_BIT, 0); 1494 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1495 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1496 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1497 set_aes_not_hash_mode(&desc[idx]); 1498 idx++; 1499 1500 /* load AES-CTR state (for last MAC calculation)*/ 1501 hw_desc_init(&desc[idx]); 1502 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1503 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1504 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1505 AES_BLOCK_SIZE, NS_BIT); 1506 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1507 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1508 set_flow_mode(&desc[idx], S_DIN_to_AES); 1509 idx++; 1510 1511 hw_desc_init(&desc[idx]); 1512 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1513 set_dout_no_dma(&desc[idx], 0, 0, 1); 1514 idx++; 1515 1516 /* encrypt the "T" value and store MAC in mac_state */ 1517 hw_desc_init(&desc[idx]); 1518 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1519 ctx->authsize, NS_BIT); 1520 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1521 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1522 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1523 idx++; 1524 1525 *seq_size = idx; 1526 return 0; 1527 } 1528 1529 static int config_ccm_adata(struct aead_request *req) 1530 { 1531 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1532 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1533 struct device *dev = drvdata_to_dev(ctx->drvdata); 1534 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1535 //unsigned int size_of_a = 0, rem_a_size = 0; 1536 unsigned int lp = req->iv[0]; 1537 /* Note: The code assume that req->iv[0] already contains the value 1538 * of L' of RFC3610 1539 */ 1540 unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1541 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1542 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1543 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1544 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1545 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1546 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1547 req->cryptlen : 1548 (req->cryptlen - ctx->authsize); 1549 int rc; 1550 1551 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1552 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1553 1554 /* taken from crypto/ccm.c */ 1555 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1556 if (l < 2 || l > 8) { 1557 dev_err(dev, "illegal iv value %X\n", req->iv[0]); 1558 return -EINVAL; 1559 } 1560 memcpy(b0, req->iv, AES_BLOCK_SIZE); 1561 1562 /* format control info per RFC 3610 and 1563 * NIST Special Publication 800-38C 1564 */ 1565 *b0 |= (8 * ((m - 2) / 2)); 1566 if (req->assoclen > 0) 1567 *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1568 1569 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1570 if (rc) { 1571 dev_err(dev, "message len overflow detected"); 1572 return rc; 1573 } 1574 /* END of "taken from crypto/ccm.c" */ 1575 1576 /* l(a) - size of associated data. */ 1577 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req->assoclen); 1578 1579 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1580 req->iv[15] = 1; 1581 1582 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1583 ctr_count_0[15] = 0; 1584 1585 return 0; 1586 } 1587 1588 static void cc_proc_rfc4309_ccm(struct aead_request *req) 1589 { 1590 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1591 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1592 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1593 1594 /* L' */ 1595 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1596 /* For RFC 4309, always use 4 bytes for message length 1597 * (at most 2^32-1 bytes). 1598 */ 1599 areq_ctx->ctr_iv[0] = 3; 1600 1601 /* In RFC 4309 there is an 11-bytes nonce+IV part, 1602 * that we build here. 1603 */ 1604 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1605 CCM_BLOCK_NONCE_SIZE); 1606 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1607 CCM_BLOCK_IV_SIZE); 1608 req->iv = areq_ctx->ctr_iv; 1609 req->assoclen -= CCM_BLOCK_IV_SIZE; 1610 } 1611 1612 static void cc_set_ghash_desc(struct aead_request *req, 1613 struct cc_hw_desc desc[], unsigned int *seq_size) 1614 { 1615 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1616 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1617 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1618 unsigned int idx = *seq_size; 1619 1620 /* load key to AES*/ 1621 hw_desc_init(&desc[idx]); 1622 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1623 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1624 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1625 ctx->enc_keylen, NS_BIT); 1626 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1627 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1628 set_flow_mode(&desc[idx], S_DIN_to_AES); 1629 idx++; 1630 1631 /* process one zero block to generate hkey */ 1632 hw_desc_init(&desc[idx]); 1633 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1634 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1635 NS_BIT, 0); 1636 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1637 idx++; 1638 1639 /* Memory Barrier */ 1640 hw_desc_init(&desc[idx]); 1641 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1642 set_dout_no_dma(&desc[idx], 0, 0, 1); 1643 idx++; 1644 1645 /* Load GHASH subkey */ 1646 hw_desc_init(&desc[idx]); 1647 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1648 AES_BLOCK_SIZE, NS_BIT); 1649 set_dout_no_dma(&desc[idx], 0, 0, 1); 1650 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1651 set_aes_not_hash_mode(&desc[idx]); 1652 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1653 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1654 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1655 idx++; 1656 1657 /* Configure Hash Engine to work with GHASH. 1658 * Since it was not possible to extend HASH submodes to add GHASH, 1659 * The following command is necessary in order to 1660 * select GHASH (according to HW designers) 1661 */ 1662 hw_desc_init(&desc[idx]); 1663 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1664 set_dout_no_dma(&desc[idx], 0, 0, 1); 1665 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1666 set_aes_not_hash_mode(&desc[idx]); 1667 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1668 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1669 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1670 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1671 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1672 idx++; 1673 1674 /* Load GHASH initial STATE (which is 0). (for any hash there is an 1675 * initial state) 1676 */ 1677 hw_desc_init(&desc[idx]); 1678 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1679 set_dout_no_dma(&desc[idx], 0, 0, 1); 1680 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1681 set_aes_not_hash_mode(&desc[idx]); 1682 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1683 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1684 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1685 idx++; 1686 1687 *seq_size = idx; 1688 } 1689 1690 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1691 unsigned int *seq_size) 1692 { 1693 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1694 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1695 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1696 unsigned int idx = *seq_size; 1697 1698 /* load key to AES*/ 1699 hw_desc_init(&desc[idx]); 1700 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1701 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1702 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1703 ctx->enc_keylen, NS_BIT); 1704 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1705 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1706 set_flow_mode(&desc[idx], S_DIN_to_AES); 1707 idx++; 1708 1709 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1710 /* load AES/CTR initial CTR value inc by 2*/ 1711 hw_desc_init(&desc[idx]); 1712 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1713 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1714 set_din_type(&desc[idx], DMA_DLLI, 1715 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1716 NS_BIT); 1717 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1718 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1719 set_flow_mode(&desc[idx], S_DIN_to_AES); 1720 idx++; 1721 } 1722 1723 *seq_size = idx; 1724 } 1725 1726 static void cc_proc_gcm_result(struct aead_request *req, 1727 struct cc_hw_desc desc[], 1728 unsigned int *seq_size) 1729 { 1730 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1731 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1732 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1733 dma_addr_t mac_result; 1734 unsigned int idx = *seq_size; 1735 1736 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1737 mac_result = req_ctx->mac_buf_dma_addr; 1738 } else { /* Encrypt */ 1739 mac_result = req_ctx->icv_dma_addr; 1740 } 1741 1742 /* process(ghash) gcm_block_len */ 1743 hw_desc_init(&desc[idx]); 1744 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1745 AES_BLOCK_SIZE, NS_BIT); 1746 set_flow_mode(&desc[idx], DIN_HASH); 1747 idx++; 1748 1749 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1750 hw_desc_init(&desc[idx]); 1751 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1752 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1753 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1754 NS_BIT, 0); 1755 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1756 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1757 set_aes_not_hash_mode(&desc[idx]); 1758 1759 idx++; 1760 1761 /* load AES/CTR initial CTR value inc by 1*/ 1762 hw_desc_init(&desc[idx]); 1763 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1764 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1765 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1766 AES_BLOCK_SIZE, NS_BIT); 1767 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1768 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1769 set_flow_mode(&desc[idx], S_DIN_to_AES); 1770 idx++; 1771 1772 /* Memory Barrier */ 1773 hw_desc_init(&desc[idx]); 1774 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1775 set_dout_no_dma(&desc[idx], 0, 0, 1); 1776 idx++; 1777 1778 /* process GCTR on stored GHASH and store MAC in mac_state*/ 1779 hw_desc_init(&desc[idx]); 1780 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1781 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1782 AES_BLOCK_SIZE, NS_BIT); 1783 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1784 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1785 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1786 idx++; 1787 1788 *seq_size = idx; 1789 } 1790 1791 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1792 unsigned int *seq_size) 1793 { 1794 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1795 unsigned int cipher_flow_mode; 1796 1797 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1798 cipher_flow_mode = AES_and_HASH; 1799 } else { /* Encrypt */ 1800 cipher_flow_mode = AES_to_HASH_and_DOUT; 1801 } 1802 1803 //in RFC4543 no data to encrypt. just copy data from src to dest. 1804 if (req_ctx->plaintext_authenticate_only) { 1805 cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1806 cc_set_ghash_desc(req, desc, seq_size); 1807 /* process(ghash) assoc data */ 1808 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1809 cc_set_gctr_desc(req, desc, seq_size); 1810 cc_proc_gcm_result(req, desc, seq_size); 1811 return 0; 1812 } 1813 1814 // for gcm and rfc4106. 1815 cc_set_ghash_desc(req, desc, seq_size); 1816 /* process(ghash) assoc data */ 1817 if (req->assoclen > 0) 1818 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1819 cc_set_gctr_desc(req, desc, seq_size); 1820 /* process(gctr+ghash) */ 1821 if (req_ctx->cryptlen) 1822 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1823 cc_proc_gcm_result(req, desc, seq_size); 1824 1825 return 0; 1826 } 1827 1828 static int config_gcm_context(struct aead_request *req) 1829 { 1830 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1831 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1832 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1833 struct device *dev = drvdata_to_dev(ctx->drvdata); 1834 1835 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1836 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1837 req->cryptlen : 1838 (req->cryptlen - ctx->authsize); 1839 __be32 counter = cpu_to_be32(2); 1840 1841 dev_dbg(dev, "%s() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n", 1842 __func__, cryptlen, req->assoclen, ctx->authsize); 1843 1844 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1845 1846 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1847 1848 memcpy(req->iv + 12, &counter, 4); 1849 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1850 1851 counter = cpu_to_be32(1); 1852 memcpy(req->iv + 12, &counter, 4); 1853 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1854 1855 if (!req_ctx->plaintext_authenticate_only) { 1856 __be64 temp64; 1857 1858 temp64 = cpu_to_be64(req->assoclen * 8); 1859 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1860 temp64 = cpu_to_be64(cryptlen * 8); 1861 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1862 } else { 1863 /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1864 * data that is nothing is encrypted. 1865 */ 1866 __be64 temp64; 1867 1868 temp64 = cpu_to_be64((req->assoclen + GCM_BLOCK_RFC4_IV_SIZE + 1869 cryptlen) * 8); 1870 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1871 temp64 = 0; 1872 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1873 } 1874 1875 return 0; 1876 } 1877 1878 static void cc_proc_rfc4_gcm(struct aead_request *req) 1879 { 1880 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1881 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1882 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1883 1884 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1885 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1886 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1887 GCM_BLOCK_RFC4_IV_SIZE); 1888 req->iv = areq_ctx->ctr_iv; 1889 req->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; 1890 } 1891 1892 static int cc_proc_aead(struct aead_request *req, 1893 enum drv_crypto_direction direct) 1894 { 1895 int rc = 0; 1896 int seq_len = 0; 1897 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1898 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1899 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1900 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1901 struct device *dev = drvdata_to_dev(ctx->drvdata); 1902 struct cc_crypto_req cc_req = {}; 1903 1904 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1905 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1906 ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1907 sg_virt(req->dst), req->dst->offset, req->cryptlen); 1908 1909 /* STAT_PHASE_0: Init and sanity checks */ 1910 1911 /* Check data length according to mode */ 1912 if (validate_data_size(ctx, direct, req)) { 1913 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1914 req->cryptlen, req->assoclen); 1915 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN); 1916 return -EINVAL; 1917 } 1918 1919 /* Setup request structure */ 1920 cc_req.user_cb = (void *)cc_aead_complete; 1921 cc_req.user_arg = (void *)req; 1922 1923 /* Setup request context */ 1924 areq_ctx->gen_ctx.op_type = direct; 1925 areq_ctx->req_authsize = ctx->authsize; 1926 areq_ctx->cipher_mode = ctx->cipher_mode; 1927 1928 /* STAT_PHASE_1: Map buffers */ 1929 1930 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1931 /* Build CTR IV - Copy nonce from last 4 bytes in 1932 * CTR key to first 4 bytes in CTR IV 1933 */ 1934 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1935 CTR_RFC3686_NONCE_SIZE); 1936 if (!areq_ctx->backup_giv) /*User none-generated IV*/ 1937 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, 1938 req->iv, CTR_RFC3686_IV_SIZE); 1939 /* Initialize counter portion of counter block */ 1940 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1941 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1942 1943 /* Replace with counter iv */ 1944 req->iv = areq_ctx->ctr_iv; 1945 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1946 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1947 (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1948 areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1949 if (areq_ctx->ctr_iv != req->iv) { 1950 memcpy(areq_ctx->ctr_iv, req->iv, 1951 crypto_aead_ivsize(tfm)); 1952 req->iv = areq_ctx->ctr_iv; 1953 } 1954 } else { 1955 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1956 } 1957 1958 if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1959 rc = config_ccm_adata(req); 1960 if (rc) { 1961 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1962 rc); 1963 goto exit; 1964 } 1965 } else { 1966 areq_ctx->ccm_hdr_size = ccm_header_size_null; 1967 } 1968 1969 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1970 rc = config_gcm_context(req); 1971 if (rc) { 1972 dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 1973 rc); 1974 goto exit; 1975 } 1976 } 1977 1978 rc = cc_map_aead_request(ctx->drvdata, req); 1979 if (rc) { 1980 dev_err(dev, "map_request() failed\n"); 1981 goto exit; 1982 } 1983 1984 /* do we need to generate IV? */ 1985 if (areq_ctx->backup_giv) { 1986 /* set the DMA mapped IV address*/ 1987 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1988 cc_req.ivgen_dma_addr[0] = 1989 areq_ctx->gen_ctx.iv_dma_addr + 1990 CTR_RFC3686_NONCE_SIZE; 1991 cc_req.ivgen_dma_addr_len = 1; 1992 } else if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1993 /* In ccm, the IV needs to exist both inside B0 and 1994 * inside the counter.It is also copied to iv_dma_addr 1995 * for other reasons (like returning it to the user). 1996 * So, using 3 (identical) IV outputs. 1997 */ 1998 cc_req.ivgen_dma_addr[0] = 1999 areq_ctx->gen_ctx.iv_dma_addr + 2000 CCM_BLOCK_IV_OFFSET; 2001 cc_req.ivgen_dma_addr[1] = 2002 sg_dma_address(&areq_ctx->ccm_adata_sg) + 2003 CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET; 2004 cc_req.ivgen_dma_addr[2] = 2005 sg_dma_address(&areq_ctx->ccm_adata_sg) + 2006 CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET; 2007 cc_req.ivgen_dma_addr_len = 3; 2008 } else { 2009 cc_req.ivgen_dma_addr[0] = 2010 areq_ctx->gen_ctx.iv_dma_addr; 2011 cc_req.ivgen_dma_addr_len = 1; 2012 } 2013 2014 /* set the IV size (8/16 B long)*/ 2015 cc_req.ivgen_size = crypto_aead_ivsize(tfm); 2016 } 2017 2018 /* STAT_PHASE_2: Create sequence */ 2019 2020 /* Load MLLI tables to SRAM if necessary */ 2021 cc_mlli_to_sram(req, desc, &seq_len); 2022 2023 /*TODO: move seq len by reference */ 2024 switch (ctx->auth_mode) { 2025 case DRV_HASH_SHA1: 2026 case DRV_HASH_SHA256: 2027 cc_hmac_authenc(req, desc, &seq_len); 2028 break; 2029 case DRV_HASH_XCBC_MAC: 2030 cc_xcbc_authenc(req, desc, &seq_len); 2031 break; 2032 case DRV_HASH_NULL: 2033 if (ctx->cipher_mode == DRV_CIPHER_CCM) 2034 cc_ccm(req, desc, &seq_len); 2035 if (ctx->cipher_mode == DRV_CIPHER_GCTR) 2036 cc_gcm(req, desc, &seq_len); 2037 break; 2038 default: 2039 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2040 cc_unmap_aead_request(dev, req); 2041 rc = -ENOTSUPP; 2042 goto exit; 2043 } 2044 2045 /* STAT_PHASE_3: Lock HW and push sequence */ 2046 2047 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2048 2049 if (rc != -EINPROGRESS && rc != -EBUSY) { 2050 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2051 cc_unmap_aead_request(dev, req); 2052 } 2053 2054 exit: 2055 return rc; 2056 } 2057 2058 static int cc_aead_encrypt(struct aead_request *req) 2059 { 2060 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2061 int rc; 2062 2063 /* No generated IV required */ 2064 areq_ctx->backup_iv = req->iv; 2065 areq_ctx->backup_giv = NULL; 2066 areq_ctx->is_gcm4543 = false; 2067 2068 areq_ctx->plaintext_authenticate_only = false; 2069 2070 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2071 if (rc != -EINPROGRESS && rc != -EBUSY) 2072 req->iv = areq_ctx->backup_iv; 2073 2074 return rc; 2075 } 2076 2077 static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2078 { 2079 /* Very similar to cc_aead_encrypt() above. */ 2080 2081 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2082 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2083 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2084 struct device *dev = drvdata_to_dev(ctx->drvdata); 2085 int rc = -EINVAL; 2086 2087 if (!valid_assoclen(req)) { 2088 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2089 goto out; 2090 } 2091 2092 /* No generated IV required */ 2093 areq_ctx->backup_iv = req->iv; 2094 areq_ctx->backup_giv = NULL; 2095 areq_ctx->is_gcm4543 = true; 2096 2097 cc_proc_rfc4309_ccm(req); 2098 2099 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2100 if (rc != -EINPROGRESS && rc != -EBUSY) 2101 req->iv = areq_ctx->backup_iv; 2102 out: 2103 return rc; 2104 } 2105 2106 static int cc_aead_decrypt(struct aead_request *req) 2107 { 2108 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2109 int rc; 2110 2111 /* No generated IV required */ 2112 areq_ctx->backup_iv = req->iv; 2113 areq_ctx->backup_giv = NULL; 2114 areq_ctx->is_gcm4543 = false; 2115 2116 areq_ctx->plaintext_authenticate_only = false; 2117 2118 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2119 if (rc != -EINPROGRESS && rc != -EBUSY) 2120 req->iv = areq_ctx->backup_iv; 2121 2122 return rc; 2123 } 2124 2125 static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2126 { 2127 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2129 struct device *dev = drvdata_to_dev(ctx->drvdata); 2130 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2131 int rc = -EINVAL; 2132 2133 if (!valid_assoclen(req)) { 2134 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2135 goto out; 2136 } 2137 2138 /* No generated IV required */ 2139 areq_ctx->backup_iv = req->iv; 2140 areq_ctx->backup_giv = NULL; 2141 2142 areq_ctx->is_gcm4543 = true; 2143 cc_proc_rfc4309_ccm(req); 2144 2145 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2146 if (rc != -EINPROGRESS && rc != -EBUSY) 2147 req->iv = areq_ctx->backup_iv; 2148 2149 out: 2150 return rc; 2151 } 2152 2153 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2154 unsigned int keylen) 2155 { 2156 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2157 struct device *dev = drvdata_to_dev(ctx->drvdata); 2158 2159 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2160 2161 if (keylen < 4) 2162 return -EINVAL; 2163 2164 keylen -= 4; 2165 memcpy(ctx->ctr_nonce, key + keylen, 4); 2166 2167 return cc_aead_setkey(tfm, key, keylen); 2168 } 2169 2170 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2171 unsigned int keylen) 2172 { 2173 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2174 struct device *dev = drvdata_to_dev(ctx->drvdata); 2175 2176 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2177 2178 if (keylen < 4) 2179 return -EINVAL; 2180 2181 keylen -= 4; 2182 memcpy(ctx->ctr_nonce, key + keylen, 4); 2183 2184 return cc_aead_setkey(tfm, key, keylen); 2185 } 2186 2187 static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2188 unsigned int authsize) 2189 { 2190 switch (authsize) { 2191 case 4: 2192 case 8: 2193 case 12: 2194 case 13: 2195 case 14: 2196 case 15: 2197 case 16: 2198 break; 2199 default: 2200 return -EINVAL; 2201 } 2202 2203 return cc_aead_setauthsize(authenc, authsize); 2204 } 2205 2206 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2207 unsigned int authsize) 2208 { 2209 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2210 struct device *dev = drvdata_to_dev(ctx->drvdata); 2211 2212 dev_dbg(dev, "authsize %d\n", authsize); 2213 2214 switch (authsize) { 2215 case 8: 2216 case 12: 2217 case 16: 2218 break; 2219 default: 2220 return -EINVAL; 2221 } 2222 2223 return cc_aead_setauthsize(authenc, authsize); 2224 } 2225 2226 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2227 unsigned int authsize) 2228 { 2229 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2230 struct device *dev = drvdata_to_dev(ctx->drvdata); 2231 2232 dev_dbg(dev, "authsize %d\n", authsize); 2233 2234 if (authsize != 16) 2235 return -EINVAL; 2236 2237 return cc_aead_setauthsize(authenc, authsize); 2238 } 2239 2240 static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2241 { 2242 /* Very similar to cc_aead_encrypt() above. */ 2243 2244 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2245 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2246 struct device *dev = drvdata_to_dev(ctx->drvdata); 2247 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2248 int rc = -EINVAL; 2249 2250 if (!valid_assoclen(req)) { 2251 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2252 goto out; 2253 } 2254 2255 /* No generated IV required */ 2256 areq_ctx->backup_iv = req->iv; 2257 areq_ctx->backup_giv = NULL; 2258 2259 areq_ctx->plaintext_authenticate_only = false; 2260 2261 cc_proc_rfc4_gcm(req); 2262 areq_ctx->is_gcm4543 = true; 2263 2264 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2265 if (rc != -EINPROGRESS && rc != -EBUSY) 2266 req->iv = areq_ctx->backup_iv; 2267 out: 2268 return rc; 2269 } 2270 2271 static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2272 { 2273 /* Very similar to cc_aead_encrypt() above. */ 2274 2275 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2276 int rc; 2277 2278 //plaintext is not encryped with rfc4543 2279 areq_ctx->plaintext_authenticate_only = true; 2280 2281 /* No generated IV required */ 2282 areq_ctx->backup_iv = req->iv; 2283 areq_ctx->backup_giv = NULL; 2284 2285 cc_proc_rfc4_gcm(req); 2286 areq_ctx->is_gcm4543 = true; 2287 2288 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2289 if (rc != -EINPROGRESS && rc != -EBUSY) 2290 req->iv = areq_ctx->backup_iv; 2291 2292 return rc; 2293 } 2294 2295 static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2296 { 2297 /* Very similar to cc_aead_decrypt() above. */ 2298 2299 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2300 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2301 struct device *dev = drvdata_to_dev(ctx->drvdata); 2302 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2303 int rc = -EINVAL; 2304 2305 if (!valid_assoclen(req)) { 2306 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2307 goto out; 2308 } 2309 2310 /* No generated IV required */ 2311 areq_ctx->backup_iv = req->iv; 2312 areq_ctx->backup_giv = NULL; 2313 2314 areq_ctx->plaintext_authenticate_only = false; 2315 2316 cc_proc_rfc4_gcm(req); 2317 areq_ctx->is_gcm4543 = true; 2318 2319 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2320 if (rc != -EINPROGRESS && rc != -EBUSY) 2321 req->iv = areq_ctx->backup_iv; 2322 out: 2323 return rc; 2324 } 2325 2326 static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2327 { 2328 /* Very similar to cc_aead_decrypt() above. */ 2329 2330 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2331 int rc; 2332 2333 //plaintext is not decryped with rfc4543 2334 areq_ctx->plaintext_authenticate_only = true; 2335 2336 /* No generated IV required */ 2337 areq_ctx->backup_iv = req->iv; 2338 areq_ctx->backup_giv = NULL; 2339 2340 cc_proc_rfc4_gcm(req); 2341 areq_ctx->is_gcm4543 = true; 2342 2343 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2344 if (rc != -EINPROGRESS && rc != -EBUSY) 2345 req->iv = areq_ctx->backup_iv; 2346 2347 return rc; 2348 } 2349 2350 /* aead alg */ 2351 static struct cc_alg_template aead_algs[] = { 2352 { 2353 .name = "authenc(hmac(sha1),cbc(aes))", 2354 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2355 .blocksize = AES_BLOCK_SIZE, 2356 .template_aead = { 2357 .setkey = cc_aead_setkey, 2358 .setauthsize = cc_aead_setauthsize, 2359 .encrypt = cc_aead_encrypt, 2360 .decrypt = cc_aead_decrypt, 2361 .init = cc_aead_init, 2362 .exit = cc_aead_exit, 2363 .ivsize = AES_BLOCK_SIZE, 2364 .maxauthsize = SHA1_DIGEST_SIZE, 2365 }, 2366 .cipher_mode = DRV_CIPHER_CBC, 2367 .flow_mode = S_DIN_to_AES, 2368 .auth_mode = DRV_HASH_SHA1, 2369 .min_hw_rev = CC_HW_REV_630, 2370 .std_body = CC_STD_NIST, 2371 }, 2372 { 2373 .name = "authenc(hmac(sha1),cbc(des3_ede))", 2374 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2375 .blocksize = DES3_EDE_BLOCK_SIZE, 2376 .template_aead = { 2377 .setkey = cc_aead_setkey, 2378 .setauthsize = cc_aead_setauthsize, 2379 .encrypt = cc_aead_encrypt, 2380 .decrypt = cc_aead_decrypt, 2381 .init = cc_aead_init, 2382 .exit = cc_aead_exit, 2383 .ivsize = DES3_EDE_BLOCK_SIZE, 2384 .maxauthsize = SHA1_DIGEST_SIZE, 2385 }, 2386 .cipher_mode = DRV_CIPHER_CBC, 2387 .flow_mode = S_DIN_to_DES, 2388 .auth_mode = DRV_HASH_SHA1, 2389 .min_hw_rev = CC_HW_REV_630, 2390 .std_body = CC_STD_NIST, 2391 }, 2392 { 2393 .name = "authenc(hmac(sha256),cbc(aes))", 2394 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2395 .blocksize = AES_BLOCK_SIZE, 2396 .template_aead = { 2397 .setkey = cc_aead_setkey, 2398 .setauthsize = cc_aead_setauthsize, 2399 .encrypt = cc_aead_encrypt, 2400 .decrypt = cc_aead_decrypt, 2401 .init = cc_aead_init, 2402 .exit = cc_aead_exit, 2403 .ivsize = AES_BLOCK_SIZE, 2404 .maxauthsize = SHA256_DIGEST_SIZE, 2405 }, 2406 .cipher_mode = DRV_CIPHER_CBC, 2407 .flow_mode = S_DIN_to_AES, 2408 .auth_mode = DRV_HASH_SHA256, 2409 .min_hw_rev = CC_HW_REV_630, 2410 .std_body = CC_STD_NIST, 2411 }, 2412 { 2413 .name = "authenc(hmac(sha256),cbc(des3_ede))", 2414 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2415 .blocksize = DES3_EDE_BLOCK_SIZE, 2416 .template_aead = { 2417 .setkey = cc_aead_setkey, 2418 .setauthsize = cc_aead_setauthsize, 2419 .encrypt = cc_aead_encrypt, 2420 .decrypt = cc_aead_decrypt, 2421 .init = cc_aead_init, 2422 .exit = cc_aead_exit, 2423 .ivsize = DES3_EDE_BLOCK_SIZE, 2424 .maxauthsize = SHA256_DIGEST_SIZE, 2425 }, 2426 .cipher_mode = DRV_CIPHER_CBC, 2427 .flow_mode = S_DIN_to_DES, 2428 .auth_mode = DRV_HASH_SHA256, 2429 .min_hw_rev = CC_HW_REV_630, 2430 .std_body = CC_STD_NIST, 2431 }, 2432 { 2433 .name = "authenc(xcbc(aes),cbc(aes))", 2434 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2435 .blocksize = AES_BLOCK_SIZE, 2436 .template_aead = { 2437 .setkey = cc_aead_setkey, 2438 .setauthsize = cc_aead_setauthsize, 2439 .encrypt = cc_aead_encrypt, 2440 .decrypt = cc_aead_decrypt, 2441 .init = cc_aead_init, 2442 .exit = cc_aead_exit, 2443 .ivsize = AES_BLOCK_SIZE, 2444 .maxauthsize = AES_BLOCK_SIZE, 2445 }, 2446 .cipher_mode = DRV_CIPHER_CBC, 2447 .flow_mode = S_DIN_to_AES, 2448 .auth_mode = DRV_HASH_XCBC_MAC, 2449 .min_hw_rev = CC_HW_REV_630, 2450 .std_body = CC_STD_NIST, 2451 }, 2452 { 2453 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2454 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2455 .blocksize = 1, 2456 .template_aead = { 2457 .setkey = cc_aead_setkey, 2458 .setauthsize = cc_aead_setauthsize, 2459 .encrypt = cc_aead_encrypt, 2460 .decrypt = cc_aead_decrypt, 2461 .init = cc_aead_init, 2462 .exit = cc_aead_exit, 2463 .ivsize = CTR_RFC3686_IV_SIZE, 2464 .maxauthsize = SHA1_DIGEST_SIZE, 2465 }, 2466 .cipher_mode = DRV_CIPHER_CTR, 2467 .flow_mode = S_DIN_to_AES, 2468 .auth_mode = DRV_HASH_SHA1, 2469 .min_hw_rev = CC_HW_REV_630, 2470 .std_body = CC_STD_NIST, 2471 }, 2472 { 2473 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2474 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2475 .blocksize = 1, 2476 .template_aead = { 2477 .setkey = cc_aead_setkey, 2478 .setauthsize = cc_aead_setauthsize, 2479 .encrypt = cc_aead_encrypt, 2480 .decrypt = cc_aead_decrypt, 2481 .init = cc_aead_init, 2482 .exit = cc_aead_exit, 2483 .ivsize = CTR_RFC3686_IV_SIZE, 2484 .maxauthsize = SHA256_DIGEST_SIZE, 2485 }, 2486 .cipher_mode = DRV_CIPHER_CTR, 2487 .flow_mode = S_DIN_to_AES, 2488 .auth_mode = DRV_HASH_SHA256, 2489 .min_hw_rev = CC_HW_REV_630, 2490 .std_body = CC_STD_NIST, 2491 }, 2492 { 2493 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2494 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2495 .blocksize = 1, 2496 .template_aead = { 2497 .setkey = cc_aead_setkey, 2498 .setauthsize = cc_aead_setauthsize, 2499 .encrypt = cc_aead_encrypt, 2500 .decrypt = cc_aead_decrypt, 2501 .init = cc_aead_init, 2502 .exit = cc_aead_exit, 2503 .ivsize = CTR_RFC3686_IV_SIZE, 2504 .maxauthsize = AES_BLOCK_SIZE, 2505 }, 2506 .cipher_mode = DRV_CIPHER_CTR, 2507 .flow_mode = S_DIN_to_AES, 2508 .auth_mode = DRV_HASH_XCBC_MAC, 2509 .min_hw_rev = CC_HW_REV_630, 2510 .std_body = CC_STD_NIST, 2511 }, 2512 { 2513 .name = "ccm(aes)", 2514 .driver_name = "ccm-aes-ccree", 2515 .blocksize = 1, 2516 .template_aead = { 2517 .setkey = cc_aead_setkey, 2518 .setauthsize = cc_ccm_setauthsize, 2519 .encrypt = cc_aead_encrypt, 2520 .decrypt = cc_aead_decrypt, 2521 .init = cc_aead_init, 2522 .exit = cc_aead_exit, 2523 .ivsize = AES_BLOCK_SIZE, 2524 .maxauthsize = AES_BLOCK_SIZE, 2525 }, 2526 .cipher_mode = DRV_CIPHER_CCM, 2527 .flow_mode = S_DIN_to_AES, 2528 .auth_mode = DRV_HASH_NULL, 2529 .min_hw_rev = CC_HW_REV_630, 2530 .std_body = CC_STD_NIST, 2531 }, 2532 { 2533 .name = "rfc4309(ccm(aes))", 2534 .driver_name = "rfc4309-ccm-aes-ccree", 2535 .blocksize = 1, 2536 .template_aead = { 2537 .setkey = cc_rfc4309_ccm_setkey, 2538 .setauthsize = cc_rfc4309_ccm_setauthsize, 2539 .encrypt = cc_rfc4309_ccm_encrypt, 2540 .decrypt = cc_rfc4309_ccm_decrypt, 2541 .init = cc_aead_init, 2542 .exit = cc_aead_exit, 2543 .ivsize = CCM_BLOCK_IV_SIZE, 2544 .maxauthsize = AES_BLOCK_SIZE, 2545 }, 2546 .cipher_mode = DRV_CIPHER_CCM, 2547 .flow_mode = S_DIN_to_AES, 2548 .auth_mode = DRV_HASH_NULL, 2549 .min_hw_rev = CC_HW_REV_630, 2550 .std_body = CC_STD_NIST, 2551 }, 2552 { 2553 .name = "gcm(aes)", 2554 .driver_name = "gcm-aes-ccree", 2555 .blocksize = 1, 2556 .template_aead = { 2557 .setkey = cc_aead_setkey, 2558 .setauthsize = cc_gcm_setauthsize, 2559 .encrypt = cc_aead_encrypt, 2560 .decrypt = cc_aead_decrypt, 2561 .init = cc_aead_init, 2562 .exit = cc_aead_exit, 2563 .ivsize = 12, 2564 .maxauthsize = AES_BLOCK_SIZE, 2565 }, 2566 .cipher_mode = DRV_CIPHER_GCTR, 2567 .flow_mode = S_DIN_to_AES, 2568 .auth_mode = DRV_HASH_NULL, 2569 .min_hw_rev = CC_HW_REV_630, 2570 .std_body = CC_STD_NIST, 2571 }, 2572 { 2573 .name = "rfc4106(gcm(aes))", 2574 .driver_name = "rfc4106-gcm-aes-ccree", 2575 .blocksize = 1, 2576 .template_aead = { 2577 .setkey = cc_rfc4106_gcm_setkey, 2578 .setauthsize = cc_rfc4106_gcm_setauthsize, 2579 .encrypt = cc_rfc4106_gcm_encrypt, 2580 .decrypt = cc_rfc4106_gcm_decrypt, 2581 .init = cc_aead_init, 2582 .exit = cc_aead_exit, 2583 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2584 .maxauthsize = AES_BLOCK_SIZE, 2585 }, 2586 .cipher_mode = DRV_CIPHER_GCTR, 2587 .flow_mode = S_DIN_to_AES, 2588 .auth_mode = DRV_HASH_NULL, 2589 .min_hw_rev = CC_HW_REV_630, 2590 .std_body = CC_STD_NIST, 2591 }, 2592 { 2593 .name = "rfc4543(gcm(aes))", 2594 .driver_name = "rfc4543-gcm-aes-ccree", 2595 .blocksize = 1, 2596 .template_aead = { 2597 .setkey = cc_rfc4543_gcm_setkey, 2598 .setauthsize = cc_rfc4543_gcm_setauthsize, 2599 .encrypt = cc_rfc4543_gcm_encrypt, 2600 .decrypt = cc_rfc4543_gcm_decrypt, 2601 .init = cc_aead_init, 2602 .exit = cc_aead_exit, 2603 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2604 .maxauthsize = AES_BLOCK_SIZE, 2605 }, 2606 .cipher_mode = DRV_CIPHER_GCTR, 2607 .flow_mode = S_DIN_to_AES, 2608 .auth_mode = DRV_HASH_NULL, 2609 .min_hw_rev = CC_HW_REV_630, 2610 .std_body = CC_STD_NIST, 2611 }, 2612 }; 2613 2614 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2615 struct device *dev) 2616 { 2617 struct cc_crypto_alg *t_alg; 2618 struct aead_alg *alg; 2619 2620 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2621 if (!t_alg) 2622 return ERR_PTR(-ENOMEM); 2623 2624 alg = &tmpl->template_aead; 2625 2626 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 2627 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2628 tmpl->driver_name); 2629 alg->base.cra_module = THIS_MODULE; 2630 alg->base.cra_priority = CC_CRA_PRIO; 2631 2632 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 2633 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2634 alg->init = cc_aead_init; 2635 alg->exit = cc_aead_exit; 2636 2637 t_alg->aead_alg = *alg; 2638 2639 t_alg->cipher_mode = tmpl->cipher_mode; 2640 t_alg->flow_mode = tmpl->flow_mode; 2641 t_alg->auth_mode = tmpl->auth_mode; 2642 2643 return t_alg; 2644 } 2645 2646 int cc_aead_free(struct cc_drvdata *drvdata) 2647 { 2648 struct cc_crypto_alg *t_alg, *n; 2649 struct cc_aead_handle *aead_handle = 2650 (struct cc_aead_handle *)drvdata->aead_handle; 2651 2652 if (aead_handle) { 2653 /* Remove registered algs */ 2654 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, 2655 entry) { 2656 crypto_unregister_aead(&t_alg->aead_alg); 2657 list_del(&t_alg->entry); 2658 kfree(t_alg); 2659 } 2660 kfree(aead_handle); 2661 drvdata->aead_handle = NULL; 2662 } 2663 2664 return 0; 2665 } 2666 2667 int cc_aead_alloc(struct cc_drvdata *drvdata) 2668 { 2669 struct cc_aead_handle *aead_handle; 2670 struct cc_crypto_alg *t_alg; 2671 int rc = -ENOMEM; 2672 int alg; 2673 struct device *dev = drvdata_to_dev(drvdata); 2674 2675 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL); 2676 if (!aead_handle) { 2677 rc = -ENOMEM; 2678 goto fail0; 2679 } 2680 2681 INIT_LIST_HEAD(&aead_handle->aead_list); 2682 drvdata->aead_handle = aead_handle; 2683 2684 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2685 MAX_HMAC_DIGEST_SIZE); 2686 2687 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2688 dev_err(dev, "SRAM pool exhausted\n"); 2689 rc = -ENOMEM; 2690 goto fail1; 2691 } 2692 2693 /* Linux crypto */ 2694 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 2695 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 2696 !(drvdata->std_bodies & aead_algs[alg].std_body)) 2697 continue; 2698 2699 t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2700 if (IS_ERR(t_alg)) { 2701 rc = PTR_ERR(t_alg); 2702 dev_err(dev, "%s alg allocation failed\n", 2703 aead_algs[alg].driver_name); 2704 goto fail1; 2705 } 2706 t_alg->drvdata = drvdata; 2707 rc = crypto_register_aead(&t_alg->aead_alg); 2708 if (rc) { 2709 dev_err(dev, "%s alg registration failed\n", 2710 t_alg->aead_alg.base.cra_driver_name); 2711 goto fail2; 2712 } else { 2713 list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2714 dev_dbg(dev, "Registered %s\n", 2715 t_alg->aead_alg.base.cra_driver_name); 2716 } 2717 } 2718 2719 return 0; 2720 2721 fail2: 2722 kfree(t_alg); 2723 fail1: 2724 cc_aead_free(drvdata); 2725 fail0: 2726 return rc; 2727 } 2728