1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/internal/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/internal/des.h> 10 #include <linux/rtnetlink.h> 11 #include "cc_driver.h" 12 #include "cc_buffer_mgr.h" 13 #include "cc_aead.h" 14 #include "cc_request_mgr.h" 15 #include "cc_hash.h" 16 #include "cc_sram_mgr.h" 17 18 #define template_aead template_u.aead 19 20 #define MAX_AEAD_SETKEY_SEQ 12 21 #define MAX_AEAD_PROCESS_SEQ 23 22 23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 25 26 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 27 28 struct cc_aead_handle { 29 cc_sram_addr_t sram_workspace_addr; 30 struct list_head aead_list; 31 }; 32 33 struct cc_hmac_s { 34 u8 *padded_authkey; 35 u8 *ipad_opad; /* IPAD, OPAD*/ 36 dma_addr_t padded_authkey_dma_addr; 37 dma_addr_t ipad_opad_dma_addr; 38 }; 39 40 struct cc_xcbc_s { 41 u8 *xcbc_keys; /* K1,K2,K3 */ 42 dma_addr_t xcbc_keys_dma_addr; 43 }; 44 45 struct cc_aead_ctx { 46 struct cc_drvdata *drvdata; 47 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 48 u8 *enckey; 49 dma_addr_t enckey_dma_addr; 50 union { 51 struct cc_hmac_s hmac; 52 struct cc_xcbc_s xcbc; 53 } auth_state; 54 unsigned int enc_keylen; 55 unsigned int auth_keylen; 56 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 57 unsigned int hash_len; 58 enum drv_cipher_mode cipher_mode; 59 enum cc_flow_mode flow_mode; 60 enum drv_hash_mode auth_mode; 61 }; 62 63 static inline bool valid_assoclen(struct aead_request *req) 64 { 65 return ((req->assoclen == 16) || (req->assoclen == 20)); 66 } 67 68 static void cc_aead_exit(struct crypto_aead *tfm) 69 { 70 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 71 struct device *dev = drvdata_to_dev(ctx->drvdata); 72 73 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 74 crypto_tfm_alg_name(&tfm->base)); 75 76 /* Unmap enckey buffer */ 77 if (ctx->enckey) { 78 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 79 ctx->enckey_dma_addr); 80 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 81 &ctx->enckey_dma_addr); 82 ctx->enckey_dma_addr = 0; 83 ctx->enckey = NULL; 84 } 85 86 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 87 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 88 89 if (xcbc->xcbc_keys) { 90 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 91 xcbc->xcbc_keys, 92 xcbc->xcbc_keys_dma_addr); 93 } 94 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 95 &xcbc->xcbc_keys_dma_addr); 96 xcbc->xcbc_keys_dma_addr = 0; 97 xcbc->xcbc_keys = NULL; 98 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 99 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 100 101 if (hmac->ipad_opad) { 102 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 103 hmac->ipad_opad, 104 hmac->ipad_opad_dma_addr); 105 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 106 &hmac->ipad_opad_dma_addr); 107 hmac->ipad_opad_dma_addr = 0; 108 hmac->ipad_opad = NULL; 109 } 110 if (hmac->padded_authkey) { 111 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 112 hmac->padded_authkey, 113 hmac->padded_authkey_dma_addr); 114 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 115 &hmac->padded_authkey_dma_addr); 116 hmac->padded_authkey_dma_addr = 0; 117 hmac->padded_authkey = NULL; 118 } 119 } 120 } 121 122 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 123 { 124 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 125 126 return cc_get_default_hash_len(ctx->drvdata); 127 } 128 129 static int cc_aead_init(struct crypto_aead *tfm) 130 { 131 struct aead_alg *alg = crypto_aead_alg(tfm); 132 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 133 struct cc_crypto_alg *cc_alg = 134 container_of(alg, struct cc_crypto_alg, aead_alg); 135 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 136 137 dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 138 crypto_tfm_alg_name(&tfm->base)); 139 140 /* Initialize modes in instance */ 141 ctx->cipher_mode = cc_alg->cipher_mode; 142 ctx->flow_mode = cc_alg->flow_mode; 143 ctx->auth_mode = cc_alg->auth_mode; 144 ctx->drvdata = cc_alg->drvdata; 145 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx)); 146 147 /* Allocate key buffer, cache line aligned */ 148 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 149 &ctx->enckey_dma_addr, GFP_KERNEL); 150 if (!ctx->enckey) { 151 dev_err(dev, "Failed allocating key buffer\n"); 152 goto init_failed; 153 } 154 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 155 ctx->enckey); 156 157 /* Set default authlen value */ 158 159 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 160 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 161 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 162 163 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 164 /* (and temporary for user key - up to 256b) */ 165 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 166 &xcbc->xcbc_keys_dma_addr, 167 GFP_KERNEL); 168 if (!xcbc->xcbc_keys) { 169 dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 170 goto init_failed; 171 } 172 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 173 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 174 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 175 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 176 177 /* Allocate dma-coherent buffer for IPAD + OPAD */ 178 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 179 &hmac->ipad_opad_dma_addr, 180 GFP_KERNEL); 181 182 if (!hmac->ipad_opad) { 183 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 184 goto init_failed; 185 } 186 187 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 188 hmac->ipad_opad); 189 190 hmac->padded_authkey = dma_alloc_coherent(dev, 191 MAX_HMAC_BLOCK_SIZE, 192 pkey_dma, 193 GFP_KERNEL); 194 195 if (!hmac->padded_authkey) { 196 dev_err(dev, "failed to allocate padded_authkey\n"); 197 goto init_failed; 198 } 199 } else { 200 ctx->auth_state.hmac.ipad_opad = NULL; 201 ctx->auth_state.hmac.padded_authkey = NULL; 202 } 203 ctx->hash_len = cc_get_aead_hash_len(tfm); 204 205 return 0; 206 207 init_failed: 208 cc_aead_exit(tfm); 209 return -ENOMEM; 210 } 211 212 static void cc_aead_complete(struct device *dev, void *cc_req, int err) 213 { 214 struct aead_request *areq = (struct aead_request *)cc_req; 215 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 216 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 217 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 218 219 /* BACKLOG notification */ 220 if (err == -EINPROGRESS) 221 goto done; 222 223 cc_unmap_aead_request(dev, areq); 224 225 /* Restore ordinary iv pointer */ 226 areq->iv = areq_ctx->backup_iv; 227 228 if (err) 229 goto done; 230 231 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 232 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 233 ctx->authsize) != 0) { 234 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 235 ctx->authsize, ctx->cipher_mode); 236 /* In case of payload authentication failure, MUST NOT 237 * revealed the decrypted message --> zero its memory. 238 */ 239 sg_zero_buffer(areq->dst, sg_nents(areq->dst), 240 areq->cryptlen, areq->assoclen); 241 err = -EBADMSG; 242 } 243 /*ENCRYPT*/ 244 } else if (areq_ctx->is_icv_fragmented) { 245 u32 skip = areq->cryptlen + areq_ctx->dst_offset; 246 247 cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl, 248 skip, (skip + ctx->authsize), 249 CC_SG_FROM_BUF); 250 } 251 done: 252 aead_request_complete(areq, err); 253 } 254 255 static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 256 struct cc_aead_ctx *ctx) 257 { 258 /* Load the AES key */ 259 hw_desc_init(&desc[0]); 260 /* We are using for the source/user key the same buffer 261 * as for the output keys, * because after this key loading it 262 * is not needed anymore 263 */ 264 set_din_type(&desc[0], DMA_DLLI, 265 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 266 NS_BIT); 267 set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 268 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 269 set_key_size_aes(&desc[0], ctx->auth_keylen); 270 set_flow_mode(&desc[0], S_DIN_to_AES); 271 set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 272 273 hw_desc_init(&desc[1]); 274 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 275 set_flow_mode(&desc[1], DIN_AES_DOUT); 276 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 277 AES_KEYSIZE_128, NS_BIT, 0); 278 279 hw_desc_init(&desc[2]); 280 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 281 set_flow_mode(&desc[2], DIN_AES_DOUT); 282 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 283 + AES_KEYSIZE_128), 284 AES_KEYSIZE_128, NS_BIT, 0); 285 286 hw_desc_init(&desc[3]); 287 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 288 set_flow_mode(&desc[3], DIN_AES_DOUT); 289 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 290 + 2 * AES_KEYSIZE_128), 291 AES_KEYSIZE_128, NS_BIT, 0); 292 293 return 4; 294 } 295 296 static unsigned int hmac_setkey(struct cc_hw_desc *desc, 297 struct cc_aead_ctx *ctx) 298 { 299 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 300 unsigned int digest_ofs = 0; 301 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 302 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 303 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 304 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 305 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 306 307 unsigned int idx = 0; 308 int i; 309 310 /* calc derived HMAC key */ 311 for (i = 0; i < 2; i++) { 312 /* Load hash initial state */ 313 hw_desc_init(&desc[idx]); 314 set_cipher_mode(&desc[idx], hash_mode); 315 set_din_sram(&desc[idx], 316 cc_larval_digest_addr(ctx->drvdata, 317 ctx->auth_mode), 318 digest_size); 319 set_flow_mode(&desc[idx], S_DIN_to_HASH); 320 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 321 idx++; 322 323 /* Load the hash current length*/ 324 hw_desc_init(&desc[idx]); 325 set_cipher_mode(&desc[idx], hash_mode); 326 set_din_const(&desc[idx], 0, ctx->hash_len); 327 set_flow_mode(&desc[idx], S_DIN_to_HASH); 328 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 329 idx++; 330 331 /* Prepare ipad key */ 332 hw_desc_init(&desc[idx]); 333 set_xor_val(&desc[idx], hmac_pad_const[i]); 334 set_cipher_mode(&desc[idx], hash_mode); 335 set_flow_mode(&desc[idx], S_DIN_to_HASH); 336 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 337 idx++; 338 339 /* Perform HASH update */ 340 hw_desc_init(&desc[idx]); 341 set_din_type(&desc[idx], DMA_DLLI, 342 hmac->padded_authkey_dma_addr, 343 SHA256_BLOCK_SIZE, NS_BIT); 344 set_cipher_mode(&desc[idx], hash_mode); 345 set_xor_active(&desc[idx]); 346 set_flow_mode(&desc[idx], DIN_HASH); 347 idx++; 348 349 /* Get the digset */ 350 hw_desc_init(&desc[idx]); 351 set_cipher_mode(&desc[idx], hash_mode); 352 set_dout_dlli(&desc[idx], 353 (hmac->ipad_opad_dma_addr + digest_ofs), 354 digest_size, NS_BIT, 0); 355 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 356 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 357 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 358 idx++; 359 360 digest_ofs += digest_size; 361 } 362 363 return idx; 364 } 365 366 static int validate_keys_sizes(struct cc_aead_ctx *ctx) 367 { 368 struct device *dev = drvdata_to_dev(ctx->drvdata); 369 370 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 371 ctx->enc_keylen, ctx->auth_keylen); 372 373 switch (ctx->auth_mode) { 374 case DRV_HASH_SHA1: 375 case DRV_HASH_SHA256: 376 break; 377 case DRV_HASH_XCBC_MAC: 378 if (ctx->auth_keylen != AES_KEYSIZE_128 && 379 ctx->auth_keylen != AES_KEYSIZE_192 && 380 ctx->auth_keylen != AES_KEYSIZE_256) 381 return -ENOTSUPP; 382 break; 383 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 384 if (ctx->auth_keylen > 0) 385 return -EINVAL; 386 break; 387 default: 388 dev_dbg(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 389 return -EINVAL; 390 } 391 /* Check cipher key size */ 392 if (ctx->flow_mode == S_DIN_to_DES) { 393 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 394 dev_dbg(dev, "Invalid cipher(3DES) key size: %u\n", 395 ctx->enc_keylen); 396 return -EINVAL; 397 } 398 } else { /* Default assumed to be AES ciphers */ 399 if (ctx->enc_keylen != AES_KEYSIZE_128 && 400 ctx->enc_keylen != AES_KEYSIZE_192 && 401 ctx->enc_keylen != AES_KEYSIZE_256) { 402 dev_dbg(dev, "Invalid cipher(AES) key size: %u\n", 403 ctx->enc_keylen); 404 return -EINVAL; 405 } 406 } 407 408 return 0; /* All tests of keys sizes passed */ 409 } 410 411 /* This function prepers the user key so it can pass to the hmac processing 412 * (copy to intenral buffer or hash in case of key longer than block 413 */ 414 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, 415 unsigned int keylen) 416 { 417 dma_addr_t key_dma_addr = 0; 418 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 419 struct device *dev = drvdata_to_dev(ctx->drvdata); 420 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode); 421 struct cc_crypto_req cc_req = {}; 422 unsigned int blocksize; 423 unsigned int digestsize; 424 unsigned int hashmode; 425 unsigned int idx = 0; 426 int rc = 0; 427 u8 *key = NULL; 428 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 429 dma_addr_t padded_authkey_dma_addr = 430 ctx->auth_state.hmac.padded_authkey_dma_addr; 431 432 switch (ctx->auth_mode) { /* auth_key required and >0 */ 433 case DRV_HASH_SHA1: 434 blocksize = SHA1_BLOCK_SIZE; 435 digestsize = SHA1_DIGEST_SIZE; 436 hashmode = DRV_HASH_HW_SHA1; 437 break; 438 case DRV_HASH_SHA256: 439 default: 440 blocksize = SHA256_BLOCK_SIZE; 441 digestsize = SHA256_DIGEST_SIZE; 442 hashmode = DRV_HASH_HW_SHA256; 443 } 444 445 if (keylen != 0) { 446 447 key = kmemdup(authkey, keylen, GFP_KERNEL); 448 if (!key) 449 return -ENOMEM; 450 451 key_dma_addr = dma_map_single(dev, (void *)key, keylen, 452 DMA_TO_DEVICE); 453 if (dma_mapping_error(dev, key_dma_addr)) { 454 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 455 key, keylen); 456 kzfree(key); 457 return -ENOMEM; 458 } 459 if (keylen > blocksize) { 460 /* Load hash initial state */ 461 hw_desc_init(&desc[idx]); 462 set_cipher_mode(&desc[idx], hashmode); 463 set_din_sram(&desc[idx], larval_addr, digestsize); 464 set_flow_mode(&desc[idx], S_DIN_to_HASH); 465 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 466 idx++; 467 468 /* Load the hash current length*/ 469 hw_desc_init(&desc[idx]); 470 set_cipher_mode(&desc[idx], hashmode); 471 set_din_const(&desc[idx], 0, ctx->hash_len); 472 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 473 set_flow_mode(&desc[idx], S_DIN_to_HASH); 474 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 475 idx++; 476 477 hw_desc_init(&desc[idx]); 478 set_din_type(&desc[idx], DMA_DLLI, 479 key_dma_addr, keylen, NS_BIT); 480 set_flow_mode(&desc[idx], DIN_HASH); 481 idx++; 482 483 /* Get hashed key */ 484 hw_desc_init(&desc[idx]); 485 set_cipher_mode(&desc[idx], hashmode); 486 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 487 digestsize, NS_BIT, 0); 488 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 489 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 490 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 491 set_cipher_config0(&desc[idx], 492 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 493 idx++; 494 495 hw_desc_init(&desc[idx]); 496 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 497 set_flow_mode(&desc[idx], BYPASS); 498 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 499 digestsize), (blocksize - digestsize), 500 NS_BIT, 0); 501 idx++; 502 } else { 503 hw_desc_init(&desc[idx]); 504 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 505 keylen, NS_BIT); 506 set_flow_mode(&desc[idx], BYPASS); 507 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 508 keylen, NS_BIT, 0); 509 idx++; 510 511 if ((blocksize - keylen) != 0) { 512 hw_desc_init(&desc[idx]); 513 set_din_const(&desc[idx], 0, 514 (blocksize - keylen)); 515 set_flow_mode(&desc[idx], BYPASS); 516 set_dout_dlli(&desc[idx], 517 (padded_authkey_dma_addr + 518 keylen), 519 (blocksize - keylen), NS_BIT, 0); 520 idx++; 521 } 522 } 523 } else { 524 hw_desc_init(&desc[idx]); 525 set_din_const(&desc[idx], 0, (blocksize - keylen)); 526 set_flow_mode(&desc[idx], BYPASS); 527 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 528 blocksize, NS_BIT, 0); 529 idx++; 530 } 531 532 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 533 if (rc) 534 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 535 536 if (key_dma_addr) 537 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 538 539 kzfree(key); 540 541 return rc; 542 } 543 544 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 545 unsigned int keylen) 546 { 547 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 548 struct cc_crypto_req cc_req = {}; 549 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 550 unsigned int seq_len = 0; 551 struct device *dev = drvdata_to_dev(ctx->drvdata); 552 const u8 *enckey, *authkey; 553 int rc; 554 555 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 556 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 557 558 /* STAT_PHASE_0: Init and sanity checks */ 559 560 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 561 struct crypto_authenc_keys keys; 562 563 rc = crypto_authenc_extractkeys(&keys, key, keylen); 564 if (rc) 565 return rc; 566 enckey = keys.enckey; 567 authkey = keys.authkey; 568 ctx->enc_keylen = keys.enckeylen; 569 ctx->auth_keylen = keys.authkeylen; 570 571 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 572 /* the nonce is stored in bytes at end of key */ 573 if (ctx->enc_keylen < 574 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 575 return -EINVAL; 576 /* Copy nonce from last 4 bytes in CTR key to 577 * first 4 bytes in CTR IV 578 */ 579 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - 580 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); 581 /* Set CTR key size */ 582 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 583 } 584 } else { /* non-authenc - has just one key */ 585 enckey = key; 586 authkey = NULL; 587 ctx->enc_keylen = keylen; 588 ctx->auth_keylen = 0; 589 } 590 591 rc = validate_keys_sizes(ctx); 592 if (rc) 593 return rc; 594 595 /* STAT_PHASE_1: Copy key to ctx */ 596 597 /* Get key material */ 598 memcpy(ctx->enckey, enckey, ctx->enc_keylen); 599 if (ctx->enc_keylen == 24) 600 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 601 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 602 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, 603 ctx->auth_keylen); 604 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 605 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); 606 if (rc) 607 return rc; 608 } 609 610 /* STAT_PHASE_2: Create sequence */ 611 612 switch (ctx->auth_mode) { 613 case DRV_HASH_SHA1: 614 case DRV_HASH_SHA256: 615 seq_len = hmac_setkey(desc, ctx); 616 break; 617 case DRV_HASH_XCBC_MAC: 618 seq_len = xcbc_setkey(desc, ctx); 619 break; 620 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 621 break; /* No auth. key setup */ 622 default: 623 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 624 return -ENOTSUPP; 625 } 626 627 /* STAT_PHASE_3: Submit sequence to HW */ 628 629 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 630 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 631 if (rc) { 632 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 633 return rc; 634 } 635 } 636 637 /* Update STAT_PHASE_3 */ 638 return rc; 639 } 640 641 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 642 unsigned int keylen) 643 { 644 struct crypto_authenc_keys keys; 645 int err; 646 647 err = crypto_authenc_extractkeys(&keys, key, keylen); 648 if (unlikely(err)) 649 return err; 650 651 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 652 cc_aead_setkey(aead, key, keylen); 653 654 memzero_explicit(&keys, sizeof(keys)); 655 return err; 656 } 657 658 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 659 unsigned int keylen) 660 { 661 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 662 663 if (keylen < 3) 664 return -EINVAL; 665 666 keylen -= 3; 667 memcpy(ctx->ctr_nonce, key + keylen, 3); 668 669 return cc_aead_setkey(tfm, key, keylen); 670 } 671 672 static int cc_aead_setauthsize(struct crypto_aead *authenc, 673 unsigned int authsize) 674 { 675 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 676 struct device *dev = drvdata_to_dev(ctx->drvdata); 677 678 /* Unsupported auth. sizes */ 679 if (authsize == 0 || 680 authsize > crypto_aead_maxauthsize(authenc)) { 681 return -ENOTSUPP; 682 } 683 684 ctx->authsize = authsize; 685 dev_dbg(dev, "authlen=%d\n", ctx->authsize); 686 687 return 0; 688 } 689 690 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 691 unsigned int authsize) 692 { 693 switch (authsize) { 694 case 8: 695 case 12: 696 case 16: 697 break; 698 default: 699 return -EINVAL; 700 } 701 702 return cc_aead_setauthsize(authenc, authsize); 703 } 704 705 static int cc_ccm_setauthsize(struct crypto_aead *authenc, 706 unsigned int authsize) 707 { 708 switch (authsize) { 709 case 4: 710 case 6: 711 case 8: 712 case 10: 713 case 12: 714 case 14: 715 case 16: 716 break; 717 default: 718 return -EINVAL; 719 } 720 721 return cc_aead_setauthsize(authenc, authsize); 722 } 723 724 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 725 struct cc_hw_desc desc[], unsigned int *seq_size) 726 { 727 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 728 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 729 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 730 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 731 unsigned int idx = *seq_size; 732 struct device *dev = drvdata_to_dev(ctx->drvdata); 733 734 switch (assoc_dma_type) { 735 case CC_DMA_BUF_DLLI: 736 dev_dbg(dev, "ASSOC buffer type DLLI\n"); 737 hw_desc_init(&desc[idx]); 738 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 739 areq_ctx->assoclen, NS_BIT); 740 set_flow_mode(&desc[idx], flow_mode); 741 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 742 areq_ctx->cryptlen > 0) 743 set_din_not_last_indication(&desc[idx]); 744 break; 745 case CC_DMA_BUF_MLLI: 746 dev_dbg(dev, "ASSOC buffer type MLLI\n"); 747 hw_desc_init(&desc[idx]); 748 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 749 areq_ctx->assoc.mlli_nents, NS_BIT); 750 set_flow_mode(&desc[idx], flow_mode); 751 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 752 areq_ctx->cryptlen > 0) 753 set_din_not_last_indication(&desc[idx]); 754 break; 755 case CC_DMA_BUF_NULL: 756 default: 757 dev_err(dev, "Invalid ASSOC buffer type\n"); 758 } 759 760 *seq_size = (++idx); 761 } 762 763 static void cc_proc_authen_desc(struct aead_request *areq, 764 unsigned int flow_mode, 765 struct cc_hw_desc desc[], 766 unsigned int *seq_size, int direct) 767 { 768 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 769 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 770 unsigned int idx = *seq_size; 771 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 772 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 773 struct device *dev = drvdata_to_dev(ctx->drvdata); 774 775 switch (data_dma_type) { 776 case CC_DMA_BUF_DLLI: 777 { 778 struct scatterlist *cipher = 779 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 780 areq_ctx->dst_sgl : areq_ctx->src_sgl; 781 782 unsigned int offset = 783 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 784 areq_ctx->dst_offset : areq_ctx->src_offset; 785 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 786 hw_desc_init(&desc[idx]); 787 set_din_type(&desc[idx], DMA_DLLI, 788 (sg_dma_address(cipher) + offset), 789 areq_ctx->cryptlen, NS_BIT); 790 set_flow_mode(&desc[idx], flow_mode); 791 break; 792 } 793 case CC_DMA_BUF_MLLI: 794 { 795 /* DOUBLE-PASS flow (as default) 796 * assoc. + iv + data -compact in one table 797 * if assoclen is ZERO only IV perform 798 */ 799 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr; 800 u32 mlli_nents = areq_ctx->assoc.mlli_nents; 801 802 if (areq_ctx->is_single_pass) { 803 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 804 mlli_addr = areq_ctx->dst.sram_addr; 805 mlli_nents = areq_ctx->dst.mlli_nents; 806 } else { 807 mlli_addr = areq_ctx->src.sram_addr; 808 mlli_nents = areq_ctx->src.mlli_nents; 809 } 810 } 811 812 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 813 hw_desc_init(&desc[idx]); 814 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 815 NS_BIT); 816 set_flow_mode(&desc[idx], flow_mode); 817 break; 818 } 819 case CC_DMA_BUF_NULL: 820 default: 821 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 822 } 823 824 *seq_size = (++idx); 825 } 826 827 static void cc_proc_cipher_desc(struct aead_request *areq, 828 unsigned int flow_mode, 829 struct cc_hw_desc desc[], 830 unsigned int *seq_size) 831 { 832 unsigned int idx = *seq_size; 833 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 834 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 835 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 836 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 837 struct device *dev = drvdata_to_dev(ctx->drvdata); 838 839 if (areq_ctx->cryptlen == 0) 840 return; /*null processing*/ 841 842 switch (data_dma_type) { 843 case CC_DMA_BUF_DLLI: 844 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 845 hw_desc_init(&desc[idx]); 846 set_din_type(&desc[idx], DMA_DLLI, 847 (sg_dma_address(areq_ctx->src_sgl) + 848 areq_ctx->src_offset), areq_ctx->cryptlen, 849 NS_BIT); 850 set_dout_dlli(&desc[idx], 851 (sg_dma_address(areq_ctx->dst_sgl) + 852 areq_ctx->dst_offset), 853 areq_ctx->cryptlen, NS_BIT, 0); 854 set_flow_mode(&desc[idx], flow_mode); 855 break; 856 case CC_DMA_BUF_MLLI: 857 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 858 hw_desc_init(&desc[idx]); 859 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 860 areq_ctx->src.mlli_nents, NS_BIT); 861 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 862 areq_ctx->dst.mlli_nents, NS_BIT, 0); 863 set_flow_mode(&desc[idx], flow_mode); 864 break; 865 case CC_DMA_BUF_NULL: 866 default: 867 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 868 } 869 870 *seq_size = (++idx); 871 } 872 873 static void cc_proc_digest_desc(struct aead_request *req, 874 struct cc_hw_desc desc[], 875 unsigned int *seq_size) 876 { 877 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 878 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 879 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 880 unsigned int idx = *seq_size; 881 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 882 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 883 int direct = req_ctx->gen_ctx.op_type; 884 885 /* Get final ICV result */ 886 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 887 hw_desc_init(&desc[idx]); 888 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 889 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 890 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 891 NS_BIT, 1); 892 set_queue_last_ind(ctx->drvdata, &desc[idx]); 893 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 894 set_aes_not_hash_mode(&desc[idx]); 895 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 896 } else { 897 set_cipher_config0(&desc[idx], 898 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 899 set_cipher_mode(&desc[idx], hash_mode); 900 } 901 } else { /*Decrypt*/ 902 /* Get ICV out from hardware */ 903 hw_desc_init(&desc[idx]); 904 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 905 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 906 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 907 ctx->authsize, NS_BIT, 1); 908 set_queue_last_ind(ctx->drvdata, &desc[idx]); 909 set_cipher_config0(&desc[idx], 910 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 911 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 912 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 913 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 914 set_aes_not_hash_mode(&desc[idx]); 915 } else { 916 set_cipher_mode(&desc[idx], hash_mode); 917 } 918 } 919 920 *seq_size = (++idx); 921 } 922 923 static void cc_set_cipher_desc(struct aead_request *req, 924 struct cc_hw_desc desc[], 925 unsigned int *seq_size) 926 { 927 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 928 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 929 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 930 unsigned int hw_iv_size = req_ctx->hw_iv_size; 931 unsigned int idx = *seq_size; 932 int direct = req_ctx->gen_ctx.op_type; 933 934 /* Setup cipher state */ 935 hw_desc_init(&desc[idx]); 936 set_cipher_config0(&desc[idx], direct); 937 set_flow_mode(&desc[idx], ctx->flow_mode); 938 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 939 hw_iv_size, NS_BIT); 940 if (ctx->cipher_mode == DRV_CIPHER_CTR) 941 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 942 else 943 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 944 set_cipher_mode(&desc[idx], ctx->cipher_mode); 945 idx++; 946 947 /* Setup enc. key */ 948 hw_desc_init(&desc[idx]); 949 set_cipher_config0(&desc[idx], direct); 950 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 951 set_flow_mode(&desc[idx], ctx->flow_mode); 952 if (ctx->flow_mode == S_DIN_to_AES) { 953 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 954 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 955 ctx->enc_keylen), NS_BIT); 956 set_key_size_aes(&desc[idx], ctx->enc_keylen); 957 } else { 958 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 959 ctx->enc_keylen, NS_BIT); 960 set_key_size_des(&desc[idx], ctx->enc_keylen); 961 } 962 set_cipher_mode(&desc[idx], ctx->cipher_mode); 963 idx++; 964 965 *seq_size = idx; 966 } 967 968 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 969 unsigned int *seq_size, unsigned int data_flow_mode) 970 { 971 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 972 int direct = req_ctx->gen_ctx.op_type; 973 unsigned int idx = *seq_size; 974 975 if (req_ctx->cryptlen == 0) 976 return; /*null processing*/ 977 978 cc_set_cipher_desc(req, desc, &idx); 979 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 980 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 981 /* We must wait for DMA to write all cipher */ 982 hw_desc_init(&desc[idx]); 983 set_din_no_dma(&desc[idx], 0, 0xfffff0); 984 set_dout_no_dma(&desc[idx], 0, 0, 1); 985 idx++; 986 } 987 988 *seq_size = idx; 989 } 990 991 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 992 unsigned int *seq_size) 993 { 994 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 995 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 996 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 997 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 998 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 999 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1000 unsigned int idx = *seq_size; 1001 1002 /* Loading hash ipad xor key state */ 1003 hw_desc_init(&desc[idx]); 1004 set_cipher_mode(&desc[idx], hash_mode); 1005 set_din_type(&desc[idx], DMA_DLLI, 1006 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1007 NS_BIT); 1008 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1009 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1010 idx++; 1011 1012 /* Load init. digest len (64 bytes) */ 1013 hw_desc_init(&desc[idx]); 1014 set_cipher_mode(&desc[idx], hash_mode); 1015 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1016 ctx->hash_len); 1017 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1018 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1019 idx++; 1020 1021 *seq_size = idx; 1022 } 1023 1024 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1025 unsigned int *seq_size) 1026 { 1027 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1028 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1029 unsigned int idx = *seq_size; 1030 1031 /* Loading MAC state */ 1032 hw_desc_init(&desc[idx]); 1033 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1034 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1035 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1036 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1037 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1038 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1039 set_aes_not_hash_mode(&desc[idx]); 1040 idx++; 1041 1042 /* Setup XCBC MAC K1 */ 1043 hw_desc_init(&desc[idx]); 1044 set_din_type(&desc[idx], DMA_DLLI, 1045 ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1046 AES_KEYSIZE_128, NS_BIT); 1047 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1048 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1049 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1050 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1051 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1052 set_aes_not_hash_mode(&desc[idx]); 1053 idx++; 1054 1055 /* Setup XCBC MAC K2 */ 1056 hw_desc_init(&desc[idx]); 1057 set_din_type(&desc[idx], DMA_DLLI, 1058 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1059 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1060 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1061 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1062 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1063 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1064 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1065 set_aes_not_hash_mode(&desc[idx]); 1066 idx++; 1067 1068 /* Setup XCBC MAC K3 */ 1069 hw_desc_init(&desc[idx]); 1070 set_din_type(&desc[idx], DMA_DLLI, 1071 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1072 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1073 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1074 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1075 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1076 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1077 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1078 set_aes_not_hash_mode(&desc[idx]); 1079 idx++; 1080 1081 *seq_size = idx; 1082 } 1083 1084 static void cc_proc_header_desc(struct aead_request *req, 1085 struct cc_hw_desc desc[], 1086 unsigned int *seq_size) 1087 { 1088 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1089 unsigned int idx = *seq_size; 1090 1091 /* Hash associated data */ 1092 if (areq_ctx->assoclen > 0) 1093 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1094 1095 /* Hash IV */ 1096 *seq_size = idx; 1097 } 1098 1099 static void cc_proc_scheme_desc(struct aead_request *req, 1100 struct cc_hw_desc desc[], 1101 unsigned int *seq_size) 1102 { 1103 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1104 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1105 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1106 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1107 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1108 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1109 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1110 unsigned int idx = *seq_size; 1111 1112 hw_desc_init(&desc[idx]); 1113 set_cipher_mode(&desc[idx], hash_mode); 1114 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1115 ctx->hash_len); 1116 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1117 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1118 set_cipher_do(&desc[idx], DO_PAD); 1119 idx++; 1120 1121 /* Get final ICV result */ 1122 hw_desc_init(&desc[idx]); 1123 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1124 digest_size); 1125 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1126 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1127 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1128 set_cipher_mode(&desc[idx], hash_mode); 1129 idx++; 1130 1131 /* Loading hash opad xor key state */ 1132 hw_desc_init(&desc[idx]); 1133 set_cipher_mode(&desc[idx], hash_mode); 1134 set_din_type(&desc[idx], DMA_DLLI, 1135 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1136 digest_size, NS_BIT); 1137 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1138 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1139 idx++; 1140 1141 /* Load init. digest len (64 bytes) */ 1142 hw_desc_init(&desc[idx]); 1143 set_cipher_mode(&desc[idx], hash_mode); 1144 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1145 ctx->hash_len); 1146 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1147 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1148 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1149 idx++; 1150 1151 /* Perform HASH update */ 1152 hw_desc_init(&desc[idx]); 1153 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1154 digest_size); 1155 set_flow_mode(&desc[idx], DIN_HASH); 1156 idx++; 1157 1158 *seq_size = idx; 1159 } 1160 1161 static void cc_mlli_to_sram(struct aead_request *req, 1162 struct cc_hw_desc desc[], unsigned int *seq_size) 1163 { 1164 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1165 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1166 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1167 struct device *dev = drvdata_to_dev(ctx->drvdata); 1168 1169 if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1170 req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1171 !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { 1172 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1173 (unsigned int)ctx->drvdata->mlli_sram_addr, 1174 req_ctx->mlli_params.mlli_len); 1175 /* Copy MLLI table host-to-sram */ 1176 hw_desc_init(&desc[*seq_size]); 1177 set_din_type(&desc[*seq_size], DMA_DLLI, 1178 req_ctx->mlli_params.mlli_dma_addr, 1179 req_ctx->mlli_params.mlli_len, NS_BIT); 1180 set_dout_sram(&desc[*seq_size], 1181 ctx->drvdata->mlli_sram_addr, 1182 req_ctx->mlli_params.mlli_len); 1183 set_flow_mode(&desc[*seq_size], BYPASS); 1184 (*seq_size)++; 1185 } 1186 } 1187 1188 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1189 enum cc_flow_mode setup_flow_mode, 1190 bool is_single_pass) 1191 { 1192 enum cc_flow_mode data_flow_mode; 1193 1194 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1195 if (setup_flow_mode == S_DIN_to_AES) 1196 data_flow_mode = is_single_pass ? 1197 AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1198 else 1199 data_flow_mode = is_single_pass ? 1200 DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1201 } else { /* Decrypt */ 1202 if (setup_flow_mode == S_DIN_to_AES) 1203 data_flow_mode = is_single_pass ? 1204 AES_and_HASH : DIN_AES_DOUT; 1205 else 1206 data_flow_mode = is_single_pass ? 1207 DES_and_HASH : DIN_DES_DOUT; 1208 } 1209 1210 return data_flow_mode; 1211 } 1212 1213 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1214 unsigned int *seq_size) 1215 { 1216 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1217 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1218 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1219 int direct = req_ctx->gen_ctx.op_type; 1220 unsigned int data_flow_mode = 1221 cc_get_data_flow(direct, ctx->flow_mode, 1222 req_ctx->is_single_pass); 1223 1224 if (req_ctx->is_single_pass) { 1225 /** 1226 * Single-pass flow 1227 */ 1228 cc_set_hmac_desc(req, desc, seq_size); 1229 cc_set_cipher_desc(req, desc, seq_size); 1230 cc_proc_header_desc(req, desc, seq_size); 1231 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1232 cc_proc_scheme_desc(req, desc, seq_size); 1233 cc_proc_digest_desc(req, desc, seq_size); 1234 return; 1235 } 1236 1237 /** 1238 * Double-pass flow 1239 * Fallback for unsupported single-pass modes, 1240 * i.e. using assoc. data of non-word-multiple 1241 */ 1242 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1243 /* encrypt first.. */ 1244 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1245 /* authenc after..*/ 1246 cc_set_hmac_desc(req, desc, seq_size); 1247 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1248 cc_proc_scheme_desc(req, desc, seq_size); 1249 cc_proc_digest_desc(req, desc, seq_size); 1250 1251 } else { /*DECRYPT*/ 1252 /* authenc first..*/ 1253 cc_set_hmac_desc(req, desc, seq_size); 1254 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1255 cc_proc_scheme_desc(req, desc, seq_size); 1256 /* decrypt after.. */ 1257 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1258 /* read the digest result with setting the completion bit 1259 * must be after the cipher operation 1260 */ 1261 cc_proc_digest_desc(req, desc, seq_size); 1262 } 1263 } 1264 1265 static void 1266 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1267 unsigned int *seq_size) 1268 { 1269 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1270 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1271 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1272 int direct = req_ctx->gen_ctx.op_type; 1273 unsigned int data_flow_mode = 1274 cc_get_data_flow(direct, ctx->flow_mode, 1275 req_ctx->is_single_pass); 1276 1277 if (req_ctx->is_single_pass) { 1278 /** 1279 * Single-pass flow 1280 */ 1281 cc_set_xcbc_desc(req, desc, seq_size); 1282 cc_set_cipher_desc(req, desc, seq_size); 1283 cc_proc_header_desc(req, desc, seq_size); 1284 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1285 cc_proc_digest_desc(req, desc, seq_size); 1286 return; 1287 } 1288 1289 /** 1290 * Double-pass flow 1291 * Fallback for unsupported single-pass modes, 1292 * i.e. using assoc. data of non-word-multiple 1293 */ 1294 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1295 /* encrypt first.. */ 1296 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1297 /* authenc after.. */ 1298 cc_set_xcbc_desc(req, desc, seq_size); 1299 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1300 cc_proc_digest_desc(req, desc, seq_size); 1301 } else { /*DECRYPT*/ 1302 /* authenc first.. */ 1303 cc_set_xcbc_desc(req, desc, seq_size); 1304 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1305 /* decrypt after..*/ 1306 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1307 /* read the digest result with setting the completion bit 1308 * must be after the cipher operation 1309 */ 1310 cc_proc_digest_desc(req, desc, seq_size); 1311 } 1312 } 1313 1314 static int validate_data_size(struct cc_aead_ctx *ctx, 1315 enum drv_crypto_direction direct, 1316 struct aead_request *req) 1317 { 1318 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1319 struct device *dev = drvdata_to_dev(ctx->drvdata); 1320 unsigned int assoclen = areq_ctx->assoclen; 1321 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1322 (req->cryptlen - ctx->authsize) : req->cryptlen; 1323 1324 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1325 req->cryptlen < ctx->authsize) 1326 goto data_size_err; 1327 1328 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1329 1330 switch (ctx->flow_mode) { 1331 case S_DIN_to_AES: 1332 if (ctx->cipher_mode == DRV_CIPHER_CBC && 1333 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1334 goto data_size_err; 1335 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1336 break; 1337 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1338 if (areq_ctx->plaintext_authenticate_only) 1339 areq_ctx->is_single_pass = false; 1340 break; 1341 } 1342 1343 if (!IS_ALIGNED(assoclen, sizeof(u32))) 1344 areq_ctx->is_single_pass = false; 1345 1346 if (ctx->cipher_mode == DRV_CIPHER_CTR && 1347 !IS_ALIGNED(cipherlen, sizeof(u32))) 1348 areq_ctx->is_single_pass = false; 1349 1350 break; 1351 case S_DIN_to_DES: 1352 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1353 goto data_size_err; 1354 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1355 areq_ctx->is_single_pass = false; 1356 break; 1357 default: 1358 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1359 goto data_size_err; 1360 } 1361 1362 return 0; 1363 1364 data_size_err: 1365 return -EINVAL; 1366 } 1367 1368 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1369 { 1370 unsigned int len = 0; 1371 1372 if (header_size == 0) 1373 return 0; 1374 1375 if (header_size < ((1UL << 16) - (1UL << 8))) { 1376 len = 2; 1377 1378 pa0_buff[0] = (header_size >> 8) & 0xFF; 1379 pa0_buff[1] = header_size & 0xFF; 1380 } else { 1381 len = 6; 1382 1383 pa0_buff[0] = 0xFF; 1384 pa0_buff[1] = 0xFE; 1385 pa0_buff[2] = (header_size >> 24) & 0xFF; 1386 pa0_buff[3] = (header_size >> 16) & 0xFF; 1387 pa0_buff[4] = (header_size >> 8) & 0xFF; 1388 pa0_buff[5] = header_size & 0xFF; 1389 } 1390 1391 return len; 1392 } 1393 1394 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1395 { 1396 __be32 data; 1397 1398 memset(block, 0, csize); 1399 block += csize; 1400 1401 if (csize >= 4) 1402 csize = 4; 1403 else if (msglen > (1 << (8 * csize))) 1404 return -EOVERFLOW; 1405 1406 data = cpu_to_be32(msglen); 1407 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1408 1409 return 0; 1410 } 1411 1412 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1413 unsigned int *seq_size) 1414 { 1415 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1416 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1417 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1418 unsigned int idx = *seq_size; 1419 unsigned int cipher_flow_mode; 1420 dma_addr_t mac_result; 1421 1422 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1423 cipher_flow_mode = AES_to_HASH_and_DOUT; 1424 mac_result = req_ctx->mac_buf_dma_addr; 1425 } else { /* Encrypt */ 1426 cipher_flow_mode = AES_and_HASH; 1427 mac_result = req_ctx->icv_dma_addr; 1428 } 1429 1430 /* load key */ 1431 hw_desc_init(&desc[idx]); 1432 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1433 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1434 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1435 ctx->enc_keylen), NS_BIT); 1436 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1437 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1438 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1439 set_flow_mode(&desc[idx], S_DIN_to_AES); 1440 idx++; 1441 1442 /* load ctr state */ 1443 hw_desc_init(&desc[idx]); 1444 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1445 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1446 set_din_type(&desc[idx], DMA_DLLI, 1447 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1448 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1449 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1450 set_flow_mode(&desc[idx], S_DIN_to_AES); 1451 idx++; 1452 1453 /* load MAC key */ 1454 hw_desc_init(&desc[idx]); 1455 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1456 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1457 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1458 ctx->enc_keylen), NS_BIT); 1459 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1460 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1461 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1462 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1463 set_aes_not_hash_mode(&desc[idx]); 1464 idx++; 1465 1466 /* load MAC state */ 1467 hw_desc_init(&desc[idx]); 1468 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1469 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1470 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1471 AES_BLOCK_SIZE, NS_BIT); 1472 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1473 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1474 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1475 set_aes_not_hash_mode(&desc[idx]); 1476 idx++; 1477 1478 /* process assoc data */ 1479 if (req_ctx->assoclen > 0) { 1480 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1481 } else { 1482 hw_desc_init(&desc[idx]); 1483 set_din_type(&desc[idx], DMA_DLLI, 1484 sg_dma_address(&req_ctx->ccm_adata_sg), 1485 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1486 set_flow_mode(&desc[idx], DIN_HASH); 1487 idx++; 1488 } 1489 1490 /* process the cipher */ 1491 if (req_ctx->cryptlen) 1492 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1493 1494 /* Read temporal MAC */ 1495 hw_desc_init(&desc[idx]); 1496 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1497 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1498 NS_BIT, 0); 1499 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1500 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1501 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1502 set_aes_not_hash_mode(&desc[idx]); 1503 idx++; 1504 1505 /* load AES-CTR state (for last MAC calculation)*/ 1506 hw_desc_init(&desc[idx]); 1507 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1508 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1509 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1510 AES_BLOCK_SIZE, NS_BIT); 1511 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1512 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1513 set_flow_mode(&desc[idx], S_DIN_to_AES); 1514 idx++; 1515 1516 hw_desc_init(&desc[idx]); 1517 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1518 set_dout_no_dma(&desc[idx], 0, 0, 1); 1519 idx++; 1520 1521 /* encrypt the "T" value and store MAC in mac_state */ 1522 hw_desc_init(&desc[idx]); 1523 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1524 ctx->authsize, NS_BIT); 1525 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1526 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1527 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1528 idx++; 1529 1530 *seq_size = idx; 1531 return 0; 1532 } 1533 1534 static int config_ccm_adata(struct aead_request *req) 1535 { 1536 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1537 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1538 struct device *dev = drvdata_to_dev(ctx->drvdata); 1539 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1540 //unsigned int size_of_a = 0, rem_a_size = 0; 1541 unsigned int lp = req->iv[0]; 1542 /* Note: The code assume that req->iv[0] already contains the value 1543 * of L' of RFC3610 1544 */ 1545 unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1546 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1547 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1548 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1549 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1550 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1551 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1552 req->cryptlen : 1553 (req->cryptlen - ctx->authsize); 1554 int rc; 1555 1556 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1557 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1558 1559 /* taken from crypto/ccm.c */ 1560 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1561 if (l < 2 || l > 8) { 1562 dev_dbg(dev, "illegal iv value %X\n", req->iv[0]); 1563 return -EINVAL; 1564 } 1565 memcpy(b0, req->iv, AES_BLOCK_SIZE); 1566 1567 /* format control info per RFC 3610 and 1568 * NIST Special Publication 800-38C 1569 */ 1570 *b0 |= (8 * ((m - 2) / 2)); 1571 if (req_ctx->assoclen > 0) 1572 *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1573 1574 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1575 if (rc) { 1576 dev_err(dev, "message len overflow detected"); 1577 return rc; 1578 } 1579 /* END of "taken from crypto/ccm.c" */ 1580 1581 /* l(a) - size of associated data. */ 1582 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); 1583 1584 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1585 req->iv[15] = 1; 1586 1587 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1588 ctr_count_0[15] = 0; 1589 1590 return 0; 1591 } 1592 1593 static void cc_proc_rfc4309_ccm(struct aead_request *req) 1594 { 1595 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1596 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1597 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1598 1599 /* L' */ 1600 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1601 /* For RFC 4309, always use 4 bytes for message length 1602 * (at most 2^32-1 bytes). 1603 */ 1604 areq_ctx->ctr_iv[0] = 3; 1605 1606 /* In RFC 4309 there is an 11-bytes nonce+IV part, 1607 * that we build here. 1608 */ 1609 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1610 CCM_BLOCK_NONCE_SIZE); 1611 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1612 CCM_BLOCK_IV_SIZE); 1613 req->iv = areq_ctx->ctr_iv; 1614 areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE; 1615 } 1616 1617 static void cc_set_ghash_desc(struct aead_request *req, 1618 struct cc_hw_desc desc[], unsigned int *seq_size) 1619 { 1620 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1621 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1622 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1623 unsigned int idx = *seq_size; 1624 1625 /* load key to AES*/ 1626 hw_desc_init(&desc[idx]); 1627 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1628 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1629 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1630 ctx->enc_keylen, NS_BIT); 1631 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1632 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1633 set_flow_mode(&desc[idx], S_DIN_to_AES); 1634 idx++; 1635 1636 /* process one zero block to generate hkey */ 1637 hw_desc_init(&desc[idx]); 1638 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1639 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1640 NS_BIT, 0); 1641 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1642 idx++; 1643 1644 /* Memory Barrier */ 1645 hw_desc_init(&desc[idx]); 1646 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1647 set_dout_no_dma(&desc[idx], 0, 0, 1); 1648 idx++; 1649 1650 /* Load GHASH subkey */ 1651 hw_desc_init(&desc[idx]); 1652 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1653 AES_BLOCK_SIZE, NS_BIT); 1654 set_dout_no_dma(&desc[idx], 0, 0, 1); 1655 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1656 set_aes_not_hash_mode(&desc[idx]); 1657 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1658 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1659 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1660 idx++; 1661 1662 /* Configure Hash Engine to work with GHASH. 1663 * Since it was not possible to extend HASH submodes to add GHASH, 1664 * The following command is necessary in order to 1665 * select GHASH (according to HW designers) 1666 */ 1667 hw_desc_init(&desc[idx]); 1668 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1669 set_dout_no_dma(&desc[idx], 0, 0, 1); 1670 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1671 set_aes_not_hash_mode(&desc[idx]); 1672 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1673 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1674 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1675 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1676 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1677 idx++; 1678 1679 /* Load GHASH initial STATE (which is 0). (for any hash there is an 1680 * initial state) 1681 */ 1682 hw_desc_init(&desc[idx]); 1683 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1684 set_dout_no_dma(&desc[idx], 0, 0, 1); 1685 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1686 set_aes_not_hash_mode(&desc[idx]); 1687 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1688 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1689 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1690 idx++; 1691 1692 *seq_size = idx; 1693 } 1694 1695 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1696 unsigned int *seq_size) 1697 { 1698 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1699 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1700 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1701 unsigned int idx = *seq_size; 1702 1703 /* load key to AES*/ 1704 hw_desc_init(&desc[idx]); 1705 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1706 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1707 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1708 ctx->enc_keylen, NS_BIT); 1709 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1710 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1711 set_flow_mode(&desc[idx], S_DIN_to_AES); 1712 idx++; 1713 1714 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1715 /* load AES/CTR initial CTR value inc by 2*/ 1716 hw_desc_init(&desc[idx]); 1717 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1718 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1719 set_din_type(&desc[idx], DMA_DLLI, 1720 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1721 NS_BIT); 1722 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1723 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1724 set_flow_mode(&desc[idx], S_DIN_to_AES); 1725 idx++; 1726 } 1727 1728 *seq_size = idx; 1729 } 1730 1731 static void cc_proc_gcm_result(struct aead_request *req, 1732 struct cc_hw_desc desc[], 1733 unsigned int *seq_size) 1734 { 1735 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1736 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1737 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1738 dma_addr_t mac_result; 1739 unsigned int idx = *seq_size; 1740 1741 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1742 mac_result = req_ctx->mac_buf_dma_addr; 1743 } else { /* Encrypt */ 1744 mac_result = req_ctx->icv_dma_addr; 1745 } 1746 1747 /* process(ghash) gcm_block_len */ 1748 hw_desc_init(&desc[idx]); 1749 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1750 AES_BLOCK_SIZE, NS_BIT); 1751 set_flow_mode(&desc[idx], DIN_HASH); 1752 idx++; 1753 1754 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1755 hw_desc_init(&desc[idx]); 1756 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1757 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1758 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1759 NS_BIT, 0); 1760 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1761 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1762 set_aes_not_hash_mode(&desc[idx]); 1763 1764 idx++; 1765 1766 /* load AES/CTR initial CTR value inc by 1*/ 1767 hw_desc_init(&desc[idx]); 1768 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1769 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1770 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1771 AES_BLOCK_SIZE, NS_BIT); 1772 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1773 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1774 set_flow_mode(&desc[idx], S_DIN_to_AES); 1775 idx++; 1776 1777 /* Memory Barrier */ 1778 hw_desc_init(&desc[idx]); 1779 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1780 set_dout_no_dma(&desc[idx], 0, 0, 1); 1781 idx++; 1782 1783 /* process GCTR on stored GHASH and store MAC in mac_state*/ 1784 hw_desc_init(&desc[idx]); 1785 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1786 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1787 AES_BLOCK_SIZE, NS_BIT); 1788 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1789 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1790 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1791 idx++; 1792 1793 *seq_size = idx; 1794 } 1795 1796 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1797 unsigned int *seq_size) 1798 { 1799 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1800 unsigned int cipher_flow_mode; 1801 1802 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1803 cipher_flow_mode = AES_and_HASH; 1804 } else { /* Encrypt */ 1805 cipher_flow_mode = AES_to_HASH_and_DOUT; 1806 } 1807 1808 //in RFC4543 no data to encrypt. just copy data from src to dest. 1809 if (req_ctx->plaintext_authenticate_only) { 1810 cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1811 cc_set_ghash_desc(req, desc, seq_size); 1812 /* process(ghash) assoc data */ 1813 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1814 cc_set_gctr_desc(req, desc, seq_size); 1815 cc_proc_gcm_result(req, desc, seq_size); 1816 return 0; 1817 } 1818 1819 // for gcm and rfc4106. 1820 cc_set_ghash_desc(req, desc, seq_size); 1821 /* process(ghash) assoc data */ 1822 if (req_ctx->assoclen > 0) 1823 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1824 cc_set_gctr_desc(req, desc, seq_size); 1825 /* process(gctr+ghash) */ 1826 if (req_ctx->cryptlen) 1827 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1828 cc_proc_gcm_result(req, desc, seq_size); 1829 1830 return 0; 1831 } 1832 1833 static int config_gcm_context(struct aead_request *req) 1834 { 1835 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1836 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1837 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1838 struct device *dev = drvdata_to_dev(ctx->drvdata); 1839 1840 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1841 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1842 req->cryptlen : 1843 (req->cryptlen - ctx->authsize); 1844 __be32 counter = cpu_to_be32(2); 1845 1846 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", 1847 __func__, cryptlen, req_ctx->assoclen, ctx->authsize); 1848 1849 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1850 1851 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1852 1853 memcpy(req->iv + 12, &counter, 4); 1854 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1855 1856 counter = cpu_to_be32(1); 1857 memcpy(req->iv + 12, &counter, 4); 1858 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1859 1860 if (!req_ctx->plaintext_authenticate_only) { 1861 __be64 temp64; 1862 1863 temp64 = cpu_to_be64(req_ctx->assoclen * 8); 1864 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1865 temp64 = cpu_to_be64(cryptlen * 8); 1866 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1867 } else { 1868 /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1869 * data that is nothing is encrypted. 1870 */ 1871 __be64 temp64; 1872 1873 temp64 = cpu_to_be64((req_ctx->assoclen + 1874 GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8); 1875 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1876 temp64 = 0; 1877 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1878 } 1879 1880 return 0; 1881 } 1882 1883 static void cc_proc_rfc4_gcm(struct aead_request *req) 1884 { 1885 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1886 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1887 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1888 1889 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1890 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1891 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1892 GCM_BLOCK_RFC4_IV_SIZE); 1893 req->iv = areq_ctx->ctr_iv; 1894 areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; 1895 } 1896 1897 static int cc_proc_aead(struct aead_request *req, 1898 enum drv_crypto_direction direct) 1899 { 1900 int rc = 0; 1901 int seq_len = 0; 1902 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1903 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1904 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1905 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1906 struct device *dev = drvdata_to_dev(ctx->drvdata); 1907 struct cc_crypto_req cc_req = {}; 1908 1909 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1910 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1911 ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1912 sg_virt(req->dst), req->dst->offset, req->cryptlen); 1913 1914 /* STAT_PHASE_0: Init and sanity checks */ 1915 1916 /* Check data length according to mode */ 1917 if (validate_data_size(ctx, direct, req)) { 1918 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1919 req->cryptlen, areq_ctx->assoclen); 1920 return -EINVAL; 1921 } 1922 1923 /* Setup request structure */ 1924 cc_req.user_cb = (void *)cc_aead_complete; 1925 cc_req.user_arg = (void *)req; 1926 1927 /* Setup request context */ 1928 areq_ctx->gen_ctx.op_type = direct; 1929 areq_ctx->req_authsize = ctx->authsize; 1930 areq_ctx->cipher_mode = ctx->cipher_mode; 1931 1932 /* STAT_PHASE_1: Map buffers */ 1933 1934 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1935 /* Build CTR IV - Copy nonce from last 4 bytes in 1936 * CTR key to first 4 bytes in CTR IV 1937 */ 1938 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1939 CTR_RFC3686_NONCE_SIZE); 1940 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1941 CTR_RFC3686_IV_SIZE); 1942 /* Initialize counter portion of counter block */ 1943 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1944 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1945 1946 /* Replace with counter iv */ 1947 req->iv = areq_ctx->ctr_iv; 1948 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1949 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1950 (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1951 areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1952 if (areq_ctx->ctr_iv != req->iv) { 1953 memcpy(areq_ctx->ctr_iv, req->iv, 1954 crypto_aead_ivsize(tfm)); 1955 req->iv = areq_ctx->ctr_iv; 1956 } 1957 } else { 1958 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1959 } 1960 1961 if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1962 rc = config_ccm_adata(req); 1963 if (rc) { 1964 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1965 rc); 1966 goto exit; 1967 } 1968 } else { 1969 areq_ctx->ccm_hdr_size = ccm_header_size_null; 1970 } 1971 1972 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1973 rc = config_gcm_context(req); 1974 if (rc) { 1975 dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 1976 rc); 1977 goto exit; 1978 } 1979 } 1980 1981 rc = cc_map_aead_request(ctx->drvdata, req); 1982 if (rc) { 1983 dev_err(dev, "map_request() failed\n"); 1984 goto exit; 1985 } 1986 1987 /* STAT_PHASE_2: Create sequence */ 1988 1989 /* Load MLLI tables to SRAM if necessary */ 1990 cc_mlli_to_sram(req, desc, &seq_len); 1991 1992 /*TODO: move seq len by reference */ 1993 switch (ctx->auth_mode) { 1994 case DRV_HASH_SHA1: 1995 case DRV_HASH_SHA256: 1996 cc_hmac_authenc(req, desc, &seq_len); 1997 break; 1998 case DRV_HASH_XCBC_MAC: 1999 cc_xcbc_authenc(req, desc, &seq_len); 2000 break; 2001 case DRV_HASH_NULL: 2002 if (ctx->cipher_mode == DRV_CIPHER_CCM) 2003 cc_ccm(req, desc, &seq_len); 2004 if (ctx->cipher_mode == DRV_CIPHER_GCTR) 2005 cc_gcm(req, desc, &seq_len); 2006 break; 2007 default: 2008 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2009 cc_unmap_aead_request(dev, req); 2010 rc = -ENOTSUPP; 2011 goto exit; 2012 } 2013 2014 /* STAT_PHASE_3: Lock HW and push sequence */ 2015 2016 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2017 2018 if (rc != -EINPROGRESS && rc != -EBUSY) { 2019 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2020 cc_unmap_aead_request(dev, req); 2021 } 2022 2023 exit: 2024 return rc; 2025 } 2026 2027 static int cc_aead_encrypt(struct aead_request *req) 2028 { 2029 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2030 int rc; 2031 2032 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2033 2034 /* No generated IV required */ 2035 areq_ctx->backup_iv = req->iv; 2036 areq_ctx->assoclen = req->assoclen; 2037 areq_ctx->is_gcm4543 = false; 2038 2039 areq_ctx->plaintext_authenticate_only = false; 2040 2041 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2042 if (rc != -EINPROGRESS && rc != -EBUSY) 2043 req->iv = areq_ctx->backup_iv; 2044 2045 return rc; 2046 } 2047 2048 static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2049 { 2050 /* Very similar to cc_aead_encrypt() above. */ 2051 2052 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2053 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2054 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2055 struct device *dev = drvdata_to_dev(ctx->drvdata); 2056 int rc = -EINVAL; 2057 2058 if (!valid_assoclen(req)) { 2059 dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2060 goto out; 2061 } 2062 2063 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2064 2065 /* No generated IV required */ 2066 areq_ctx->backup_iv = req->iv; 2067 areq_ctx->assoclen = req->assoclen; 2068 areq_ctx->is_gcm4543 = true; 2069 2070 cc_proc_rfc4309_ccm(req); 2071 2072 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2073 if (rc != -EINPROGRESS && rc != -EBUSY) 2074 req->iv = areq_ctx->backup_iv; 2075 out: 2076 return rc; 2077 } 2078 2079 static int cc_aead_decrypt(struct aead_request *req) 2080 { 2081 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2082 int rc; 2083 2084 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2085 2086 /* No generated IV required */ 2087 areq_ctx->backup_iv = req->iv; 2088 areq_ctx->assoclen = req->assoclen; 2089 areq_ctx->is_gcm4543 = false; 2090 2091 areq_ctx->plaintext_authenticate_only = false; 2092 2093 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2094 if (rc != -EINPROGRESS && rc != -EBUSY) 2095 req->iv = areq_ctx->backup_iv; 2096 2097 return rc; 2098 } 2099 2100 static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2101 { 2102 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2103 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2104 struct device *dev = drvdata_to_dev(ctx->drvdata); 2105 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2106 int rc = -EINVAL; 2107 2108 if (!valid_assoclen(req)) { 2109 dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2110 goto out; 2111 } 2112 2113 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2114 2115 /* No generated IV required */ 2116 areq_ctx->backup_iv = req->iv; 2117 areq_ctx->assoclen = req->assoclen; 2118 2119 areq_ctx->is_gcm4543 = true; 2120 cc_proc_rfc4309_ccm(req); 2121 2122 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2123 if (rc != -EINPROGRESS && rc != -EBUSY) 2124 req->iv = areq_ctx->backup_iv; 2125 2126 out: 2127 return rc; 2128 } 2129 2130 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2131 unsigned int keylen) 2132 { 2133 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2134 struct device *dev = drvdata_to_dev(ctx->drvdata); 2135 2136 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2137 2138 if (keylen < 4) 2139 return -EINVAL; 2140 2141 keylen -= 4; 2142 memcpy(ctx->ctr_nonce, key + keylen, 4); 2143 2144 return cc_aead_setkey(tfm, key, keylen); 2145 } 2146 2147 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2148 unsigned int keylen) 2149 { 2150 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2151 struct device *dev = drvdata_to_dev(ctx->drvdata); 2152 2153 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2154 2155 if (keylen < 4) 2156 return -EINVAL; 2157 2158 keylen -= 4; 2159 memcpy(ctx->ctr_nonce, key + keylen, 4); 2160 2161 return cc_aead_setkey(tfm, key, keylen); 2162 } 2163 2164 static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2165 unsigned int authsize) 2166 { 2167 switch (authsize) { 2168 case 4: 2169 case 8: 2170 case 12: 2171 case 13: 2172 case 14: 2173 case 15: 2174 case 16: 2175 break; 2176 default: 2177 return -EINVAL; 2178 } 2179 2180 return cc_aead_setauthsize(authenc, authsize); 2181 } 2182 2183 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2184 unsigned int authsize) 2185 { 2186 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2187 struct device *dev = drvdata_to_dev(ctx->drvdata); 2188 2189 dev_dbg(dev, "authsize %d\n", authsize); 2190 2191 switch (authsize) { 2192 case 8: 2193 case 12: 2194 case 16: 2195 break; 2196 default: 2197 return -EINVAL; 2198 } 2199 2200 return cc_aead_setauthsize(authenc, authsize); 2201 } 2202 2203 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2204 unsigned int authsize) 2205 { 2206 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2207 struct device *dev = drvdata_to_dev(ctx->drvdata); 2208 2209 dev_dbg(dev, "authsize %d\n", authsize); 2210 2211 if (authsize != 16) 2212 return -EINVAL; 2213 2214 return cc_aead_setauthsize(authenc, authsize); 2215 } 2216 2217 static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2218 { 2219 /* Very similar to cc_aead_encrypt() above. */ 2220 2221 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2222 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2223 struct device *dev = drvdata_to_dev(ctx->drvdata); 2224 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2225 int rc = -EINVAL; 2226 2227 if (!valid_assoclen(req)) { 2228 dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2229 goto out; 2230 } 2231 2232 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2233 2234 /* No generated IV required */ 2235 areq_ctx->backup_iv = req->iv; 2236 areq_ctx->assoclen = req->assoclen; 2237 areq_ctx->plaintext_authenticate_only = false; 2238 2239 cc_proc_rfc4_gcm(req); 2240 areq_ctx->is_gcm4543 = true; 2241 2242 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2243 if (rc != -EINPROGRESS && rc != -EBUSY) 2244 req->iv = areq_ctx->backup_iv; 2245 out: 2246 return rc; 2247 } 2248 2249 static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2250 { 2251 /* Very similar to cc_aead_encrypt() above. */ 2252 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2253 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2254 struct device *dev = drvdata_to_dev(ctx->drvdata); 2255 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2256 int rc = -EINVAL; 2257 2258 if (!valid_assoclen(req)) { 2259 dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2260 goto out; 2261 } 2262 2263 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2264 2265 //plaintext is not encryped with rfc4543 2266 areq_ctx->plaintext_authenticate_only = true; 2267 2268 /* No generated IV required */ 2269 areq_ctx->backup_iv = req->iv; 2270 areq_ctx->assoclen = req->assoclen; 2271 2272 cc_proc_rfc4_gcm(req); 2273 areq_ctx->is_gcm4543 = true; 2274 2275 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2276 if (rc != -EINPROGRESS && rc != -EBUSY) 2277 req->iv = areq_ctx->backup_iv; 2278 out: 2279 return rc; 2280 } 2281 2282 static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2283 { 2284 /* Very similar to cc_aead_decrypt() above. */ 2285 2286 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2287 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2288 struct device *dev = drvdata_to_dev(ctx->drvdata); 2289 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2290 int rc = -EINVAL; 2291 2292 if (!valid_assoclen(req)) { 2293 dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2294 goto out; 2295 } 2296 2297 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2298 2299 /* No generated IV required */ 2300 areq_ctx->backup_iv = req->iv; 2301 areq_ctx->assoclen = req->assoclen; 2302 areq_ctx->plaintext_authenticate_only = false; 2303 2304 cc_proc_rfc4_gcm(req); 2305 areq_ctx->is_gcm4543 = true; 2306 2307 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2308 if (rc != -EINPROGRESS && rc != -EBUSY) 2309 req->iv = areq_ctx->backup_iv; 2310 out: 2311 return rc; 2312 } 2313 2314 static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2315 { 2316 /* Very similar to cc_aead_decrypt() above. */ 2317 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2318 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2319 struct device *dev = drvdata_to_dev(ctx->drvdata); 2320 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2321 int rc = -EINVAL; 2322 2323 if (!valid_assoclen(req)) { 2324 dev_dbg(dev, "invalid Assoclen:%u\n", req->assoclen); 2325 goto out; 2326 } 2327 2328 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2329 2330 //plaintext is not decryped with rfc4543 2331 areq_ctx->plaintext_authenticate_only = true; 2332 2333 /* No generated IV required */ 2334 areq_ctx->backup_iv = req->iv; 2335 areq_ctx->assoclen = req->assoclen; 2336 2337 cc_proc_rfc4_gcm(req); 2338 areq_ctx->is_gcm4543 = true; 2339 2340 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2341 if (rc != -EINPROGRESS && rc != -EBUSY) 2342 req->iv = areq_ctx->backup_iv; 2343 out: 2344 return rc; 2345 } 2346 2347 /* aead alg */ 2348 static struct cc_alg_template aead_algs[] = { 2349 { 2350 .name = "authenc(hmac(sha1),cbc(aes))", 2351 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2352 .blocksize = AES_BLOCK_SIZE, 2353 .template_aead = { 2354 .setkey = cc_aead_setkey, 2355 .setauthsize = cc_aead_setauthsize, 2356 .encrypt = cc_aead_encrypt, 2357 .decrypt = cc_aead_decrypt, 2358 .init = cc_aead_init, 2359 .exit = cc_aead_exit, 2360 .ivsize = AES_BLOCK_SIZE, 2361 .maxauthsize = SHA1_DIGEST_SIZE, 2362 }, 2363 .cipher_mode = DRV_CIPHER_CBC, 2364 .flow_mode = S_DIN_to_AES, 2365 .auth_mode = DRV_HASH_SHA1, 2366 .min_hw_rev = CC_HW_REV_630, 2367 .std_body = CC_STD_NIST, 2368 }, 2369 { 2370 .name = "authenc(hmac(sha1),cbc(des3_ede))", 2371 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2372 .blocksize = DES3_EDE_BLOCK_SIZE, 2373 .template_aead = { 2374 .setkey = cc_des3_aead_setkey, 2375 .setauthsize = cc_aead_setauthsize, 2376 .encrypt = cc_aead_encrypt, 2377 .decrypt = cc_aead_decrypt, 2378 .init = cc_aead_init, 2379 .exit = cc_aead_exit, 2380 .ivsize = DES3_EDE_BLOCK_SIZE, 2381 .maxauthsize = SHA1_DIGEST_SIZE, 2382 }, 2383 .cipher_mode = DRV_CIPHER_CBC, 2384 .flow_mode = S_DIN_to_DES, 2385 .auth_mode = DRV_HASH_SHA1, 2386 .min_hw_rev = CC_HW_REV_630, 2387 .std_body = CC_STD_NIST, 2388 }, 2389 { 2390 .name = "authenc(hmac(sha256),cbc(aes))", 2391 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2392 .blocksize = AES_BLOCK_SIZE, 2393 .template_aead = { 2394 .setkey = cc_aead_setkey, 2395 .setauthsize = cc_aead_setauthsize, 2396 .encrypt = cc_aead_encrypt, 2397 .decrypt = cc_aead_decrypt, 2398 .init = cc_aead_init, 2399 .exit = cc_aead_exit, 2400 .ivsize = AES_BLOCK_SIZE, 2401 .maxauthsize = SHA256_DIGEST_SIZE, 2402 }, 2403 .cipher_mode = DRV_CIPHER_CBC, 2404 .flow_mode = S_DIN_to_AES, 2405 .auth_mode = DRV_HASH_SHA256, 2406 .min_hw_rev = CC_HW_REV_630, 2407 .std_body = CC_STD_NIST, 2408 }, 2409 { 2410 .name = "authenc(hmac(sha256),cbc(des3_ede))", 2411 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2412 .blocksize = DES3_EDE_BLOCK_SIZE, 2413 .template_aead = { 2414 .setkey = cc_des3_aead_setkey, 2415 .setauthsize = cc_aead_setauthsize, 2416 .encrypt = cc_aead_encrypt, 2417 .decrypt = cc_aead_decrypt, 2418 .init = cc_aead_init, 2419 .exit = cc_aead_exit, 2420 .ivsize = DES3_EDE_BLOCK_SIZE, 2421 .maxauthsize = SHA256_DIGEST_SIZE, 2422 }, 2423 .cipher_mode = DRV_CIPHER_CBC, 2424 .flow_mode = S_DIN_to_DES, 2425 .auth_mode = DRV_HASH_SHA256, 2426 .min_hw_rev = CC_HW_REV_630, 2427 .std_body = CC_STD_NIST, 2428 }, 2429 { 2430 .name = "authenc(xcbc(aes),cbc(aes))", 2431 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2432 .blocksize = AES_BLOCK_SIZE, 2433 .template_aead = { 2434 .setkey = cc_aead_setkey, 2435 .setauthsize = cc_aead_setauthsize, 2436 .encrypt = cc_aead_encrypt, 2437 .decrypt = cc_aead_decrypt, 2438 .init = cc_aead_init, 2439 .exit = cc_aead_exit, 2440 .ivsize = AES_BLOCK_SIZE, 2441 .maxauthsize = AES_BLOCK_SIZE, 2442 }, 2443 .cipher_mode = DRV_CIPHER_CBC, 2444 .flow_mode = S_DIN_to_AES, 2445 .auth_mode = DRV_HASH_XCBC_MAC, 2446 .min_hw_rev = CC_HW_REV_630, 2447 .std_body = CC_STD_NIST, 2448 }, 2449 { 2450 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2451 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2452 .blocksize = 1, 2453 .template_aead = { 2454 .setkey = cc_aead_setkey, 2455 .setauthsize = cc_aead_setauthsize, 2456 .encrypt = cc_aead_encrypt, 2457 .decrypt = cc_aead_decrypt, 2458 .init = cc_aead_init, 2459 .exit = cc_aead_exit, 2460 .ivsize = CTR_RFC3686_IV_SIZE, 2461 .maxauthsize = SHA1_DIGEST_SIZE, 2462 }, 2463 .cipher_mode = DRV_CIPHER_CTR, 2464 .flow_mode = S_DIN_to_AES, 2465 .auth_mode = DRV_HASH_SHA1, 2466 .min_hw_rev = CC_HW_REV_630, 2467 .std_body = CC_STD_NIST, 2468 }, 2469 { 2470 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2471 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2472 .blocksize = 1, 2473 .template_aead = { 2474 .setkey = cc_aead_setkey, 2475 .setauthsize = cc_aead_setauthsize, 2476 .encrypt = cc_aead_encrypt, 2477 .decrypt = cc_aead_decrypt, 2478 .init = cc_aead_init, 2479 .exit = cc_aead_exit, 2480 .ivsize = CTR_RFC3686_IV_SIZE, 2481 .maxauthsize = SHA256_DIGEST_SIZE, 2482 }, 2483 .cipher_mode = DRV_CIPHER_CTR, 2484 .flow_mode = S_DIN_to_AES, 2485 .auth_mode = DRV_HASH_SHA256, 2486 .min_hw_rev = CC_HW_REV_630, 2487 .std_body = CC_STD_NIST, 2488 }, 2489 { 2490 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2491 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2492 .blocksize = 1, 2493 .template_aead = { 2494 .setkey = cc_aead_setkey, 2495 .setauthsize = cc_aead_setauthsize, 2496 .encrypt = cc_aead_encrypt, 2497 .decrypt = cc_aead_decrypt, 2498 .init = cc_aead_init, 2499 .exit = cc_aead_exit, 2500 .ivsize = CTR_RFC3686_IV_SIZE, 2501 .maxauthsize = AES_BLOCK_SIZE, 2502 }, 2503 .cipher_mode = DRV_CIPHER_CTR, 2504 .flow_mode = S_DIN_to_AES, 2505 .auth_mode = DRV_HASH_XCBC_MAC, 2506 .min_hw_rev = CC_HW_REV_630, 2507 .std_body = CC_STD_NIST, 2508 }, 2509 { 2510 .name = "ccm(aes)", 2511 .driver_name = "ccm-aes-ccree", 2512 .blocksize = 1, 2513 .template_aead = { 2514 .setkey = cc_aead_setkey, 2515 .setauthsize = cc_ccm_setauthsize, 2516 .encrypt = cc_aead_encrypt, 2517 .decrypt = cc_aead_decrypt, 2518 .init = cc_aead_init, 2519 .exit = cc_aead_exit, 2520 .ivsize = AES_BLOCK_SIZE, 2521 .maxauthsize = AES_BLOCK_SIZE, 2522 }, 2523 .cipher_mode = DRV_CIPHER_CCM, 2524 .flow_mode = S_DIN_to_AES, 2525 .auth_mode = DRV_HASH_NULL, 2526 .min_hw_rev = CC_HW_REV_630, 2527 .std_body = CC_STD_NIST, 2528 }, 2529 { 2530 .name = "rfc4309(ccm(aes))", 2531 .driver_name = "rfc4309-ccm-aes-ccree", 2532 .blocksize = 1, 2533 .template_aead = { 2534 .setkey = cc_rfc4309_ccm_setkey, 2535 .setauthsize = cc_rfc4309_ccm_setauthsize, 2536 .encrypt = cc_rfc4309_ccm_encrypt, 2537 .decrypt = cc_rfc4309_ccm_decrypt, 2538 .init = cc_aead_init, 2539 .exit = cc_aead_exit, 2540 .ivsize = CCM_BLOCK_IV_SIZE, 2541 .maxauthsize = AES_BLOCK_SIZE, 2542 }, 2543 .cipher_mode = DRV_CIPHER_CCM, 2544 .flow_mode = S_DIN_to_AES, 2545 .auth_mode = DRV_HASH_NULL, 2546 .min_hw_rev = CC_HW_REV_630, 2547 .std_body = CC_STD_NIST, 2548 }, 2549 { 2550 .name = "gcm(aes)", 2551 .driver_name = "gcm-aes-ccree", 2552 .blocksize = 1, 2553 .template_aead = { 2554 .setkey = cc_aead_setkey, 2555 .setauthsize = cc_gcm_setauthsize, 2556 .encrypt = cc_aead_encrypt, 2557 .decrypt = cc_aead_decrypt, 2558 .init = cc_aead_init, 2559 .exit = cc_aead_exit, 2560 .ivsize = 12, 2561 .maxauthsize = AES_BLOCK_SIZE, 2562 }, 2563 .cipher_mode = DRV_CIPHER_GCTR, 2564 .flow_mode = S_DIN_to_AES, 2565 .auth_mode = DRV_HASH_NULL, 2566 .min_hw_rev = CC_HW_REV_630, 2567 .std_body = CC_STD_NIST, 2568 }, 2569 { 2570 .name = "rfc4106(gcm(aes))", 2571 .driver_name = "rfc4106-gcm-aes-ccree", 2572 .blocksize = 1, 2573 .template_aead = { 2574 .setkey = cc_rfc4106_gcm_setkey, 2575 .setauthsize = cc_rfc4106_gcm_setauthsize, 2576 .encrypt = cc_rfc4106_gcm_encrypt, 2577 .decrypt = cc_rfc4106_gcm_decrypt, 2578 .init = cc_aead_init, 2579 .exit = cc_aead_exit, 2580 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2581 .maxauthsize = AES_BLOCK_SIZE, 2582 }, 2583 .cipher_mode = DRV_CIPHER_GCTR, 2584 .flow_mode = S_DIN_to_AES, 2585 .auth_mode = DRV_HASH_NULL, 2586 .min_hw_rev = CC_HW_REV_630, 2587 .std_body = CC_STD_NIST, 2588 }, 2589 { 2590 .name = "rfc4543(gcm(aes))", 2591 .driver_name = "rfc4543-gcm-aes-ccree", 2592 .blocksize = 1, 2593 .template_aead = { 2594 .setkey = cc_rfc4543_gcm_setkey, 2595 .setauthsize = cc_rfc4543_gcm_setauthsize, 2596 .encrypt = cc_rfc4543_gcm_encrypt, 2597 .decrypt = cc_rfc4543_gcm_decrypt, 2598 .init = cc_aead_init, 2599 .exit = cc_aead_exit, 2600 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2601 .maxauthsize = AES_BLOCK_SIZE, 2602 }, 2603 .cipher_mode = DRV_CIPHER_GCTR, 2604 .flow_mode = S_DIN_to_AES, 2605 .auth_mode = DRV_HASH_NULL, 2606 .min_hw_rev = CC_HW_REV_630, 2607 .std_body = CC_STD_NIST, 2608 }, 2609 }; 2610 2611 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2612 struct device *dev) 2613 { 2614 struct cc_crypto_alg *t_alg; 2615 struct aead_alg *alg; 2616 2617 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2618 if (!t_alg) 2619 return ERR_PTR(-ENOMEM); 2620 2621 alg = &tmpl->template_aead; 2622 2623 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 2624 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2625 tmpl->driver_name); 2626 alg->base.cra_module = THIS_MODULE; 2627 alg->base.cra_priority = CC_CRA_PRIO; 2628 2629 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 2630 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2631 alg->init = cc_aead_init; 2632 alg->exit = cc_aead_exit; 2633 2634 t_alg->aead_alg = *alg; 2635 2636 t_alg->cipher_mode = tmpl->cipher_mode; 2637 t_alg->flow_mode = tmpl->flow_mode; 2638 t_alg->auth_mode = tmpl->auth_mode; 2639 2640 return t_alg; 2641 } 2642 2643 int cc_aead_free(struct cc_drvdata *drvdata) 2644 { 2645 struct cc_crypto_alg *t_alg, *n; 2646 struct cc_aead_handle *aead_handle = 2647 (struct cc_aead_handle *)drvdata->aead_handle; 2648 2649 if (aead_handle) { 2650 /* Remove registered algs */ 2651 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, 2652 entry) { 2653 crypto_unregister_aead(&t_alg->aead_alg); 2654 list_del(&t_alg->entry); 2655 kfree(t_alg); 2656 } 2657 kfree(aead_handle); 2658 drvdata->aead_handle = NULL; 2659 } 2660 2661 return 0; 2662 } 2663 2664 int cc_aead_alloc(struct cc_drvdata *drvdata) 2665 { 2666 struct cc_aead_handle *aead_handle; 2667 struct cc_crypto_alg *t_alg; 2668 int rc = -ENOMEM; 2669 int alg; 2670 struct device *dev = drvdata_to_dev(drvdata); 2671 2672 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL); 2673 if (!aead_handle) { 2674 rc = -ENOMEM; 2675 goto fail0; 2676 } 2677 2678 INIT_LIST_HEAD(&aead_handle->aead_list); 2679 drvdata->aead_handle = aead_handle; 2680 2681 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2682 MAX_HMAC_DIGEST_SIZE); 2683 2684 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2685 dev_err(dev, "SRAM pool exhausted\n"); 2686 rc = -ENOMEM; 2687 goto fail1; 2688 } 2689 2690 /* Linux crypto */ 2691 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 2692 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 2693 !(drvdata->std_bodies & aead_algs[alg].std_body)) 2694 continue; 2695 2696 t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2697 if (IS_ERR(t_alg)) { 2698 rc = PTR_ERR(t_alg); 2699 dev_err(dev, "%s alg allocation failed\n", 2700 aead_algs[alg].driver_name); 2701 goto fail1; 2702 } 2703 t_alg->drvdata = drvdata; 2704 rc = crypto_register_aead(&t_alg->aead_alg); 2705 if (rc) { 2706 dev_err(dev, "%s alg registration failed\n", 2707 t_alg->aead_alg.base.cra_driver_name); 2708 goto fail2; 2709 } else { 2710 list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2711 dev_dbg(dev, "Registered %s\n", 2712 t_alg->aead_alg.base.cra_driver_name); 2713 } 2714 } 2715 2716 return 0; 2717 2718 fail2: 2719 kfree(t_alg); 2720 fail1: 2721 cc_aead_free(drvdata); 2722 fail0: 2723 return rc; 2724 } 2725