1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/internal/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/gcm.h> 10 #include <linux/rtnetlink.h> 11 #include <crypto/internal/des.h> 12 #include "cc_driver.h" 13 #include "cc_buffer_mgr.h" 14 #include "cc_aead.h" 15 #include "cc_request_mgr.h" 16 #include "cc_hash.h" 17 #include "cc_sram_mgr.h" 18 19 #define template_aead template_u.aead 20 21 #define MAX_AEAD_SETKEY_SEQ 12 22 #define MAX_AEAD_PROCESS_SEQ 23 23 24 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 25 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 26 27 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 28 29 struct cc_aead_handle { 30 u32 sram_workspace_addr; 31 struct list_head aead_list; 32 }; 33 34 struct cc_hmac_s { 35 u8 *padded_authkey; 36 u8 *ipad_opad; /* IPAD, OPAD*/ 37 dma_addr_t padded_authkey_dma_addr; 38 dma_addr_t ipad_opad_dma_addr; 39 }; 40 41 struct cc_xcbc_s { 42 u8 *xcbc_keys; /* K1,K2,K3 */ 43 dma_addr_t xcbc_keys_dma_addr; 44 }; 45 46 struct cc_aead_ctx { 47 struct cc_drvdata *drvdata; 48 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 49 u8 *enckey; 50 dma_addr_t enckey_dma_addr; 51 union { 52 struct cc_hmac_s hmac; 53 struct cc_xcbc_s xcbc; 54 } auth_state; 55 unsigned int enc_keylen; 56 unsigned int auth_keylen; 57 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 58 unsigned int hash_len; 59 enum drv_cipher_mode cipher_mode; 60 enum cc_flow_mode flow_mode; 61 enum drv_hash_mode auth_mode; 62 }; 63 64 static void cc_aead_exit(struct crypto_aead *tfm) 65 { 66 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 67 struct device *dev = drvdata_to_dev(ctx->drvdata); 68 69 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 70 crypto_tfm_alg_name(&tfm->base)); 71 72 /* Unmap enckey buffer */ 73 if (ctx->enckey) { 74 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 75 ctx->enckey_dma_addr); 76 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 77 &ctx->enckey_dma_addr); 78 ctx->enckey_dma_addr = 0; 79 ctx->enckey = NULL; 80 } 81 82 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 83 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 84 85 if (xcbc->xcbc_keys) { 86 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 87 xcbc->xcbc_keys, 88 xcbc->xcbc_keys_dma_addr); 89 } 90 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 91 &xcbc->xcbc_keys_dma_addr); 92 xcbc->xcbc_keys_dma_addr = 0; 93 xcbc->xcbc_keys = NULL; 94 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 95 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 96 97 if (hmac->ipad_opad) { 98 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 99 hmac->ipad_opad, 100 hmac->ipad_opad_dma_addr); 101 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 102 &hmac->ipad_opad_dma_addr); 103 hmac->ipad_opad_dma_addr = 0; 104 hmac->ipad_opad = NULL; 105 } 106 if (hmac->padded_authkey) { 107 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 108 hmac->padded_authkey, 109 hmac->padded_authkey_dma_addr); 110 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 111 &hmac->padded_authkey_dma_addr); 112 hmac->padded_authkey_dma_addr = 0; 113 hmac->padded_authkey = NULL; 114 } 115 } 116 } 117 118 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 119 { 120 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 121 122 return cc_get_default_hash_len(ctx->drvdata); 123 } 124 125 static int cc_aead_init(struct crypto_aead *tfm) 126 { 127 struct aead_alg *alg = crypto_aead_alg(tfm); 128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 129 struct cc_crypto_alg *cc_alg = 130 container_of(alg, struct cc_crypto_alg, aead_alg); 131 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 132 133 dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 134 crypto_tfm_alg_name(&tfm->base)); 135 136 /* Initialize modes in instance */ 137 ctx->cipher_mode = cc_alg->cipher_mode; 138 ctx->flow_mode = cc_alg->flow_mode; 139 ctx->auth_mode = cc_alg->auth_mode; 140 ctx->drvdata = cc_alg->drvdata; 141 crypto_aead_set_reqsize_dma(tfm, sizeof(struct aead_req_ctx)); 142 143 /* Allocate key buffer, cache line aligned */ 144 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 145 &ctx->enckey_dma_addr, GFP_KERNEL); 146 if (!ctx->enckey) { 147 dev_err(dev, "Failed allocating key buffer\n"); 148 goto init_failed; 149 } 150 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 151 ctx->enckey); 152 153 /* Set default authlen value */ 154 155 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 156 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 157 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 158 159 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 160 /* (and temporary for user key - up to 256b) */ 161 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 162 &xcbc->xcbc_keys_dma_addr, 163 GFP_KERNEL); 164 if (!xcbc->xcbc_keys) { 165 dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 166 goto init_failed; 167 } 168 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 169 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 170 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 171 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 172 173 /* Allocate dma-coherent buffer for IPAD + OPAD */ 174 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 175 &hmac->ipad_opad_dma_addr, 176 GFP_KERNEL); 177 178 if (!hmac->ipad_opad) { 179 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 180 goto init_failed; 181 } 182 183 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 184 hmac->ipad_opad); 185 186 hmac->padded_authkey = dma_alloc_coherent(dev, 187 MAX_HMAC_BLOCK_SIZE, 188 pkey_dma, 189 GFP_KERNEL); 190 191 if (!hmac->padded_authkey) { 192 dev_err(dev, "failed to allocate padded_authkey\n"); 193 goto init_failed; 194 } 195 } else { 196 ctx->auth_state.hmac.ipad_opad = NULL; 197 ctx->auth_state.hmac.padded_authkey = NULL; 198 } 199 ctx->hash_len = cc_get_aead_hash_len(tfm); 200 201 return 0; 202 203 init_failed: 204 cc_aead_exit(tfm); 205 return -ENOMEM; 206 } 207 208 static void cc_aead_complete(struct device *dev, void *cc_req, int err) 209 { 210 struct aead_request *areq = (struct aead_request *)cc_req; 211 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 212 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 213 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 214 215 /* BACKLOG notification */ 216 if (err == -EINPROGRESS) 217 goto done; 218 219 cc_unmap_aead_request(dev, areq); 220 221 /* Restore ordinary iv pointer */ 222 areq->iv = areq_ctx->backup_iv; 223 224 if (err) 225 goto done; 226 227 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 228 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 229 ctx->authsize) != 0) { 230 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 231 ctx->authsize, ctx->cipher_mode); 232 /* In case of payload authentication failure, MUST NOT 233 * revealed the decrypted message --> zero its memory. 234 */ 235 sg_zero_buffer(areq->dst, sg_nents(areq->dst), 236 areq->cryptlen, areq->assoclen); 237 err = -EBADMSG; 238 } 239 /*ENCRYPT*/ 240 } else if (areq_ctx->is_icv_fragmented) { 241 u32 skip = areq->cryptlen + areq_ctx->dst_offset; 242 243 cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl, 244 skip, (skip + ctx->authsize), 245 CC_SG_FROM_BUF); 246 } 247 done: 248 aead_request_complete(areq, err); 249 } 250 251 static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 252 struct cc_aead_ctx *ctx) 253 { 254 /* Load the AES key */ 255 hw_desc_init(&desc[0]); 256 /* We are using for the source/user key the same buffer 257 * as for the output keys, * because after this key loading it 258 * is not needed anymore 259 */ 260 set_din_type(&desc[0], DMA_DLLI, 261 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 262 NS_BIT); 263 set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 264 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 265 set_key_size_aes(&desc[0], ctx->auth_keylen); 266 set_flow_mode(&desc[0], S_DIN_to_AES); 267 set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 268 269 hw_desc_init(&desc[1]); 270 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 271 set_flow_mode(&desc[1], DIN_AES_DOUT); 272 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 273 AES_KEYSIZE_128, NS_BIT, 0); 274 275 hw_desc_init(&desc[2]); 276 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 277 set_flow_mode(&desc[2], DIN_AES_DOUT); 278 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 279 + AES_KEYSIZE_128), 280 AES_KEYSIZE_128, NS_BIT, 0); 281 282 hw_desc_init(&desc[3]); 283 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 284 set_flow_mode(&desc[3], DIN_AES_DOUT); 285 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 286 + 2 * AES_KEYSIZE_128), 287 AES_KEYSIZE_128, NS_BIT, 0); 288 289 return 4; 290 } 291 292 static unsigned int hmac_setkey(struct cc_hw_desc *desc, 293 struct cc_aead_ctx *ctx) 294 { 295 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 296 unsigned int digest_ofs = 0; 297 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 298 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 299 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 300 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 301 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 302 303 unsigned int idx = 0; 304 int i; 305 306 /* calc derived HMAC key */ 307 for (i = 0; i < 2; i++) { 308 /* Load hash initial state */ 309 hw_desc_init(&desc[idx]); 310 set_cipher_mode(&desc[idx], hash_mode); 311 set_din_sram(&desc[idx], 312 cc_larval_digest_addr(ctx->drvdata, 313 ctx->auth_mode), 314 digest_size); 315 set_flow_mode(&desc[idx], S_DIN_to_HASH); 316 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 317 idx++; 318 319 /* Load the hash current length*/ 320 hw_desc_init(&desc[idx]); 321 set_cipher_mode(&desc[idx], hash_mode); 322 set_din_const(&desc[idx], 0, ctx->hash_len); 323 set_flow_mode(&desc[idx], S_DIN_to_HASH); 324 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 325 idx++; 326 327 /* Prepare ipad key */ 328 hw_desc_init(&desc[idx]); 329 set_xor_val(&desc[idx], hmac_pad_const[i]); 330 set_cipher_mode(&desc[idx], hash_mode); 331 set_flow_mode(&desc[idx], S_DIN_to_HASH); 332 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 333 idx++; 334 335 /* Perform HASH update */ 336 hw_desc_init(&desc[idx]); 337 set_din_type(&desc[idx], DMA_DLLI, 338 hmac->padded_authkey_dma_addr, 339 SHA256_BLOCK_SIZE, NS_BIT); 340 set_cipher_mode(&desc[idx], hash_mode); 341 set_xor_active(&desc[idx]); 342 set_flow_mode(&desc[idx], DIN_HASH); 343 idx++; 344 345 /* Get the digset */ 346 hw_desc_init(&desc[idx]); 347 set_cipher_mode(&desc[idx], hash_mode); 348 set_dout_dlli(&desc[idx], 349 (hmac->ipad_opad_dma_addr + digest_ofs), 350 digest_size, NS_BIT, 0); 351 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 352 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 353 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 354 idx++; 355 356 digest_ofs += digest_size; 357 } 358 359 return idx; 360 } 361 362 static int validate_keys_sizes(struct cc_aead_ctx *ctx) 363 { 364 struct device *dev = drvdata_to_dev(ctx->drvdata); 365 366 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 367 ctx->enc_keylen, ctx->auth_keylen); 368 369 switch (ctx->auth_mode) { 370 case DRV_HASH_SHA1: 371 case DRV_HASH_SHA256: 372 break; 373 case DRV_HASH_XCBC_MAC: 374 if (ctx->auth_keylen != AES_KEYSIZE_128 && 375 ctx->auth_keylen != AES_KEYSIZE_192 && 376 ctx->auth_keylen != AES_KEYSIZE_256) 377 return -ENOTSUPP; 378 break; 379 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 380 if (ctx->auth_keylen > 0) 381 return -EINVAL; 382 break; 383 default: 384 dev_dbg(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 385 return -EINVAL; 386 } 387 /* Check cipher key size */ 388 if (ctx->flow_mode == S_DIN_to_DES) { 389 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 390 dev_dbg(dev, "Invalid cipher(3DES) key size: %u\n", 391 ctx->enc_keylen); 392 return -EINVAL; 393 } 394 } else { /* Default assumed to be AES ciphers */ 395 if (ctx->enc_keylen != AES_KEYSIZE_128 && 396 ctx->enc_keylen != AES_KEYSIZE_192 && 397 ctx->enc_keylen != AES_KEYSIZE_256) { 398 dev_dbg(dev, "Invalid cipher(AES) key size: %u\n", 399 ctx->enc_keylen); 400 return -EINVAL; 401 } 402 } 403 404 return 0; /* All tests of keys sizes passed */ 405 } 406 407 /* This function prepers the user key so it can pass to the hmac processing 408 * (copy to intenral buffer or hash in case of key longer than block 409 */ 410 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, 411 unsigned int keylen) 412 { 413 dma_addr_t key_dma_addr = 0; 414 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 415 struct device *dev = drvdata_to_dev(ctx->drvdata); 416 u32 larval_addr; 417 struct cc_crypto_req cc_req = {}; 418 unsigned int blocksize; 419 unsigned int digestsize; 420 unsigned int hashmode; 421 unsigned int idx = 0; 422 int rc = 0; 423 u8 *key = NULL; 424 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 425 dma_addr_t padded_authkey_dma_addr = 426 ctx->auth_state.hmac.padded_authkey_dma_addr; 427 428 switch (ctx->auth_mode) { /* auth_key required and >0 */ 429 case DRV_HASH_SHA1: 430 blocksize = SHA1_BLOCK_SIZE; 431 digestsize = SHA1_DIGEST_SIZE; 432 hashmode = DRV_HASH_HW_SHA1; 433 break; 434 case DRV_HASH_SHA256: 435 default: 436 blocksize = SHA256_BLOCK_SIZE; 437 digestsize = SHA256_DIGEST_SIZE; 438 hashmode = DRV_HASH_HW_SHA256; 439 } 440 441 if (keylen != 0) { 442 443 key = kmemdup(authkey, keylen, GFP_KERNEL); 444 if (!key) 445 return -ENOMEM; 446 447 key_dma_addr = dma_map_single(dev, key, keylen, DMA_TO_DEVICE); 448 if (dma_mapping_error(dev, key_dma_addr)) { 449 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 450 key, keylen); 451 kfree_sensitive(key); 452 return -ENOMEM; 453 } 454 if (keylen > blocksize) { 455 /* Load hash initial state */ 456 hw_desc_init(&desc[idx]); 457 set_cipher_mode(&desc[idx], hashmode); 458 larval_addr = cc_larval_digest_addr(ctx->drvdata, 459 ctx->auth_mode); 460 set_din_sram(&desc[idx], larval_addr, digestsize); 461 set_flow_mode(&desc[idx], S_DIN_to_HASH); 462 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 463 idx++; 464 465 /* Load the hash current length*/ 466 hw_desc_init(&desc[idx]); 467 set_cipher_mode(&desc[idx], hashmode); 468 set_din_const(&desc[idx], 0, ctx->hash_len); 469 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 470 set_flow_mode(&desc[idx], S_DIN_to_HASH); 471 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 472 idx++; 473 474 hw_desc_init(&desc[idx]); 475 set_din_type(&desc[idx], DMA_DLLI, 476 key_dma_addr, keylen, NS_BIT); 477 set_flow_mode(&desc[idx], DIN_HASH); 478 idx++; 479 480 /* Get hashed key */ 481 hw_desc_init(&desc[idx]); 482 set_cipher_mode(&desc[idx], hashmode); 483 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 484 digestsize, NS_BIT, 0); 485 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 486 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 487 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 488 set_cipher_config0(&desc[idx], 489 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 490 idx++; 491 492 hw_desc_init(&desc[idx]); 493 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 494 set_flow_mode(&desc[idx], BYPASS); 495 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 496 digestsize), (blocksize - digestsize), 497 NS_BIT, 0); 498 idx++; 499 } else { 500 hw_desc_init(&desc[idx]); 501 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 502 keylen, NS_BIT); 503 set_flow_mode(&desc[idx], BYPASS); 504 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 505 keylen, NS_BIT, 0); 506 idx++; 507 508 if ((blocksize - keylen) != 0) { 509 hw_desc_init(&desc[idx]); 510 set_din_const(&desc[idx], 0, 511 (blocksize - keylen)); 512 set_flow_mode(&desc[idx], BYPASS); 513 set_dout_dlli(&desc[idx], 514 (padded_authkey_dma_addr + 515 keylen), 516 (blocksize - keylen), NS_BIT, 0); 517 idx++; 518 } 519 } 520 } else { 521 hw_desc_init(&desc[idx]); 522 set_din_const(&desc[idx], 0, (blocksize - keylen)); 523 set_flow_mode(&desc[idx], BYPASS); 524 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 525 blocksize, NS_BIT, 0); 526 idx++; 527 } 528 529 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 530 if (rc) 531 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 532 533 if (key_dma_addr) 534 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 535 536 kfree_sensitive(key); 537 538 return rc; 539 } 540 541 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 542 unsigned int keylen) 543 { 544 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 545 struct cc_crypto_req cc_req = {}; 546 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 547 unsigned int seq_len = 0; 548 struct device *dev = drvdata_to_dev(ctx->drvdata); 549 const u8 *enckey, *authkey; 550 int rc; 551 552 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 553 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 554 555 /* STAT_PHASE_0: Init and sanity checks */ 556 557 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 558 struct crypto_authenc_keys keys; 559 560 rc = crypto_authenc_extractkeys(&keys, key, keylen); 561 if (rc) 562 return rc; 563 enckey = keys.enckey; 564 authkey = keys.authkey; 565 ctx->enc_keylen = keys.enckeylen; 566 ctx->auth_keylen = keys.authkeylen; 567 568 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 569 /* the nonce is stored in bytes at end of key */ 570 if (ctx->enc_keylen < 571 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 572 return -EINVAL; 573 /* Copy nonce from last 4 bytes in CTR key to 574 * first 4 bytes in CTR IV 575 */ 576 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - 577 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); 578 /* Set CTR key size */ 579 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 580 } 581 } else { /* non-authenc - has just one key */ 582 enckey = key; 583 authkey = NULL; 584 ctx->enc_keylen = keylen; 585 ctx->auth_keylen = 0; 586 } 587 588 rc = validate_keys_sizes(ctx); 589 if (rc) 590 return rc; 591 592 /* STAT_PHASE_1: Copy key to ctx */ 593 594 /* Get key material */ 595 memcpy(ctx->enckey, enckey, ctx->enc_keylen); 596 if (ctx->enc_keylen == 24) 597 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 598 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 599 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, 600 ctx->auth_keylen); 601 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 602 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); 603 if (rc) 604 return rc; 605 } 606 607 /* STAT_PHASE_2: Create sequence */ 608 609 switch (ctx->auth_mode) { 610 case DRV_HASH_SHA1: 611 case DRV_HASH_SHA256: 612 seq_len = hmac_setkey(desc, ctx); 613 break; 614 case DRV_HASH_XCBC_MAC: 615 seq_len = xcbc_setkey(desc, ctx); 616 break; 617 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 618 break; /* No auth. key setup */ 619 default: 620 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 621 return -ENOTSUPP; 622 } 623 624 /* STAT_PHASE_3: Submit sequence to HW */ 625 626 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 627 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 628 if (rc) { 629 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 630 return rc; 631 } 632 } 633 634 /* Update STAT_PHASE_3 */ 635 return rc; 636 } 637 638 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 639 unsigned int keylen) 640 { 641 struct crypto_authenc_keys keys; 642 int err; 643 644 err = crypto_authenc_extractkeys(&keys, key, keylen); 645 if (unlikely(err)) 646 return err; 647 648 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 649 cc_aead_setkey(aead, key, keylen); 650 651 memzero_explicit(&keys, sizeof(keys)); 652 return err; 653 } 654 655 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 656 unsigned int keylen) 657 { 658 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 659 660 if (keylen < 3) 661 return -EINVAL; 662 663 keylen -= 3; 664 memcpy(ctx->ctr_nonce, key + keylen, 3); 665 666 return cc_aead_setkey(tfm, key, keylen); 667 } 668 669 static int cc_aead_setauthsize(struct crypto_aead *authenc, 670 unsigned int authsize) 671 { 672 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 673 struct device *dev = drvdata_to_dev(ctx->drvdata); 674 675 /* Unsupported auth. sizes */ 676 if (authsize == 0 || 677 authsize > crypto_aead_maxauthsize(authenc)) { 678 return -ENOTSUPP; 679 } 680 681 ctx->authsize = authsize; 682 dev_dbg(dev, "authlen=%d\n", ctx->authsize); 683 684 return 0; 685 } 686 687 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 688 unsigned int authsize) 689 { 690 switch (authsize) { 691 case 8: 692 case 12: 693 case 16: 694 break; 695 default: 696 return -EINVAL; 697 } 698 699 return cc_aead_setauthsize(authenc, authsize); 700 } 701 702 static int cc_ccm_setauthsize(struct crypto_aead *authenc, 703 unsigned int authsize) 704 { 705 switch (authsize) { 706 case 4: 707 case 6: 708 case 8: 709 case 10: 710 case 12: 711 case 14: 712 case 16: 713 break; 714 default: 715 return -EINVAL; 716 } 717 718 return cc_aead_setauthsize(authenc, authsize); 719 } 720 721 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 722 struct cc_hw_desc desc[], unsigned int *seq_size) 723 { 724 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 725 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 726 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 727 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 728 unsigned int idx = *seq_size; 729 struct device *dev = drvdata_to_dev(ctx->drvdata); 730 731 switch (assoc_dma_type) { 732 case CC_DMA_BUF_DLLI: 733 dev_dbg(dev, "ASSOC buffer type DLLI\n"); 734 hw_desc_init(&desc[idx]); 735 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 736 areq_ctx->assoclen, NS_BIT); 737 set_flow_mode(&desc[idx], flow_mode); 738 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 739 areq_ctx->cryptlen > 0) 740 set_din_not_last_indication(&desc[idx]); 741 break; 742 case CC_DMA_BUF_MLLI: 743 dev_dbg(dev, "ASSOC buffer type MLLI\n"); 744 hw_desc_init(&desc[idx]); 745 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 746 areq_ctx->assoc.mlli_nents, NS_BIT); 747 set_flow_mode(&desc[idx], flow_mode); 748 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 749 areq_ctx->cryptlen > 0) 750 set_din_not_last_indication(&desc[idx]); 751 break; 752 case CC_DMA_BUF_NULL: 753 default: 754 dev_err(dev, "Invalid ASSOC buffer type\n"); 755 } 756 757 *seq_size = (++idx); 758 } 759 760 static void cc_proc_authen_desc(struct aead_request *areq, 761 unsigned int flow_mode, 762 struct cc_hw_desc desc[], 763 unsigned int *seq_size, int direct) 764 { 765 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 766 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 767 unsigned int idx = *seq_size; 768 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 769 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 770 struct device *dev = drvdata_to_dev(ctx->drvdata); 771 772 switch (data_dma_type) { 773 case CC_DMA_BUF_DLLI: 774 { 775 struct scatterlist *cipher = 776 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 777 areq_ctx->dst_sgl : areq_ctx->src_sgl; 778 779 unsigned int offset = 780 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 781 areq_ctx->dst_offset : areq_ctx->src_offset; 782 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 783 hw_desc_init(&desc[idx]); 784 set_din_type(&desc[idx], DMA_DLLI, 785 (sg_dma_address(cipher) + offset), 786 areq_ctx->cryptlen, NS_BIT); 787 set_flow_mode(&desc[idx], flow_mode); 788 break; 789 } 790 case CC_DMA_BUF_MLLI: 791 { 792 /* DOUBLE-PASS flow (as default) 793 * assoc. + iv + data -compact in one table 794 * if assoclen is ZERO only IV perform 795 */ 796 u32 mlli_addr = areq_ctx->assoc.sram_addr; 797 u32 mlli_nents = areq_ctx->assoc.mlli_nents; 798 799 if (areq_ctx->is_single_pass) { 800 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 801 mlli_addr = areq_ctx->dst.sram_addr; 802 mlli_nents = areq_ctx->dst.mlli_nents; 803 } else { 804 mlli_addr = areq_ctx->src.sram_addr; 805 mlli_nents = areq_ctx->src.mlli_nents; 806 } 807 } 808 809 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 810 hw_desc_init(&desc[idx]); 811 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 812 NS_BIT); 813 set_flow_mode(&desc[idx], flow_mode); 814 break; 815 } 816 case CC_DMA_BUF_NULL: 817 default: 818 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 819 } 820 821 *seq_size = (++idx); 822 } 823 824 static void cc_proc_cipher_desc(struct aead_request *areq, 825 unsigned int flow_mode, 826 struct cc_hw_desc desc[], 827 unsigned int *seq_size) 828 { 829 unsigned int idx = *seq_size; 830 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 831 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 832 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 833 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 834 struct device *dev = drvdata_to_dev(ctx->drvdata); 835 836 if (areq_ctx->cryptlen == 0) 837 return; /*null processing*/ 838 839 switch (data_dma_type) { 840 case CC_DMA_BUF_DLLI: 841 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 842 hw_desc_init(&desc[idx]); 843 set_din_type(&desc[idx], DMA_DLLI, 844 (sg_dma_address(areq_ctx->src_sgl) + 845 areq_ctx->src_offset), areq_ctx->cryptlen, 846 NS_BIT); 847 set_dout_dlli(&desc[idx], 848 (sg_dma_address(areq_ctx->dst_sgl) + 849 areq_ctx->dst_offset), 850 areq_ctx->cryptlen, NS_BIT, 0); 851 set_flow_mode(&desc[idx], flow_mode); 852 break; 853 case CC_DMA_BUF_MLLI: 854 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 855 hw_desc_init(&desc[idx]); 856 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 857 areq_ctx->src.mlli_nents, NS_BIT); 858 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 859 areq_ctx->dst.mlli_nents, NS_BIT, 0); 860 set_flow_mode(&desc[idx], flow_mode); 861 break; 862 case CC_DMA_BUF_NULL: 863 default: 864 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 865 } 866 867 *seq_size = (++idx); 868 } 869 870 static void cc_proc_digest_desc(struct aead_request *req, 871 struct cc_hw_desc desc[], 872 unsigned int *seq_size) 873 { 874 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 875 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 876 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 877 unsigned int idx = *seq_size; 878 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 879 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 880 int direct = req_ctx->gen_ctx.op_type; 881 882 /* Get final ICV result */ 883 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 884 hw_desc_init(&desc[idx]); 885 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 886 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 887 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 888 NS_BIT, 1); 889 set_queue_last_ind(ctx->drvdata, &desc[idx]); 890 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 891 set_aes_not_hash_mode(&desc[idx]); 892 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 893 } else { 894 set_cipher_config0(&desc[idx], 895 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 896 set_cipher_mode(&desc[idx], hash_mode); 897 } 898 } else { /*Decrypt*/ 899 /* Get ICV out from hardware */ 900 hw_desc_init(&desc[idx]); 901 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 902 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 903 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 904 ctx->authsize, NS_BIT, 1); 905 set_queue_last_ind(ctx->drvdata, &desc[idx]); 906 set_cipher_config0(&desc[idx], 907 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 908 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 909 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 910 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 911 set_aes_not_hash_mode(&desc[idx]); 912 } else { 913 set_cipher_mode(&desc[idx], hash_mode); 914 } 915 } 916 917 *seq_size = (++idx); 918 } 919 920 static void cc_set_cipher_desc(struct aead_request *req, 921 struct cc_hw_desc desc[], 922 unsigned int *seq_size) 923 { 924 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 925 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 926 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 927 unsigned int hw_iv_size = req_ctx->hw_iv_size; 928 unsigned int idx = *seq_size; 929 int direct = req_ctx->gen_ctx.op_type; 930 931 /* Setup cipher state */ 932 hw_desc_init(&desc[idx]); 933 set_cipher_config0(&desc[idx], direct); 934 set_flow_mode(&desc[idx], ctx->flow_mode); 935 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 936 hw_iv_size, NS_BIT); 937 if (ctx->cipher_mode == DRV_CIPHER_CTR) 938 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 939 else 940 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 941 set_cipher_mode(&desc[idx], ctx->cipher_mode); 942 idx++; 943 944 /* Setup enc. key */ 945 hw_desc_init(&desc[idx]); 946 set_cipher_config0(&desc[idx], direct); 947 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 948 set_flow_mode(&desc[idx], ctx->flow_mode); 949 if (ctx->flow_mode == S_DIN_to_AES) { 950 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 951 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 952 ctx->enc_keylen), NS_BIT); 953 set_key_size_aes(&desc[idx], ctx->enc_keylen); 954 } else { 955 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 956 ctx->enc_keylen, NS_BIT); 957 set_key_size_des(&desc[idx], ctx->enc_keylen); 958 } 959 set_cipher_mode(&desc[idx], ctx->cipher_mode); 960 idx++; 961 962 *seq_size = idx; 963 } 964 965 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 966 unsigned int *seq_size, unsigned int data_flow_mode) 967 { 968 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 969 int direct = req_ctx->gen_ctx.op_type; 970 unsigned int idx = *seq_size; 971 972 if (req_ctx->cryptlen == 0) 973 return; /*null processing*/ 974 975 cc_set_cipher_desc(req, desc, &idx); 976 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 977 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 978 /* We must wait for DMA to write all cipher */ 979 hw_desc_init(&desc[idx]); 980 set_din_no_dma(&desc[idx], 0, 0xfffff0); 981 set_dout_no_dma(&desc[idx], 0, 0, 1); 982 idx++; 983 } 984 985 *seq_size = idx; 986 } 987 988 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 989 unsigned int *seq_size) 990 { 991 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 992 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 993 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 994 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 995 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 996 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 997 unsigned int idx = *seq_size; 998 999 /* Loading hash ipad xor key state */ 1000 hw_desc_init(&desc[idx]); 1001 set_cipher_mode(&desc[idx], hash_mode); 1002 set_din_type(&desc[idx], DMA_DLLI, 1003 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1004 NS_BIT); 1005 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1006 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1007 idx++; 1008 1009 /* Load init. digest len (64 bytes) */ 1010 hw_desc_init(&desc[idx]); 1011 set_cipher_mode(&desc[idx], hash_mode); 1012 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1013 ctx->hash_len); 1014 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1015 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1016 idx++; 1017 1018 *seq_size = idx; 1019 } 1020 1021 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1022 unsigned int *seq_size) 1023 { 1024 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1025 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1026 unsigned int idx = *seq_size; 1027 1028 /* Loading MAC state */ 1029 hw_desc_init(&desc[idx]); 1030 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1031 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1032 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1033 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1034 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1035 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1036 set_aes_not_hash_mode(&desc[idx]); 1037 idx++; 1038 1039 /* Setup XCBC MAC K1 */ 1040 hw_desc_init(&desc[idx]); 1041 set_din_type(&desc[idx], DMA_DLLI, 1042 ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1043 AES_KEYSIZE_128, NS_BIT); 1044 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1045 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1046 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1047 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1048 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1049 set_aes_not_hash_mode(&desc[idx]); 1050 idx++; 1051 1052 /* Setup XCBC MAC K2 */ 1053 hw_desc_init(&desc[idx]); 1054 set_din_type(&desc[idx], DMA_DLLI, 1055 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1056 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1057 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1058 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1059 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1060 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1061 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1062 set_aes_not_hash_mode(&desc[idx]); 1063 idx++; 1064 1065 /* Setup XCBC MAC K3 */ 1066 hw_desc_init(&desc[idx]); 1067 set_din_type(&desc[idx], DMA_DLLI, 1068 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1069 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1070 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1071 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1072 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1073 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1074 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1075 set_aes_not_hash_mode(&desc[idx]); 1076 idx++; 1077 1078 *seq_size = idx; 1079 } 1080 1081 static void cc_proc_header_desc(struct aead_request *req, 1082 struct cc_hw_desc desc[], 1083 unsigned int *seq_size) 1084 { 1085 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1086 unsigned int idx = *seq_size; 1087 1088 /* Hash associated data */ 1089 if (areq_ctx->assoclen > 0) 1090 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1091 1092 /* Hash IV */ 1093 *seq_size = idx; 1094 } 1095 1096 static void cc_proc_scheme_desc(struct aead_request *req, 1097 struct cc_hw_desc desc[], 1098 unsigned int *seq_size) 1099 { 1100 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1101 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1102 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1103 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1104 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1105 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1106 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1107 unsigned int idx = *seq_size; 1108 1109 hw_desc_init(&desc[idx]); 1110 set_cipher_mode(&desc[idx], hash_mode); 1111 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1112 ctx->hash_len); 1113 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1114 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1115 set_cipher_do(&desc[idx], DO_PAD); 1116 idx++; 1117 1118 /* Get final ICV result */ 1119 hw_desc_init(&desc[idx]); 1120 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1121 digest_size); 1122 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1123 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1124 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1125 set_cipher_mode(&desc[idx], hash_mode); 1126 idx++; 1127 1128 /* Loading hash opad xor key state */ 1129 hw_desc_init(&desc[idx]); 1130 set_cipher_mode(&desc[idx], hash_mode); 1131 set_din_type(&desc[idx], DMA_DLLI, 1132 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1133 digest_size, NS_BIT); 1134 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1135 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1136 idx++; 1137 1138 /* Load init. digest len (64 bytes) */ 1139 hw_desc_init(&desc[idx]); 1140 set_cipher_mode(&desc[idx], hash_mode); 1141 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1142 ctx->hash_len); 1143 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1144 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1145 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1146 idx++; 1147 1148 /* Perform HASH update */ 1149 hw_desc_init(&desc[idx]); 1150 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1151 digest_size); 1152 set_flow_mode(&desc[idx], DIN_HASH); 1153 idx++; 1154 1155 *seq_size = idx; 1156 } 1157 1158 static void cc_mlli_to_sram(struct aead_request *req, 1159 struct cc_hw_desc desc[], unsigned int *seq_size) 1160 { 1161 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1162 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1163 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1164 struct device *dev = drvdata_to_dev(ctx->drvdata); 1165 1166 if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1167 req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1168 !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { 1169 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1170 ctx->drvdata->mlli_sram_addr, 1171 req_ctx->mlli_params.mlli_len); 1172 /* Copy MLLI table host-to-sram */ 1173 hw_desc_init(&desc[*seq_size]); 1174 set_din_type(&desc[*seq_size], DMA_DLLI, 1175 req_ctx->mlli_params.mlli_dma_addr, 1176 req_ctx->mlli_params.mlli_len, NS_BIT); 1177 set_dout_sram(&desc[*seq_size], 1178 ctx->drvdata->mlli_sram_addr, 1179 req_ctx->mlli_params.mlli_len); 1180 set_flow_mode(&desc[*seq_size], BYPASS); 1181 (*seq_size)++; 1182 } 1183 } 1184 1185 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1186 enum cc_flow_mode setup_flow_mode, 1187 bool is_single_pass) 1188 { 1189 enum cc_flow_mode data_flow_mode; 1190 1191 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1192 if (setup_flow_mode == S_DIN_to_AES) 1193 data_flow_mode = is_single_pass ? 1194 AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1195 else 1196 data_flow_mode = is_single_pass ? 1197 DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1198 } else { /* Decrypt */ 1199 if (setup_flow_mode == S_DIN_to_AES) 1200 data_flow_mode = is_single_pass ? 1201 AES_and_HASH : DIN_AES_DOUT; 1202 else 1203 data_flow_mode = is_single_pass ? 1204 DES_and_HASH : DIN_DES_DOUT; 1205 } 1206 1207 return data_flow_mode; 1208 } 1209 1210 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1211 unsigned int *seq_size) 1212 { 1213 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1214 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1215 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1216 int direct = req_ctx->gen_ctx.op_type; 1217 unsigned int data_flow_mode = 1218 cc_get_data_flow(direct, ctx->flow_mode, 1219 req_ctx->is_single_pass); 1220 1221 if (req_ctx->is_single_pass) { 1222 /* 1223 * Single-pass flow 1224 */ 1225 cc_set_hmac_desc(req, desc, seq_size); 1226 cc_set_cipher_desc(req, desc, seq_size); 1227 cc_proc_header_desc(req, desc, seq_size); 1228 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1229 cc_proc_scheme_desc(req, desc, seq_size); 1230 cc_proc_digest_desc(req, desc, seq_size); 1231 return; 1232 } 1233 1234 /* 1235 * Double-pass flow 1236 * Fallback for unsupported single-pass modes, 1237 * i.e. using assoc. data of non-word-multiple 1238 */ 1239 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1240 /* encrypt first.. */ 1241 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1242 /* authenc after..*/ 1243 cc_set_hmac_desc(req, desc, seq_size); 1244 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1245 cc_proc_scheme_desc(req, desc, seq_size); 1246 cc_proc_digest_desc(req, desc, seq_size); 1247 1248 } else { /*DECRYPT*/ 1249 /* authenc first..*/ 1250 cc_set_hmac_desc(req, desc, seq_size); 1251 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1252 cc_proc_scheme_desc(req, desc, seq_size); 1253 /* decrypt after.. */ 1254 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1255 /* read the digest result with setting the completion bit 1256 * must be after the cipher operation 1257 */ 1258 cc_proc_digest_desc(req, desc, seq_size); 1259 } 1260 } 1261 1262 static void 1263 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1264 unsigned int *seq_size) 1265 { 1266 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1267 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1268 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1269 int direct = req_ctx->gen_ctx.op_type; 1270 unsigned int data_flow_mode = 1271 cc_get_data_flow(direct, ctx->flow_mode, 1272 req_ctx->is_single_pass); 1273 1274 if (req_ctx->is_single_pass) { 1275 /* 1276 * Single-pass flow 1277 */ 1278 cc_set_xcbc_desc(req, desc, seq_size); 1279 cc_set_cipher_desc(req, desc, seq_size); 1280 cc_proc_header_desc(req, desc, seq_size); 1281 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1282 cc_proc_digest_desc(req, desc, seq_size); 1283 return; 1284 } 1285 1286 /* 1287 * Double-pass flow 1288 * Fallback for unsupported single-pass modes, 1289 * i.e. using assoc. data of non-word-multiple 1290 */ 1291 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1292 /* encrypt first.. */ 1293 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1294 /* authenc after.. */ 1295 cc_set_xcbc_desc(req, desc, seq_size); 1296 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1297 cc_proc_digest_desc(req, desc, seq_size); 1298 } else { /*DECRYPT*/ 1299 /* authenc first.. */ 1300 cc_set_xcbc_desc(req, desc, seq_size); 1301 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1302 /* decrypt after..*/ 1303 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1304 /* read the digest result with setting the completion bit 1305 * must be after the cipher operation 1306 */ 1307 cc_proc_digest_desc(req, desc, seq_size); 1308 } 1309 } 1310 1311 static int validate_data_size(struct cc_aead_ctx *ctx, 1312 enum drv_crypto_direction direct, 1313 struct aead_request *req) 1314 { 1315 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1316 struct device *dev = drvdata_to_dev(ctx->drvdata); 1317 unsigned int assoclen = areq_ctx->assoclen; 1318 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1319 (req->cryptlen - ctx->authsize) : req->cryptlen; 1320 1321 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1322 req->cryptlen < ctx->authsize) 1323 goto data_size_err; 1324 1325 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1326 1327 switch (ctx->flow_mode) { 1328 case S_DIN_to_AES: 1329 if (ctx->cipher_mode == DRV_CIPHER_CBC && 1330 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1331 goto data_size_err; 1332 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1333 break; 1334 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1335 if (areq_ctx->plaintext_authenticate_only) 1336 areq_ctx->is_single_pass = false; 1337 break; 1338 } 1339 1340 if (!IS_ALIGNED(assoclen, sizeof(u32))) 1341 areq_ctx->is_single_pass = false; 1342 1343 if (ctx->cipher_mode == DRV_CIPHER_CTR && 1344 !IS_ALIGNED(cipherlen, sizeof(u32))) 1345 areq_ctx->is_single_pass = false; 1346 1347 break; 1348 case S_DIN_to_DES: 1349 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1350 goto data_size_err; 1351 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1352 areq_ctx->is_single_pass = false; 1353 break; 1354 default: 1355 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1356 goto data_size_err; 1357 } 1358 1359 return 0; 1360 1361 data_size_err: 1362 return -EINVAL; 1363 } 1364 1365 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1366 { 1367 unsigned int len = 0; 1368 1369 if (header_size == 0) 1370 return 0; 1371 1372 if (header_size < ((1UL << 16) - (1UL << 8))) { 1373 len = 2; 1374 1375 pa0_buff[0] = (header_size >> 8) & 0xFF; 1376 pa0_buff[1] = header_size & 0xFF; 1377 } else { 1378 len = 6; 1379 1380 pa0_buff[0] = 0xFF; 1381 pa0_buff[1] = 0xFE; 1382 pa0_buff[2] = (header_size >> 24) & 0xFF; 1383 pa0_buff[3] = (header_size >> 16) & 0xFF; 1384 pa0_buff[4] = (header_size >> 8) & 0xFF; 1385 pa0_buff[5] = header_size & 0xFF; 1386 } 1387 1388 return len; 1389 } 1390 1391 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1392 { 1393 __be32 data; 1394 1395 memset(block, 0, csize); 1396 block += csize; 1397 1398 if (csize >= 4) 1399 csize = 4; 1400 else if (msglen > (1 << (8 * csize))) 1401 return -EOVERFLOW; 1402 1403 data = cpu_to_be32(msglen); 1404 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1405 1406 return 0; 1407 } 1408 1409 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1410 unsigned int *seq_size) 1411 { 1412 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1413 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1414 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1415 unsigned int idx = *seq_size; 1416 unsigned int cipher_flow_mode; 1417 dma_addr_t mac_result; 1418 1419 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1420 cipher_flow_mode = AES_to_HASH_and_DOUT; 1421 mac_result = req_ctx->mac_buf_dma_addr; 1422 } else { /* Encrypt */ 1423 cipher_flow_mode = AES_and_HASH; 1424 mac_result = req_ctx->icv_dma_addr; 1425 } 1426 1427 /* load key */ 1428 hw_desc_init(&desc[idx]); 1429 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1430 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1431 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1432 ctx->enc_keylen), NS_BIT); 1433 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1434 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1435 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1436 set_flow_mode(&desc[idx], S_DIN_to_AES); 1437 idx++; 1438 1439 /* load ctr state */ 1440 hw_desc_init(&desc[idx]); 1441 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1442 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1443 set_din_type(&desc[idx], DMA_DLLI, 1444 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1445 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1446 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1447 set_flow_mode(&desc[idx], S_DIN_to_AES); 1448 idx++; 1449 1450 /* load MAC key */ 1451 hw_desc_init(&desc[idx]); 1452 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1453 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1454 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1455 ctx->enc_keylen), NS_BIT); 1456 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1457 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1458 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1459 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1460 set_aes_not_hash_mode(&desc[idx]); 1461 idx++; 1462 1463 /* load MAC state */ 1464 hw_desc_init(&desc[idx]); 1465 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1466 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1467 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1468 AES_BLOCK_SIZE, NS_BIT); 1469 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1470 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1471 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1472 set_aes_not_hash_mode(&desc[idx]); 1473 idx++; 1474 1475 /* process assoc data */ 1476 if (req_ctx->assoclen > 0) { 1477 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1478 } else { 1479 hw_desc_init(&desc[idx]); 1480 set_din_type(&desc[idx], DMA_DLLI, 1481 sg_dma_address(&req_ctx->ccm_adata_sg), 1482 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1483 set_flow_mode(&desc[idx], DIN_HASH); 1484 idx++; 1485 } 1486 1487 /* process the cipher */ 1488 if (req_ctx->cryptlen) 1489 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1490 1491 /* Read temporal MAC */ 1492 hw_desc_init(&desc[idx]); 1493 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1494 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1495 NS_BIT, 0); 1496 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1497 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1498 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1499 set_aes_not_hash_mode(&desc[idx]); 1500 idx++; 1501 1502 /* load AES-CTR state (for last MAC calculation)*/ 1503 hw_desc_init(&desc[idx]); 1504 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1505 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1506 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1507 AES_BLOCK_SIZE, NS_BIT); 1508 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1509 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1510 set_flow_mode(&desc[idx], S_DIN_to_AES); 1511 idx++; 1512 1513 hw_desc_init(&desc[idx]); 1514 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1515 set_dout_no_dma(&desc[idx], 0, 0, 1); 1516 idx++; 1517 1518 /* encrypt the "T" value and store MAC in mac_state */ 1519 hw_desc_init(&desc[idx]); 1520 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1521 ctx->authsize, NS_BIT); 1522 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1523 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1524 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1525 idx++; 1526 1527 *seq_size = idx; 1528 return 0; 1529 } 1530 1531 static int config_ccm_adata(struct aead_request *req) 1532 { 1533 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1534 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1535 struct device *dev = drvdata_to_dev(ctx->drvdata); 1536 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1537 //unsigned int size_of_a = 0, rem_a_size = 0; 1538 unsigned int lp = req->iv[0]; 1539 /* Note: The code assume that req->iv[0] already contains the value 1540 * of L' of RFC3610 1541 */ 1542 unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1543 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1544 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1545 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1546 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1547 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1548 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1549 req->cryptlen : 1550 (req->cryptlen - ctx->authsize); 1551 int rc; 1552 1553 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1554 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1555 1556 /* taken from crypto/ccm.c */ 1557 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1558 if (l < 2 || l > 8) { 1559 dev_dbg(dev, "illegal iv value %X\n", req->iv[0]); 1560 return -EINVAL; 1561 } 1562 memcpy(b0, req->iv, AES_BLOCK_SIZE); 1563 1564 /* format control info per RFC 3610 and 1565 * NIST Special Publication 800-38C 1566 */ 1567 *b0 |= (8 * ((m - 2) / 2)); 1568 if (req_ctx->assoclen > 0) 1569 *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1570 1571 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1572 if (rc) { 1573 dev_err(dev, "message len overflow detected"); 1574 return rc; 1575 } 1576 /* END of "taken from crypto/ccm.c" */ 1577 1578 /* l(a) - size of associated data. */ 1579 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); 1580 1581 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1582 req->iv[15] = 1; 1583 1584 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1585 ctr_count_0[15] = 0; 1586 1587 return 0; 1588 } 1589 1590 static void cc_proc_rfc4309_ccm(struct aead_request *req) 1591 { 1592 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1593 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1594 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1595 1596 /* L' */ 1597 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1598 /* For RFC 4309, always use 4 bytes for message length 1599 * (at most 2^32-1 bytes). 1600 */ 1601 areq_ctx->ctr_iv[0] = 3; 1602 1603 /* In RFC 4309 there is an 11-bytes nonce+IV part, 1604 * that we build here. 1605 */ 1606 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1607 CCM_BLOCK_NONCE_SIZE); 1608 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1609 CCM_BLOCK_IV_SIZE); 1610 req->iv = areq_ctx->ctr_iv; 1611 } 1612 1613 static void cc_set_ghash_desc(struct aead_request *req, 1614 struct cc_hw_desc desc[], unsigned int *seq_size) 1615 { 1616 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1617 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1618 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1619 unsigned int idx = *seq_size; 1620 1621 /* load key to AES*/ 1622 hw_desc_init(&desc[idx]); 1623 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1624 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1625 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1626 ctx->enc_keylen, NS_BIT); 1627 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1628 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1629 set_flow_mode(&desc[idx], S_DIN_to_AES); 1630 idx++; 1631 1632 /* process one zero block to generate hkey */ 1633 hw_desc_init(&desc[idx]); 1634 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1635 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1636 NS_BIT, 0); 1637 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1638 idx++; 1639 1640 /* Memory Barrier */ 1641 hw_desc_init(&desc[idx]); 1642 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1643 set_dout_no_dma(&desc[idx], 0, 0, 1); 1644 idx++; 1645 1646 /* Load GHASH subkey */ 1647 hw_desc_init(&desc[idx]); 1648 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1649 AES_BLOCK_SIZE, NS_BIT); 1650 set_dout_no_dma(&desc[idx], 0, 0, 1); 1651 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1652 set_aes_not_hash_mode(&desc[idx]); 1653 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1654 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1655 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1656 idx++; 1657 1658 /* Configure Hash Engine to work with GHASH. 1659 * Since it was not possible to extend HASH submodes to add GHASH, 1660 * The following command is necessary in order to 1661 * select GHASH (according to HW designers) 1662 */ 1663 hw_desc_init(&desc[idx]); 1664 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1665 set_dout_no_dma(&desc[idx], 0, 0, 1); 1666 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1667 set_aes_not_hash_mode(&desc[idx]); 1668 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1669 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1670 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1671 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1672 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1673 idx++; 1674 1675 /* Load GHASH initial STATE (which is 0). (for any hash there is an 1676 * initial state) 1677 */ 1678 hw_desc_init(&desc[idx]); 1679 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1680 set_dout_no_dma(&desc[idx], 0, 0, 1); 1681 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1682 set_aes_not_hash_mode(&desc[idx]); 1683 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1684 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1685 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1686 idx++; 1687 1688 *seq_size = idx; 1689 } 1690 1691 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1692 unsigned int *seq_size) 1693 { 1694 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1695 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1696 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1697 unsigned int idx = *seq_size; 1698 1699 /* load key to AES*/ 1700 hw_desc_init(&desc[idx]); 1701 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1702 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1703 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1704 ctx->enc_keylen, NS_BIT); 1705 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1706 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1707 set_flow_mode(&desc[idx], S_DIN_to_AES); 1708 idx++; 1709 1710 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1711 /* load AES/CTR initial CTR value inc by 2*/ 1712 hw_desc_init(&desc[idx]); 1713 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1714 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1715 set_din_type(&desc[idx], DMA_DLLI, 1716 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1717 NS_BIT); 1718 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1719 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1720 set_flow_mode(&desc[idx], S_DIN_to_AES); 1721 idx++; 1722 } 1723 1724 *seq_size = idx; 1725 } 1726 1727 static void cc_proc_gcm_result(struct aead_request *req, 1728 struct cc_hw_desc desc[], 1729 unsigned int *seq_size) 1730 { 1731 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1732 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1733 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1734 dma_addr_t mac_result; 1735 unsigned int idx = *seq_size; 1736 1737 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1738 mac_result = req_ctx->mac_buf_dma_addr; 1739 } else { /* Encrypt */ 1740 mac_result = req_ctx->icv_dma_addr; 1741 } 1742 1743 /* process(ghash) gcm_block_len */ 1744 hw_desc_init(&desc[idx]); 1745 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1746 AES_BLOCK_SIZE, NS_BIT); 1747 set_flow_mode(&desc[idx], DIN_HASH); 1748 idx++; 1749 1750 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1751 hw_desc_init(&desc[idx]); 1752 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1753 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1754 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1755 NS_BIT, 0); 1756 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1757 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1758 set_aes_not_hash_mode(&desc[idx]); 1759 1760 idx++; 1761 1762 /* load AES/CTR initial CTR value inc by 1*/ 1763 hw_desc_init(&desc[idx]); 1764 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1765 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1766 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1767 AES_BLOCK_SIZE, NS_BIT); 1768 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1769 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1770 set_flow_mode(&desc[idx], S_DIN_to_AES); 1771 idx++; 1772 1773 /* Memory Barrier */ 1774 hw_desc_init(&desc[idx]); 1775 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1776 set_dout_no_dma(&desc[idx], 0, 0, 1); 1777 idx++; 1778 1779 /* process GCTR on stored GHASH and store MAC in mac_state*/ 1780 hw_desc_init(&desc[idx]); 1781 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1782 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1783 AES_BLOCK_SIZE, NS_BIT); 1784 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1785 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1786 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1787 idx++; 1788 1789 *seq_size = idx; 1790 } 1791 1792 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1793 unsigned int *seq_size) 1794 { 1795 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1796 unsigned int cipher_flow_mode; 1797 1798 //in RFC4543 no data to encrypt. just copy data from src to dest. 1799 if (req_ctx->plaintext_authenticate_only) { 1800 cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1801 cc_set_ghash_desc(req, desc, seq_size); 1802 /* process(ghash) assoc data */ 1803 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1804 cc_set_gctr_desc(req, desc, seq_size); 1805 cc_proc_gcm_result(req, desc, seq_size); 1806 return 0; 1807 } 1808 1809 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1810 cipher_flow_mode = AES_and_HASH; 1811 } else { /* Encrypt */ 1812 cipher_flow_mode = AES_to_HASH_and_DOUT; 1813 } 1814 1815 // for gcm and rfc4106. 1816 cc_set_ghash_desc(req, desc, seq_size); 1817 /* process(ghash) assoc data */ 1818 if (req_ctx->assoclen > 0) 1819 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1820 cc_set_gctr_desc(req, desc, seq_size); 1821 /* process(gctr+ghash) */ 1822 if (req_ctx->cryptlen) 1823 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1824 cc_proc_gcm_result(req, desc, seq_size); 1825 1826 return 0; 1827 } 1828 1829 static int config_gcm_context(struct aead_request *req) 1830 { 1831 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1832 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1833 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1834 struct device *dev = drvdata_to_dev(ctx->drvdata); 1835 1836 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1837 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1838 req->cryptlen : 1839 (req->cryptlen - ctx->authsize); 1840 __be32 counter = cpu_to_be32(2); 1841 1842 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", 1843 __func__, cryptlen, req_ctx->assoclen, ctx->authsize); 1844 1845 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1846 1847 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1848 1849 memcpy(req->iv + 12, &counter, 4); 1850 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1851 1852 counter = cpu_to_be32(1); 1853 memcpy(req->iv + 12, &counter, 4); 1854 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1855 1856 if (!req_ctx->plaintext_authenticate_only) { 1857 __be64 temp64; 1858 1859 temp64 = cpu_to_be64(req_ctx->assoclen * 8); 1860 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1861 temp64 = cpu_to_be64(cryptlen * 8); 1862 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1863 } else { 1864 /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1865 * data that is nothing is encrypted. 1866 */ 1867 __be64 temp64; 1868 1869 temp64 = cpu_to_be64((req_ctx->assoclen + cryptlen) * 8); 1870 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1871 temp64 = 0; 1872 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1873 } 1874 1875 return 0; 1876 } 1877 1878 static void cc_proc_rfc4_gcm(struct aead_request *req) 1879 { 1880 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1881 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1882 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1883 1884 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1885 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1886 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1887 GCM_BLOCK_RFC4_IV_SIZE); 1888 req->iv = areq_ctx->ctr_iv; 1889 } 1890 1891 static int cc_proc_aead(struct aead_request *req, 1892 enum drv_crypto_direction direct) 1893 { 1894 int rc = 0; 1895 int seq_len = 0; 1896 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1897 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1898 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1899 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1900 struct device *dev = drvdata_to_dev(ctx->drvdata); 1901 struct cc_crypto_req cc_req = {}; 1902 1903 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1904 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1905 ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1906 sg_virt(req->dst), req->dst->offset, req->cryptlen); 1907 1908 /* STAT_PHASE_0: Init and sanity checks */ 1909 1910 /* Check data length according to mode */ 1911 if (validate_data_size(ctx, direct, req)) { 1912 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1913 req->cryptlen, areq_ctx->assoclen); 1914 return -EINVAL; 1915 } 1916 1917 /* Setup request structure */ 1918 cc_req.user_cb = cc_aead_complete; 1919 cc_req.user_arg = req; 1920 1921 /* Setup request context */ 1922 areq_ctx->gen_ctx.op_type = direct; 1923 areq_ctx->req_authsize = ctx->authsize; 1924 areq_ctx->cipher_mode = ctx->cipher_mode; 1925 1926 /* STAT_PHASE_1: Map buffers */ 1927 1928 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1929 /* Build CTR IV - Copy nonce from last 4 bytes in 1930 * CTR key to first 4 bytes in CTR IV 1931 */ 1932 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1933 CTR_RFC3686_NONCE_SIZE); 1934 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1935 CTR_RFC3686_IV_SIZE); 1936 /* Initialize counter portion of counter block */ 1937 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1938 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1939 1940 /* Replace with counter iv */ 1941 req->iv = areq_ctx->ctr_iv; 1942 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1943 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1944 (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1945 areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1946 if (areq_ctx->ctr_iv != req->iv) { 1947 memcpy(areq_ctx->ctr_iv, req->iv, 1948 crypto_aead_ivsize(tfm)); 1949 req->iv = areq_ctx->ctr_iv; 1950 } 1951 } else { 1952 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1953 } 1954 1955 if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1956 rc = config_ccm_adata(req); 1957 if (rc) { 1958 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1959 rc); 1960 goto exit; 1961 } 1962 } else { 1963 areq_ctx->ccm_hdr_size = ccm_header_size_null; 1964 } 1965 1966 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1967 rc = config_gcm_context(req); 1968 if (rc) { 1969 dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 1970 rc); 1971 goto exit; 1972 } 1973 } 1974 1975 rc = cc_map_aead_request(ctx->drvdata, req); 1976 if (rc) { 1977 dev_err(dev, "map_request() failed\n"); 1978 goto exit; 1979 } 1980 1981 /* STAT_PHASE_2: Create sequence */ 1982 1983 /* Load MLLI tables to SRAM if necessary */ 1984 cc_mlli_to_sram(req, desc, &seq_len); 1985 1986 switch (ctx->auth_mode) { 1987 case DRV_HASH_SHA1: 1988 case DRV_HASH_SHA256: 1989 cc_hmac_authenc(req, desc, &seq_len); 1990 break; 1991 case DRV_HASH_XCBC_MAC: 1992 cc_xcbc_authenc(req, desc, &seq_len); 1993 break; 1994 case DRV_HASH_NULL: 1995 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1996 cc_ccm(req, desc, &seq_len); 1997 if (ctx->cipher_mode == DRV_CIPHER_GCTR) 1998 cc_gcm(req, desc, &seq_len); 1999 break; 2000 default: 2001 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2002 cc_unmap_aead_request(dev, req); 2003 rc = -ENOTSUPP; 2004 goto exit; 2005 } 2006 2007 /* STAT_PHASE_3: Lock HW and push sequence */ 2008 2009 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2010 2011 if (rc != -EINPROGRESS && rc != -EBUSY) { 2012 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2013 cc_unmap_aead_request(dev, req); 2014 } 2015 2016 exit: 2017 return rc; 2018 } 2019 2020 static int cc_aead_encrypt(struct aead_request *req) 2021 { 2022 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2023 int rc; 2024 2025 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2026 2027 /* No generated IV required */ 2028 areq_ctx->backup_iv = req->iv; 2029 areq_ctx->assoclen = req->assoclen; 2030 2031 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2032 if (rc != -EINPROGRESS && rc != -EBUSY) 2033 req->iv = areq_ctx->backup_iv; 2034 2035 return rc; 2036 } 2037 2038 static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2039 { 2040 /* Very similar to cc_aead_encrypt() above. */ 2041 2042 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2043 int rc; 2044 2045 rc = crypto_ipsec_check_assoclen(req->assoclen); 2046 if (rc) 2047 goto out; 2048 2049 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2050 2051 /* No generated IV required */ 2052 areq_ctx->backup_iv = req->iv; 2053 areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE; 2054 2055 cc_proc_rfc4309_ccm(req); 2056 2057 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2058 if (rc != -EINPROGRESS && rc != -EBUSY) 2059 req->iv = areq_ctx->backup_iv; 2060 out: 2061 return rc; 2062 } 2063 2064 static int cc_aead_decrypt(struct aead_request *req) 2065 { 2066 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2067 int rc; 2068 2069 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2070 2071 /* No generated IV required */ 2072 areq_ctx->backup_iv = req->iv; 2073 areq_ctx->assoclen = req->assoclen; 2074 2075 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2076 if (rc != -EINPROGRESS && rc != -EBUSY) 2077 req->iv = areq_ctx->backup_iv; 2078 2079 return rc; 2080 } 2081 2082 static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2083 { 2084 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2085 int rc; 2086 2087 rc = crypto_ipsec_check_assoclen(req->assoclen); 2088 if (rc) 2089 goto out; 2090 2091 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2092 2093 /* No generated IV required */ 2094 areq_ctx->backup_iv = req->iv; 2095 areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE; 2096 2097 cc_proc_rfc4309_ccm(req); 2098 2099 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2100 if (rc != -EINPROGRESS && rc != -EBUSY) 2101 req->iv = areq_ctx->backup_iv; 2102 2103 out: 2104 return rc; 2105 } 2106 2107 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2108 unsigned int keylen) 2109 { 2110 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2111 struct device *dev = drvdata_to_dev(ctx->drvdata); 2112 2113 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2114 2115 if (keylen < 4) 2116 return -EINVAL; 2117 2118 keylen -= 4; 2119 memcpy(ctx->ctr_nonce, key + keylen, 4); 2120 2121 return cc_aead_setkey(tfm, key, keylen); 2122 } 2123 2124 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2125 unsigned int keylen) 2126 { 2127 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2128 struct device *dev = drvdata_to_dev(ctx->drvdata); 2129 2130 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2131 2132 if (keylen < 4) 2133 return -EINVAL; 2134 2135 keylen -= 4; 2136 memcpy(ctx->ctr_nonce, key + keylen, 4); 2137 2138 return cc_aead_setkey(tfm, key, keylen); 2139 } 2140 2141 static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2142 unsigned int authsize) 2143 { 2144 switch (authsize) { 2145 case 4: 2146 case 8: 2147 case 12: 2148 case 13: 2149 case 14: 2150 case 15: 2151 case 16: 2152 break; 2153 default: 2154 return -EINVAL; 2155 } 2156 2157 return cc_aead_setauthsize(authenc, authsize); 2158 } 2159 2160 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2161 unsigned int authsize) 2162 { 2163 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2164 struct device *dev = drvdata_to_dev(ctx->drvdata); 2165 2166 dev_dbg(dev, "authsize %d\n", authsize); 2167 2168 switch (authsize) { 2169 case 8: 2170 case 12: 2171 case 16: 2172 break; 2173 default: 2174 return -EINVAL; 2175 } 2176 2177 return cc_aead_setauthsize(authenc, authsize); 2178 } 2179 2180 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2181 unsigned int authsize) 2182 { 2183 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2184 struct device *dev = drvdata_to_dev(ctx->drvdata); 2185 2186 dev_dbg(dev, "authsize %d\n", authsize); 2187 2188 if (authsize != 16) 2189 return -EINVAL; 2190 2191 return cc_aead_setauthsize(authenc, authsize); 2192 } 2193 2194 static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2195 { 2196 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2197 int rc; 2198 2199 rc = crypto_ipsec_check_assoclen(req->assoclen); 2200 if (rc) 2201 goto out; 2202 2203 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2204 2205 /* No generated IV required */ 2206 areq_ctx->backup_iv = req->iv; 2207 areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE; 2208 2209 cc_proc_rfc4_gcm(req); 2210 2211 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2212 if (rc != -EINPROGRESS && rc != -EBUSY) 2213 req->iv = areq_ctx->backup_iv; 2214 out: 2215 return rc; 2216 } 2217 2218 static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2219 { 2220 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2221 int rc; 2222 2223 rc = crypto_ipsec_check_assoclen(req->assoclen); 2224 if (rc) 2225 goto out; 2226 2227 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2228 2229 //plaintext is not encryped with rfc4543 2230 areq_ctx->plaintext_authenticate_only = true; 2231 2232 /* No generated IV required */ 2233 areq_ctx->backup_iv = req->iv; 2234 areq_ctx->assoclen = req->assoclen; 2235 2236 cc_proc_rfc4_gcm(req); 2237 2238 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2239 if (rc != -EINPROGRESS && rc != -EBUSY) 2240 req->iv = areq_ctx->backup_iv; 2241 out: 2242 return rc; 2243 } 2244 2245 static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2246 { 2247 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2248 int rc; 2249 2250 rc = crypto_ipsec_check_assoclen(req->assoclen); 2251 if (rc) 2252 goto out; 2253 2254 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2255 2256 /* No generated IV required */ 2257 areq_ctx->backup_iv = req->iv; 2258 areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE; 2259 2260 cc_proc_rfc4_gcm(req); 2261 2262 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2263 if (rc != -EINPROGRESS && rc != -EBUSY) 2264 req->iv = areq_ctx->backup_iv; 2265 out: 2266 return rc; 2267 } 2268 2269 static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2270 { 2271 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2272 int rc; 2273 2274 rc = crypto_ipsec_check_assoclen(req->assoclen); 2275 if (rc) 2276 goto out; 2277 2278 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2279 2280 //plaintext is not decryped with rfc4543 2281 areq_ctx->plaintext_authenticate_only = true; 2282 2283 /* No generated IV required */ 2284 areq_ctx->backup_iv = req->iv; 2285 areq_ctx->assoclen = req->assoclen; 2286 2287 cc_proc_rfc4_gcm(req); 2288 2289 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2290 if (rc != -EINPROGRESS && rc != -EBUSY) 2291 req->iv = areq_ctx->backup_iv; 2292 out: 2293 return rc; 2294 } 2295 2296 /* aead alg */ 2297 static struct cc_alg_template aead_algs[] = { 2298 { 2299 .name = "authenc(hmac(sha1),cbc(aes))", 2300 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2301 .blocksize = AES_BLOCK_SIZE, 2302 .template_aead = { 2303 .setkey = cc_aead_setkey, 2304 .setauthsize = cc_aead_setauthsize, 2305 .encrypt = cc_aead_encrypt, 2306 .decrypt = cc_aead_decrypt, 2307 .init = cc_aead_init, 2308 .exit = cc_aead_exit, 2309 .ivsize = AES_BLOCK_SIZE, 2310 .maxauthsize = SHA1_DIGEST_SIZE, 2311 }, 2312 .cipher_mode = DRV_CIPHER_CBC, 2313 .flow_mode = S_DIN_to_AES, 2314 .auth_mode = DRV_HASH_SHA1, 2315 .min_hw_rev = CC_HW_REV_630, 2316 .std_body = CC_STD_NIST, 2317 }, 2318 { 2319 .name = "authenc(hmac(sha1),cbc(des3_ede))", 2320 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2321 .blocksize = DES3_EDE_BLOCK_SIZE, 2322 .template_aead = { 2323 .setkey = cc_des3_aead_setkey, 2324 .setauthsize = cc_aead_setauthsize, 2325 .encrypt = cc_aead_encrypt, 2326 .decrypt = cc_aead_decrypt, 2327 .init = cc_aead_init, 2328 .exit = cc_aead_exit, 2329 .ivsize = DES3_EDE_BLOCK_SIZE, 2330 .maxauthsize = SHA1_DIGEST_SIZE, 2331 }, 2332 .cipher_mode = DRV_CIPHER_CBC, 2333 .flow_mode = S_DIN_to_DES, 2334 .auth_mode = DRV_HASH_SHA1, 2335 .min_hw_rev = CC_HW_REV_630, 2336 .std_body = CC_STD_NIST, 2337 }, 2338 { 2339 .name = "authenc(hmac(sha256),cbc(aes))", 2340 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2341 .blocksize = AES_BLOCK_SIZE, 2342 .template_aead = { 2343 .setkey = cc_aead_setkey, 2344 .setauthsize = cc_aead_setauthsize, 2345 .encrypt = cc_aead_encrypt, 2346 .decrypt = cc_aead_decrypt, 2347 .init = cc_aead_init, 2348 .exit = cc_aead_exit, 2349 .ivsize = AES_BLOCK_SIZE, 2350 .maxauthsize = SHA256_DIGEST_SIZE, 2351 }, 2352 .cipher_mode = DRV_CIPHER_CBC, 2353 .flow_mode = S_DIN_to_AES, 2354 .auth_mode = DRV_HASH_SHA256, 2355 .min_hw_rev = CC_HW_REV_630, 2356 .std_body = CC_STD_NIST, 2357 }, 2358 { 2359 .name = "authenc(hmac(sha256),cbc(des3_ede))", 2360 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2361 .blocksize = DES3_EDE_BLOCK_SIZE, 2362 .template_aead = { 2363 .setkey = cc_des3_aead_setkey, 2364 .setauthsize = cc_aead_setauthsize, 2365 .encrypt = cc_aead_encrypt, 2366 .decrypt = cc_aead_decrypt, 2367 .init = cc_aead_init, 2368 .exit = cc_aead_exit, 2369 .ivsize = DES3_EDE_BLOCK_SIZE, 2370 .maxauthsize = SHA256_DIGEST_SIZE, 2371 }, 2372 .cipher_mode = DRV_CIPHER_CBC, 2373 .flow_mode = S_DIN_to_DES, 2374 .auth_mode = DRV_HASH_SHA256, 2375 .min_hw_rev = CC_HW_REV_630, 2376 .std_body = CC_STD_NIST, 2377 }, 2378 { 2379 .name = "authenc(xcbc(aes),cbc(aes))", 2380 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2381 .blocksize = AES_BLOCK_SIZE, 2382 .template_aead = { 2383 .setkey = cc_aead_setkey, 2384 .setauthsize = cc_aead_setauthsize, 2385 .encrypt = cc_aead_encrypt, 2386 .decrypt = cc_aead_decrypt, 2387 .init = cc_aead_init, 2388 .exit = cc_aead_exit, 2389 .ivsize = AES_BLOCK_SIZE, 2390 .maxauthsize = AES_BLOCK_SIZE, 2391 }, 2392 .cipher_mode = DRV_CIPHER_CBC, 2393 .flow_mode = S_DIN_to_AES, 2394 .auth_mode = DRV_HASH_XCBC_MAC, 2395 .min_hw_rev = CC_HW_REV_630, 2396 .std_body = CC_STD_NIST, 2397 }, 2398 { 2399 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2400 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2401 .blocksize = 1, 2402 .template_aead = { 2403 .setkey = cc_aead_setkey, 2404 .setauthsize = cc_aead_setauthsize, 2405 .encrypt = cc_aead_encrypt, 2406 .decrypt = cc_aead_decrypt, 2407 .init = cc_aead_init, 2408 .exit = cc_aead_exit, 2409 .ivsize = CTR_RFC3686_IV_SIZE, 2410 .maxauthsize = SHA1_DIGEST_SIZE, 2411 }, 2412 .cipher_mode = DRV_CIPHER_CTR, 2413 .flow_mode = S_DIN_to_AES, 2414 .auth_mode = DRV_HASH_SHA1, 2415 .min_hw_rev = CC_HW_REV_630, 2416 .std_body = CC_STD_NIST, 2417 }, 2418 { 2419 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2420 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2421 .blocksize = 1, 2422 .template_aead = { 2423 .setkey = cc_aead_setkey, 2424 .setauthsize = cc_aead_setauthsize, 2425 .encrypt = cc_aead_encrypt, 2426 .decrypt = cc_aead_decrypt, 2427 .init = cc_aead_init, 2428 .exit = cc_aead_exit, 2429 .ivsize = CTR_RFC3686_IV_SIZE, 2430 .maxauthsize = SHA256_DIGEST_SIZE, 2431 }, 2432 .cipher_mode = DRV_CIPHER_CTR, 2433 .flow_mode = S_DIN_to_AES, 2434 .auth_mode = DRV_HASH_SHA256, 2435 .min_hw_rev = CC_HW_REV_630, 2436 .std_body = CC_STD_NIST, 2437 }, 2438 { 2439 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2440 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2441 .blocksize = 1, 2442 .template_aead = { 2443 .setkey = cc_aead_setkey, 2444 .setauthsize = cc_aead_setauthsize, 2445 .encrypt = cc_aead_encrypt, 2446 .decrypt = cc_aead_decrypt, 2447 .init = cc_aead_init, 2448 .exit = cc_aead_exit, 2449 .ivsize = CTR_RFC3686_IV_SIZE, 2450 .maxauthsize = AES_BLOCK_SIZE, 2451 }, 2452 .cipher_mode = DRV_CIPHER_CTR, 2453 .flow_mode = S_DIN_to_AES, 2454 .auth_mode = DRV_HASH_XCBC_MAC, 2455 .min_hw_rev = CC_HW_REV_630, 2456 .std_body = CC_STD_NIST, 2457 }, 2458 { 2459 .name = "ccm(aes)", 2460 .driver_name = "ccm-aes-ccree", 2461 .blocksize = 1, 2462 .template_aead = { 2463 .setkey = cc_aead_setkey, 2464 .setauthsize = cc_ccm_setauthsize, 2465 .encrypt = cc_aead_encrypt, 2466 .decrypt = cc_aead_decrypt, 2467 .init = cc_aead_init, 2468 .exit = cc_aead_exit, 2469 .ivsize = AES_BLOCK_SIZE, 2470 .maxauthsize = AES_BLOCK_SIZE, 2471 }, 2472 .cipher_mode = DRV_CIPHER_CCM, 2473 .flow_mode = S_DIN_to_AES, 2474 .auth_mode = DRV_HASH_NULL, 2475 .min_hw_rev = CC_HW_REV_630, 2476 .std_body = CC_STD_NIST, 2477 }, 2478 { 2479 .name = "rfc4309(ccm(aes))", 2480 .driver_name = "rfc4309-ccm-aes-ccree", 2481 .blocksize = 1, 2482 .template_aead = { 2483 .setkey = cc_rfc4309_ccm_setkey, 2484 .setauthsize = cc_rfc4309_ccm_setauthsize, 2485 .encrypt = cc_rfc4309_ccm_encrypt, 2486 .decrypt = cc_rfc4309_ccm_decrypt, 2487 .init = cc_aead_init, 2488 .exit = cc_aead_exit, 2489 .ivsize = CCM_BLOCK_IV_SIZE, 2490 .maxauthsize = AES_BLOCK_SIZE, 2491 }, 2492 .cipher_mode = DRV_CIPHER_CCM, 2493 .flow_mode = S_DIN_to_AES, 2494 .auth_mode = DRV_HASH_NULL, 2495 .min_hw_rev = CC_HW_REV_630, 2496 .std_body = CC_STD_NIST, 2497 }, 2498 { 2499 .name = "gcm(aes)", 2500 .driver_name = "gcm-aes-ccree", 2501 .blocksize = 1, 2502 .template_aead = { 2503 .setkey = cc_aead_setkey, 2504 .setauthsize = cc_gcm_setauthsize, 2505 .encrypt = cc_aead_encrypt, 2506 .decrypt = cc_aead_decrypt, 2507 .init = cc_aead_init, 2508 .exit = cc_aead_exit, 2509 .ivsize = 12, 2510 .maxauthsize = AES_BLOCK_SIZE, 2511 }, 2512 .cipher_mode = DRV_CIPHER_GCTR, 2513 .flow_mode = S_DIN_to_AES, 2514 .auth_mode = DRV_HASH_NULL, 2515 .min_hw_rev = CC_HW_REV_630, 2516 .std_body = CC_STD_NIST, 2517 }, 2518 { 2519 .name = "rfc4106(gcm(aes))", 2520 .driver_name = "rfc4106-gcm-aes-ccree", 2521 .blocksize = 1, 2522 .template_aead = { 2523 .setkey = cc_rfc4106_gcm_setkey, 2524 .setauthsize = cc_rfc4106_gcm_setauthsize, 2525 .encrypt = cc_rfc4106_gcm_encrypt, 2526 .decrypt = cc_rfc4106_gcm_decrypt, 2527 .init = cc_aead_init, 2528 .exit = cc_aead_exit, 2529 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2530 .maxauthsize = AES_BLOCK_SIZE, 2531 }, 2532 .cipher_mode = DRV_CIPHER_GCTR, 2533 .flow_mode = S_DIN_to_AES, 2534 .auth_mode = DRV_HASH_NULL, 2535 .min_hw_rev = CC_HW_REV_630, 2536 .std_body = CC_STD_NIST, 2537 }, 2538 { 2539 .name = "rfc4543(gcm(aes))", 2540 .driver_name = "rfc4543-gcm-aes-ccree", 2541 .blocksize = 1, 2542 .template_aead = { 2543 .setkey = cc_rfc4543_gcm_setkey, 2544 .setauthsize = cc_rfc4543_gcm_setauthsize, 2545 .encrypt = cc_rfc4543_gcm_encrypt, 2546 .decrypt = cc_rfc4543_gcm_decrypt, 2547 .init = cc_aead_init, 2548 .exit = cc_aead_exit, 2549 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2550 .maxauthsize = AES_BLOCK_SIZE, 2551 }, 2552 .cipher_mode = DRV_CIPHER_GCTR, 2553 .flow_mode = S_DIN_to_AES, 2554 .auth_mode = DRV_HASH_NULL, 2555 .min_hw_rev = CC_HW_REV_630, 2556 .std_body = CC_STD_NIST, 2557 }, 2558 }; 2559 2560 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2561 struct device *dev) 2562 { 2563 struct cc_crypto_alg *t_alg; 2564 struct aead_alg *alg; 2565 2566 t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); 2567 if (!t_alg) 2568 return ERR_PTR(-ENOMEM); 2569 2570 alg = &tmpl->template_aead; 2571 2572 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 2573 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2574 tmpl->driver_name); 2575 alg->base.cra_module = THIS_MODULE; 2576 alg->base.cra_priority = CC_CRA_PRIO; 2577 2578 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 2579 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2580 alg->base.cra_blocksize = tmpl->blocksize; 2581 alg->init = cc_aead_init; 2582 alg->exit = cc_aead_exit; 2583 2584 t_alg->aead_alg = *alg; 2585 2586 t_alg->cipher_mode = tmpl->cipher_mode; 2587 t_alg->flow_mode = tmpl->flow_mode; 2588 t_alg->auth_mode = tmpl->auth_mode; 2589 2590 return t_alg; 2591 } 2592 2593 int cc_aead_free(struct cc_drvdata *drvdata) 2594 { 2595 struct cc_crypto_alg *t_alg, *n; 2596 struct cc_aead_handle *aead_handle = drvdata->aead_handle; 2597 2598 /* Remove registered algs */ 2599 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, entry) { 2600 crypto_unregister_aead(&t_alg->aead_alg); 2601 list_del(&t_alg->entry); 2602 } 2603 2604 return 0; 2605 } 2606 2607 int cc_aead_alloc(struct cc_drvdata *drvdata) 2608 { 2609 struct cc_aead_handle *aead_handle; 2610 struct cc_crypto_alg *t_alg; 2611 int rc = -ENOMEM; 2612 int alg; 2613 struct device *dev = drvdata_to_dev(drvdata); 2614 2615 aead_handle = devm_kmalloc(dev, sizeof(*aead_handle), GFP_KERNEL); 2616 if (!aead_handle) { 2617 rc = -ENOMEM; 2618 goto fail0; 2619 } 2620 2621 INIT_LIST_HEAD(&aead_handle->aead_list); 2622 drvdata->aead_handle = aead_handle; 2623 2624 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2625 MAX_HMAC_DIGEST_SIZE); 2626 2627 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2628 rc = -ENOMEM; 2629 goto fail1; 2630 } 2631 2632 /* Linux crypto */ 2633 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 2634 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 2635 !(drvdata->std_bodies & aead_algs[alg].std_body)) 2636 continue; 2637 2638 t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2639 if (IS_ERR(t_alg)) { 2640 rc = PTR_ERR(t_alg); 2641 dev_err(dev, "%s alg allocation failed\n", 2642 aead_algs[alg].driver_name); 2643 goto fail1; 2644 } 2645 t_alg->drvdata = drvdata; 2646 rc = crypto_register_aead(&t_alg->aead_alg); 2647 if (rc) { 2648 dev_err(dev, "%s alg registration failed\n", 2649 t_alg->aead_alg.base.cra_driver_name); 2650 goto fail1; 2651 } 2652 2653 list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2654 dev_dbg(dev, "Registered %s\n", 2655 t_alg->aead_alg.base.cra_driver_name); 2656 } 2657 2658 return 0; 2659 2660 fail1: 2661 cc_aead_free(drvdata); 2662 fail0: 2663 return rc; 2664 } 2665