1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel Keem Bay OCS AES Crypto Driver. 4 * 5 * Copyright (C) 2018-2020 Intel Corporation 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/completion.h> 10 #include <linux/crypto.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/types.h> 18 19 #include <crypto/aes.h> 20 #include <crypto/engine.h> 21 #include <crypto/gcm.h> 22 #include <crypto/scatterwalk.h> 23 24 #include <crypto/internal/aead.h> 25 #include <crypto/internal/skcipher.h> 26 27 #include "ocs-aes.h" 28 29 #define KMB_OCS_PRIORITY 350 30 #define DRV_NAME "keembay-ocs-aes" 31 32 #define OCS_AES_MIN_KEY_SIZE 16 33 #define OCS_AES_MAX_KEY_SIZE 32 34 #define OCS_AES_KEYSIZE_128 16 35 #define OCS_AES_KEYSIZE_192 24 36 #define OCS_AES_KEYSIZE_256 32 37 #define OCS_SM4_KEY_SIZE 16 38 39 /** 40 * struct ocs_aes_tctx - OCS AES Transform context 41 * @engine_ctx: Engine context. 42 * @aes_dev: The OCS AES device. 43 * @key: AES/SM4 key. 44 * @key_len: The length (in bytes) of @key. 45 * @cipher: OCS cipher to use (either AES or SM4). 46 * @sw_cipher: The cipher to use as fallback. 47 * @use_fallback: Whether or not fallback cipher should be used. 48 */ 49 struct ocs_aes_tctx { 50 struct crypto_engine_ctx engine_ctx; 51 struct ocs_aes_dev *aes_dev; 52 u8 key[OCS_AES_KEYSIZE_256]; 53 unsigned int key_len; 54 enum ocs_cipher cipher; 55 union { 56 struct crypto_sync_skcipher *sk; 57 struct crypto_aead *aead; 58 } sw_cipher; 59 bool use_fallback; 60 }; 61 62 /** 63 * struct ocs_aes_rctx - OCS AES Request context. 64 * @instruction: Instruction to be executed (encrypt / decrypt). 65 * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) 66 * @src_nents: Number of source SG entries. 67 * @dst_nents: Number of destination SG entries. 68 * @src_dma_count: The number of DMA-mapped entries of the source SG. 69 * @dst_dma_count: The number of DMA-mapped entries of the destination SG. 70 * @in_place: Whether or not this is an in place request, i.e., 71 * src_sg == dst_sg. 72 * @src_dll: OCS DMA linked list for input data. 73 * @dst_dll: OCS DMA linked list for output data. 74 * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC 75 * mode). 76 * @cts_swap: Whether or not CTS swap must be performed. 77 * @aad_src_dll: OCS DMA linked list for input AAD data. 78 * @aad_dst_dll: OCS DMA linked list for output AAD data. 79 * @in_tag: Buffer to hold input encrypted tag (only used for 80 * CCM/GCM decrypt). 81 * @out_tag: Buffer to hold output encrypted / decrypted tag (only 82 * used for GCM encrypt / decrypt). 83 */ 84 struct ocs_aes_rctx { 85 /* Fields common across all modes. */ 86 enum ocs_instruction instruction; 87 enum ocs_mode mode; 88 int src_nents; 89 int dst_nents; 90 int src_dma_count; 91 int dst_dma_count; 92 bool in_place; 93 struct ocs_dll_desc src_dll; 94 struct ocs_dll_desc dst_dll; 95 96 /* CBC specific */ 97 u8 last_ct_blk[AES_BLOCK_SIZE]; 98 99 /* CTS specific */ 100 int cts_swap; 101 102 /* CCM/GCM specific */ 103 struct ocs_dll_desc aad_src_dll; 104 struct ocs_dll_desc aad_dst_dll; 105 u8 in_tag[AES_BLOCK_SIZE]; 106 107 /* GCM specific */ 108 u8 out_tag[AES_BLOCK_SIZE]; 109 }; 110 111 /* Driver data. */ 112 struct ocs_aes_drv { 113 struct list_head dev_list; 114 spinlock_t lock; /* Protects dev_list. */ 115 }; 116 117 static struct ocs_aes_drv ocs_aes = { 118 .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), 119 .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), 120 }; 121 122 static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) 123 { 124 struct ocs_aes_dev *aes_dev; 125 126 spin_lock(&ocs_aes.lock); 127 128 if (tctx->aes_dev) { 129 aes_dev = tctx->aes_dev; 130 goto exit; 131 } 132 133 /* Only a single OCS device available */ 134 aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); 135 tctx->aes_dev = aes_dev; 136 137 exit: 138 spin_unlock(&ocs_aes.lock); 139 140 return aes_dev; 141 } 142 143 /* 144 * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual 145 * key is being passed in. 146 * 147 * Return: 0 if key is valid, -EINVAL otherwise. 148 */ 149 static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) 150 { 151 if (!in_key) 152 return -EINVAL; 153 154 /* For AES, only 128-byte or 256-byte keys are supported. */ 155 if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || 156 key_len == OCS_AES_KEYSIZE_256)) 157 return 0; 158 159 /* For SM4, only 128-byte keys are supported. */ 160 if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) 161 return 0; 162 163 /* Everything else is unsupported. */ 164 return -EINVAL; 165 } 166 167 /* Save key into transformation context. */ 168 static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, 169 enum ocs_cipher cipher) 170 { 171 int ret; 172 173 ret = check_key(in_key, key_len, cipher); 174 if (ret) 175 return ret; 176 177 memcpy(tctx->key, in_key, key_len); 178 tctx->key_len = key_len; 179 tctx->cipher = cipher; 180 181 return 0; 182 } 183 184 /* Set key for symmetric cypher. */ 185 static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 186 size_t key_len, enum ocs_cipher cipher) 187 { 188 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 189 190 /* Fallback is used for AES with 192-bit key. */ 191 tctx->use_fallback = (cipher == OCS_AES && 192 key_len == OCS_AES_KEYSIZE_192); 193 194 if (!tctx->use_fallback) 195 return save_key(tctx, in_key, key_len, cipher); 196 197 crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, 198 CRYPTO_TFM_REQ_MASK); 199 crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, 200 tfm->base.crt_flags & 201 CRYPTO_TFM_REQ_MASK); 202 203 return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); 204 } 205 206 /* Set key for AEAD cipher. */ 207 static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, 208 size_t key_len, enum ocs_cipher cipher) 209 { 210 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 211 212 /* Fallback is used for AES with 192-bit key. */ 213 tctx->use_fallback = (cipher == OCS_AES && 214 key_len == OCS_AES_KEYSIZE_192); 215 216 if (!tctx->use_fallback) 217 return save_key(tctx, in_key, key_len, cipher); 218 219 crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); 220 crypto_aead_set_flags(tctx->sw_cipher.aead, 221 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 222 223 return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); 224 } 225 226 /* Swap two AES blocks in SG lists. */ 227 static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, 228 off_t blk1_offset, off_t blk2_offset) 229 { 230 u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; 231 232 /* 233 * No easy way to copy within sg list, so copy both blocks to temporary 234 * buffers first. 235 */ 236 sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); 237 sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); 238 sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); 239 sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); 240 } 241 242 /* Initialize request context to default values. */ 243 static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) 244 { 245 /* Zero everything. */ 246 memset(rctx, 0, sizeof(*rctx)); 247 248 /* Set initial value for DMA addresses. */ 249 rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; 250 rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; 251 rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; 252 rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; 253 } 254 255 static int kmb_ocs_sk_validate_input(struct skcipher_request *req, 256 enum ocs_mode mode) 257 { 258 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 259 int iv_size = crypto_skcipher_ivsize(tfm); 260 261 switch (mode) { 262 case OCS_MODE_ECB: 263 /* Ensure input length is multiple of block size */ 264 if (req->cryptlen % AES_BLOCK_SIZE != 0) 265 return -EINVAL; 266 267 return 0; 268 269 case OCS_MODE_CBC: 270 /* Ensure input length is multiple of block size */ 271 if (req->cryptlen % AES_BLOCK_SIZE != 0) 272 return -EINVAL; 273 274 /* Ensure IV is present and block size in length */ 275 if (!req->iv || iv_size != AES_BLOCK_SIZE) 276 return -EINVAL; 277 /* 278 * NOTE: Since req->cryptlen == 0 case was already handled in 279 * kmb_ocs_sk_common(), the above two conditions also guarantee 280 * that: cryptlen >= iv_size 281 */ 282 return 0; 283 284 case OCS_MODE_CTR: 285 /* Ensure IV is present and block size in length */ 286 if (!req->iv || iv_size != AES_BLOCK_SIZE) 287 return -EINVAL; 288 return 0; 289 290 case OCS_MODE_CTS: 291 /* Ensure input length >= block size */ 292 if (req->cryptlen < AES_BLOCK_SIZE) 293 return -EINVAL; 294 295 /* Ensure IV is present and block size in length */ 296 if (!req->iv || iv_size != AES_BLOCK_SIZE) 297 return -EINVAL; 298 299 return 0; 300 default: 301 return -EINVAL; 302 } 303 } 304 305 /* 306 * Called by encrypt() / decrypt() skcipher functions. 307 * 308 * Use fallback if needed, otherwise initialize context and enqueue request 309 * into engine. 310 */ 311 static int kmb_ocs_sk_common(struct skcipher_request *req, 312 enum ocs_cipher cipher, 313 enum ocs_instruction instruction, 314 enum ocs_mode mode) 315 { 316 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 317 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 318 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 319 struct ocs_aes_dev *aes_dev; 320 int rc; 321 322 if (tctx->use_fallback) { 323 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); 324 325 skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); 326 skcipher_request_set_callback(subreq, req->base.flags, NULL, 327 NULL); 328 skcipher_request_set_crypt(subreq, req->src, req->dst, 329 req->cryptlen, req->iv); 330 331 if (instruction == OCS_ENCRYPT) 332 rc = crypto_skcipher_encrypt(subreq); 333 else 334 rc = crypto_skcipher_decrypt(subreq); 335 336 skcipher_request_zero(subreq); 337 338 return rc; 339 } 340 341 /* 342 * If cryptlen == 0, no processing needed for ECB, CBC and CTR. 343 * 344 * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. 345 */ 346 if (!req->cryptlen && mode != OCS_MODE_CTS) 347 return 0; 348 349 rc = kmb_ocs_sk_validate_input(req, mode); 350 if (rc) 351 return rc; 352 353 aes_dev = kmb_ocs_aes_find_dev(tctx); 354 if (!aes_dev) 355 return -ENODEV; 356 357 if (cipher != tctx->cipher) 358 return -EINVAL; 359 360 ocs_aes_init_rctx(rctx); 361 rctx->instruction = instruction; 362 rctx->mode = mode; 363 364 return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); 365 } 366 367 static void cleanup_ocs_dma_linked_list(struct device *dev, 368 struct ocs_dll_desc *dll) 369 { 370 if (dll->vaddr) 371 dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); 372 dll->vaddr = NULL; 373 dll->size = 0; 374 dll->dma_addr = DMA_MAPPING_ERROR; 375 } 376 377 static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) 378 { 379 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 380 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 381 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 382 struct device *dev = tctx->aes_dev->dev; 383 384 if (rctx->src_dma_count) { 385 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 386 rctx->src_dma_count = 0; 387 } 388 389 if (rctx->dst_dma_count) { 390 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? 391 DMA_BIDIRECTIONAL : 392 DMA_FROM_DEVICE); 393 rctx->dst_dma_count = 0; 394 } 395 396 /* Clean up OCS DMA linked lists */ 397 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); 398 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); 399 } 400 401 static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) 402 { 403 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 404 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 405 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 406 int iv_size = crypto_skcipher_ivsize(tfm); 407 int rc; 408 409 /* 410 * For CBC decrypt, save last block (iv) to last_ct_blk buffer. 411 * 412 * Note: if we are here, we already checked that cryptlen >= iv_size 413 * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see 414 * kmb_ocs_sk_validate_input(). 415 */ 416 if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) 417 scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, 418 req->cryptlen - iv_size, iv_size, 0); 419 420 /* For CTS decrypt, swap last two blocks, if needed. */ 421 if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) 422 sg_swap_blocks(req->dst, rctx->dst_nents, 423 req->cryptlen - AES_BLOCK_SIZE, 424 req->cryptlen - (2 * AES_BLOCK_SIZE)); 425 426 /* src and dst buffers are the same, use bidirectional DMA mapping. */ 427 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, 428 rctx->dst_nents, DMA_BIDIRECTIONAL); 429 if (rctx->dst_dma_count == 0) { 430 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); 431 return -ENOMEM; 432 } 433 434 /* Create DST linked list */ 435 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 436 rctx->dst_dma_count, &rctx->dst_dll, 437 req->cryptlen, 0); 438 if (rc) 439 return rc; 440 /* 441 * If descriptor creation was successful, set the src_dll.dma_addr to 442 * the value of dst_dll.dma_addr, as we do in-place AES operation on 443 * the src. 444 */ 445 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; 446 447 return 0; 448 } 449 450 static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) 451 { 452 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 453 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 454 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 455 int rc; 456 457 rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); 458 if (rctx->src_nents < 0) 459 return -EBADMSG; 460 461 /* Map SRC SG. */ 462 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, 463 rctx->src_nents, DMA_TO_DEVICE); 464 if (rctx->src_dma_count == 0) { 465 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); 466 return -ENOMEM; 467 } 468 469 /* Create SRC linked list */ 470 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, 471 rctx->src_dma_count, &rctx->src_dll, 472 req->cryptlen, 0); 473 if (rc) 474 return rc; 475 476 /* Map DST SG. */ 477 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, 478 rctx->dst_nents, DMA_FROM_DEVICE); 479 if (rctx->dst_dma_count == 0) { 480 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); 481 return -ENOMEM; 482 } 483 484 /* Create DST linked list */ 485 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 486 rctx->dst_dma_count, &rctx->dst_dll, 487 req->cryptlen, 0); 488 if (rc) 489 return rc; 490 491 /* If this is not a CTS decrypt operation with swapping, we are done. */ 492 if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) 493 return 0; 494 495 /* 496 * Otherwise, we have to copy src to dst (as we cannot modify src). 497 * Use OCS AES bypass mode to copy src to dst via DMA. 498 * 499 * NOTE: for anything other than small data sizes this is rather 500 * inefficient. 501 */ 502 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, 503 rctx->src_dll.dma_addr, req->cryptlen); 504 if (rc) 505 return rc; 506 507 /* 508 * Now dst == src, so clean up what we did so far and use in_place 509 * logic. 510 */ 511 kmb_ocs_sk_dma_cleanup(req); 512 rctx->in_place = true; 513 514 return kmb_ocs_sk_prepare_inplace(req); 515 } 516 517 static int kmb_ocs_sk_run(struct skcipher_request *req) 518 { 519 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 520 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 521 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 522 struct ocs_aes_dev *aes_dev = tctx->aes_dev; 523 int iv_size = crypto_skcipher_ivsize(tfm); 524 int rc; 525 526 rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 527 if (rctx->dst_nents < 0) 528 return -EBADMSG; 529 530 /* 531 * If 2 blocks or greater, and multiple of block size swap last two 532 * blocks to be compatible with other crypto API CTS implementations: 533 * OCS mode uses CBC-CS2, whereas other crypto API implementations use 534 * CBC-CS3. 535 * CBC-CS2 and CBC-CS3 defined by: 536 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf 537 */ 538 rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && 539 req->cryptlen > AES_BLOCK_SIZE && 540 req->cryptlen % AES_BLOCK_SIZE == 0); 541 542 rctx->in_place = (req->src == req->dst); 543 544 if (rctx->in_place) 545 rc = kmb_ocs_sk_prepare_inplace(req); 546 else 547 rc = kmb_ocs_sk_prepare_notinplace(req); 548 549 if (rc) 550 goto error; 551 552 rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, 553 rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, 554 req->cryptlen, req->iv, iv_size); 555 if (rc) 556 goto error; 557 558 /* Clean-up DMA before further processing output. */ 559 kmb_ocs_sk_dma_cleanup(req); 560 561 /* For CTS Encrypt, swap last 2 blocks, if needed. */ 562 if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { 563 sg_swap_blocks(req->dst, rctx->dst_nents, 564 req->cryptlen - AES_BLOCK_SIZE, 565 req->cryptlen - (2 * AES_BLOCK_SIZE)); 566 return 0; 567 } 568 569 /* For CBC copy IV to req->IV. */ 570 if (rctx->mode == OCS_MODE_CBC) { 571 /* CBC encrypt case. */ 572 if (rctx->instruction == OCS_ENCRYPT) { 573 scatterwalk_map_and_copy(req->iv, req->dst, 574 req->cryptlen - iv_size, 575 iv_size, 0); 576 return 0; 577 } 578 /* CBC decrypt case. */ 579 if (rctx->in_place) 580 memcpy(req->iv, rctx->last_ct_blk, iv_size); 581 else 582 scatterwalk_map_and_copy(req->iv, req->src, 583 req->cryptlen - iv_size, 584 iv_size, 0); 585 return 0; 586 } 587 /* For all other modes there's nothing to do. */ 588 589 return 0; 590 591 error: 592 kmb_ocs_sk_dma_cleanup(req); 593 594 return rc; 595 } 596 597 static int kmb_ocs_aead_validate_input(struct aead_request *req, 598 enum ocs_instruction instruction, 599 enum ocs_mode mode) 600 { 601 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 602 int tag_size = crypto_aead_authsize(tfm); 603 int iv_size = crypto_aead_ivsize(tfm); 604 605 /* For decrypt crytplen == len(PT) + len(tag). */ 606 if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) 607 return -EINVAL; 608 609 /* IV is mandatory. */ 610 if (!req->iv) 611 return -EINVAL; 612 613 switch (mode) { 614 case OCS_MODE_GCM: 615 if (iv_size != GCM_AES_IV_SIZE) 616 return -EINVAL; 617 618 return 0; 619 620 case OCS_MODE_CCM: 621 /* Ensure IV is present and block size in length */ 622 if (iv_size != AES_BLOCK_SIZE) 623 return -EINVAL; 624 625 return 0; 626 627 default: 628 return -EINVAL; 629 } 630 } 631 632 /* 633 * Called by encrypt() / decrypt() aead functions. 634 * 635 * Use fallback if needed, otherwise initialize context and enqueue request 636 * into engine. 637 */ 638 static int kmb_ocs_aead_common(struct aead_request *req, 639 enum ocs_cipher cipher, 640 enum ocs_instruction instruction, 641 enum ocs_mode mode) 642 { 643 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 644 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 645 struct ocs_aes_dev *dd; 646 int rc; 647 648 if (tctx->use_fallback) { 649 struct aead_request *subreq = aead_request_ctx(req); 650 651 aead_request_set_tfm(subreq, tctx->sw_cipher.aead); 652 aead_request_set_callback(subreq, req->base.flags, 653 req->base.complete, req->base.data); 654 aead_request_set_crypt(subreq, req->src, req->dst, 655 req->cryptlen, req->iv); 656 aead_request_set_ad(subreq, req->assoclen); 657 rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, 658 crypto_aead_authsize(crypto_aead_reqtfm(req))); 659 if (rc) 660 return rc; 661 662 return (instruction == OCS_ENCRYPT) ? 663 crypto_aead_encrypt(subreq) : 664 crypto_aead_decrypt(subreq); 665 } 666 667 rc = kmb_ocs_aead_validate_input(req, instruction, mode); 668 if (rc) 669 return rc; 670 671 dd = kmb_ocs_aes_find_dev(tctx); 672 if (!dd) 673 return -ENODEV; 674 675 if (cipher != tctx->cipher) 676 return -EINVAL; 677 678 ocs_aes_init_rctx(rctx); 679 rctx->instruction = instruction; 680 rctx->mode = mode; 681 682 return crypto_transfer_aead_request_to_engine(dd->engine, req); 683 } 684 685 static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) 686 { 687 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 688 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 689 struct device *dev = tctx->aes_dev->dev; 690 691 if (rctx->src_dma_count) { 692 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 693 rctx->src_dma_count = 0; 694 } 695 696 if (rctx->dst_dma_count) { 697 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? 698 DMA_BIDIRECTIONAL : 699 DMA_FROM_DEVICE); 700 rctx->dst_dma_count = 0; 701 } 702 /* Clean up OCS DMA linked lists */ 703 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); 704 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); 705 cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); 706 cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); 707 } 708 709 /** 710 * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. 711 * @req: The AEAD request being processed. 712 * @src_dll_size: Where to store the length of the data mapped into the 713 * src_dll OCS DMA list. 714 * 715 * Do the following: 716 * - DMA map req->src and req->dst 717 * - Initialize the following OCS DMA linked lists: rctx->src_dll, 718 * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. 719 * 720 * Return: 0 on success, negative error code otherwise. 721 */ 722 static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) 723 { 724 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 725 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); 726 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 727 u32 in_size; /* The length of the data to be mapped by src_dll. */ 728 u32 out_size; /* The length of the data to be mapped by dst_dll. */ 729 u32 dst_size; /* The length of the data in dst_sg. */ 730 int rc; 731 732 /* Get number of entries in input data SG list. */ 733 rctx->src_nents = sg_nents_for_len(req->src, 734 req->assoclen + req->cryptlen); 735 if (rctx->src_nents < 0) 736 return -EBADMSG; 737 738 if (rctx->instruction == OCS_DECRYPT) { 739 /* 740 * For decrypt: 741 * - src sg list is: AAD|CT|tag 742 * - dst sg list expects: AAD|PT 743 * 744 * in_size == len(CT); out_size == len(PT) 745 */ 746 747 /* req->cryptlen includes both CT and tag. */ 748 in_size = req->cryptlen - tag_size; 749 750 /* out_size = PT size == CT size */ 751 out_size = in_size; 752 753 /* len(dst_sg) == len(AAD) + len(PT) */ 754 dst_size = req->assoclen + out_size; 755 756 /* 757 * Copy tag from source SG list to 'in_tag' buffer. 758 * 759 * Note: this needs to be done here, before DMA mapping src_sg. 760 */ 761 sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, 762 tag_size, req->assoclen + in_size); 763 764 } else { /* OCS_ENCRYPT */ 765 /* 766 * For encrypt: 767 * src sg list is: AAD|PT 768 * dst sg list expects: AAD|CT|tag 769 */ 770 /* in_size == len(PT) */ 771 in_size = req->cryptlen; 772 773 /* 774 * In CCM mode the OCS engine appends the tag to the ciphertext, 775 * but in GCM mode the tag must be read from the tag registers 776 * and appended manually below 777 */ 778 out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : 779 in_size; 780 /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ 781 dst_size = req->assoclen + in_size + tag_size; 782 } 783 *src_dll_size = in_size; 784 785 /* Get number of entries in output data SG list. */ 786 rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); 787 if (rctx->dst_nents < 0) 788 return -EBADMSG; 789 790 rctx->in_place = (req->src == req->dst) ? 1 : 0; 791 792 /* Map destination; use bidirectional mapping for in-place case. */ 793 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, 794 rctx->dst_nents, 795 rctx->in_place ? DMA_BIDIRECTIONAL : 796 DMA_FROM_DEVICE); 797 if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { 798 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); 799 return -ENOMEM; 800 } 801 802 /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ 803 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 804 rctx->dst_dma_count, 805 &rctx->aad_dst_dll, req->assoclen, 806 0); 807 if (rc) 808 return rc; 809 810 /* Create DST list: maps dst[AAD_SIZE:out_size] */ 811 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 812 rctx->dst_dma_count, &rctx->dst_dll, 813 out_size, req->assoclen); 814 if (rc) 815 return rc; 816 817 if (rctx->in_place) { 818 /* If this is not CCM encrypt, we are done. */ 819 if (!(rctx->mode == OCS_MODE_CCM && 820 rctx->instruction == OCS_ENCRYPT)) { 821 /* 822 * SRC and DST are the same, so re-use the same DMA 823 * addresses (to avoid allocating new DMA lists 824 * identical to the dst ones). 825 */ 826 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; 827 rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; 828 829 return 0; 830 } 831 /* 832 * For CCM encrypt the input and output linked lists contain 833 * different amounts of data, so, we need to create different 834 * SRC and AAD SRC lists, even for the in-place case. 835 */ 836 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 837 rctx->dst_dma_count, 838 &rctx->aad_src_dll, 839 req->assoclen, 0); 840 if (rc) 841 return rc; 842 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 843 rctx->dst_dma_count, 844 &rctx->src_dll, in_size, 845 req->assoclen); 846 if (rc) 847 return rc; 848 849 return 0; 850 } 851 /* Not in-place case. */ 852 853 /* Map source SG. */ 854 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, 855 rctx->src_nents, DMA_TO_DEVICE); 856 if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { 857 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); 858 return -ENOMEM; 859 } 860 861 /* Create AAD SRC list. */ 862 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, 863 rctx->src_dma_count, 864 &rctx->aad_src_dll, 865 req->assoclen, 0); 866 if (rc) 867 return rc; 868 869 /* Create SRC list. */ 870 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, 871 rctx->src_dma_count, 872 &rctx->src_dll, in_size, 873 req->assoclen); 874 if (rc) 875 return rc; 876 877 if (req->assoclen == 0) 878 return 0; 879 880 /* Copy AAD from src sg to dst sg using OCS DMA. */ 881 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, 882 rctx->aad_src_dll.dma_addr, req->cryptlen); 883 if (rc) 884 dev_err(tctx->aes_dev->dev, 885 "Failed to copy source AAD to destination AAD\n"); 886 887 return rc; 888 } 889 890 static int kmb_ocs_aead_run(struct aead_request *req) 891 { 892 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 893 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); 894 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 895 u32 in_size; /* The length of the data mapped by src_dll. */ 896 int rc; 897 898 rc = kmb_ocs_aead_dma_prepare(req, &in_size); 899 if (rc) 900 goto exit; 901 902 /* For CCM, we just call the OCS processing and we are done. */ 903 if (rctx->mode == OCS_MODE_CCM) { 904 rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, 905 rctx->instruction, rctx->dst_dll.dma_addr, 906 rctx->src_dll.dma_addr, in_size, 907 req->iv, 908 rctx->aad_src_dll.dma_addr, req->assoclen, 909 rctx->in_tag, tag_size); 910 goto exit; 911 } 912 /* GCM case; invoke OCS processing. */ 913 rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, 914 rctx->instruction, 915 rctx->dst_dll.dma_addr, 916 rctx->src_dll.dma_addr, in_size, 917 req->iv, 918 rctx->aad_src_dll.dma_addr, req->assoclen, 919 rctx->out_tag, tag_size); 920 if (rc) 921 goto exit; 922 923 /* For GCM decrypt, we have to compare in_tag with out_tag. */ 924 if (rctx->instruction == OCS_DECRYPT) { 925 rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? 926 -EBADMSG : 0; 927 goto exit; 928 } 929 930 /* For GCM encrypt, we must manually copy out_tag to DST sg. */ 931 932 /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ 933 kmb_ocs_aead_dma_cleanup(req); 934 935 /* Copy tag to destination sg after AAD and CT. */ 936 sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, 937 tag_size, req->assoclen + req->cryptlen); 938 939 /* Return directly as DMA cleanup already done. */ 940 return 0; 941 942 exit: 943 kmb_ocs_aead_dma_cleanup(req); 944 945 return rc; 946 } 947 948 static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, 949 void *areq) 950 { 951 struct skcipher_request *req = 952 container_of(areq, struct skcipher_request, base); 953 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 954 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 955 int err; 956 957 if (!tctx->aes_dev) { 958 err = -ENODEV; 959 goto exit; 960 } 961 962 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, 963 tctx->cipher); 964 if (err) 965 goto exit; 966 967 err = kmb_ocs_sk_run(req); 968 969 exit: 970 crypto_finalize_skcipher_request(engine, req, err); 971 972 return 0; 973 } 974 975 static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, 976 void *areq) 977 { 978 struct aead_request *req = container_of(areq, 979 struct aead_request, base); 980 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 981 int err; 982 983 if (!tctx->aes_dev) 984 return -ENODEV; 985 986 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, 987 tctx->cipher); 988 if (err) 989 goto exit; 990 991 err = kmb_ocs_aead_run(req); 992 993 exit: 994 crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); 995 996 return 0; 997 } 998 999 static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 1000 unsigned int key_len) 1001 { 1002 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); 1003 } 1004 1005 static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, 1006 unsigned int key_len) 1007 { 1008 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); 1009 } 1010 1011 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1012 static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) 1013 { 1014 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); 1015 } 1016 1017 static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) 1018 { 1019 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); 1020 } 1021 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1022 1023 static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) 1024 { 1025 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); 1026 } 1027 1028 static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) 1029 { 1030 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); 1031 } 1032 1033 static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) 1034 { 1035 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); 1036 } 1037 1038 static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) 1039 { 1040 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); 1041 } 1042 1043 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1044 static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) 1045 { 1046 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); 1047 } 1048 1049 static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) 1050 { 1051 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); 1052 } 1053 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1054 1055 static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) 1056 { 1057 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); 1058 } 1059 1060 static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) 1061 { 1062 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); 1063 } 1064 1065 static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) 1066 { 1067 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); 1068 } 1069 1070 static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) 1071 { 1072 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); 1073 } 1074 1075 static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 1076 unsigned int key_len) 1077 { 1078 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); 1079 } 1080 1081 static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, 1082 unsigned int key_len) 1083 { 1084 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); 1085 } 1086 1087 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1088 static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) 1089 { 1090 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); 1091 } 1092 1093 static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) 1094 { 1095 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); 1096 } 1097 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1098 1099 static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) 1100 { 1101 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); 1102 } 1103 1104 static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) 1105 { 1106 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); 1107 } 1108 1109 static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) 1110 { 1111 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); 1112 } 1113 1114 static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) 1115 { 1116 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); 1117 } 1118 1119 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1120 static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) 1121 { 1122 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); 1123 } 1124 1125 static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) 1126 { 1127 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); 1128 } 1129 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1130 1131 static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) 1132 { 1133 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); 1134 } 1135 1136 static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) 1137 { 1138 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); 1139 } 1140 1141 static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) 1142 { 1143 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); 1144 } 1145 1146 static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) 1147 { 1148 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); 1149 } 1150 1151 static inline int ocs_common_init(struct ocs_aes_tctx *tctx) 1152 { 1153 tctx->engine_ctx.op.prepare_request = NULL; 1154 tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; 1155 tctx->engine_ctx.op.unprepare_request = NULL; 1156 1157 return 0; 1158 } 1159 1160 static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) 1161 { 1162 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 1163 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 1164 struct crypto_sync_skcipher *blk; 1165 1166 /* set fallback cipher in case it will be needed */ 1167 blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); 1168 if (IS_ERR(blk)) 1169 return PTR_ERR(blk); 1170 1171 tctx->sw_cipher.sk = blk; 1172 1173 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); 1174 1175 return ocs_common_init(tctx); 1176 } 1177 1178 static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) 1179 { 1180 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 1181 1182 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); 1183 1184 return ocs_common_init(tctx); 1185 } 1186 1187 static inline void clear_key(struct ocs_aes_tctx *tctx) 1188 { 1189 memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); 1190 1191 /* Zero key registers if set */ 1192 if (tctx->aes_dev) 1193 ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, 1194 tctx->key, OCS_AES); 1195 } 1196 1197 static void ocs_exit_tfm(struct crypto_skcipher *tfm) 1198 { 1199 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 1200 1201 clear_key(tctx); 1202 1203 if (tctx->sw_cipher.sk) { 1204 crypto_free_sync_skcipher(tctx->sw_cipher.sk); 1205 tctx->sw_cipher.sk = NULL; 1206 } 1207 } 1208 1209 static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) 1210 { 1211 tctx->engine_ctx.op.prepare_request = NULL; 1212 tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; 1213 tctx->engine_ctx.op.unprepare_request = NULL; 1214 1215 return 0; 1216 } 1217 1218 static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) 1219 { 1220 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 1221 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 1222 struct crypto_aead *blk; 1223 1224 /* Set fallback cipher in case it will be needed */ 1225 blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); 1226 if (IS_ERR(blk)) 1227 return PTR_ERR(blk); 1228 1229 tctx->sw_cipher.aead = blk; 1230 1231 crypto_aead_set_reqsize(tfm, 1232 max(sizeof(struct ocs_aes_rctx), 1233 (sizeof(struct aead_request) + 1234 crypto_aead_reqsize(tctx->sw_cipher.aead)))); 1235 1236 return ocs_common_aead_init(tctx); 1237 } 1238 1239 static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, 1240 unsigned int authsize) 1241 { 1242 switch (authsize) { 1243 case 4: 1244 case 6: 1245 case 8: 1246 case 10: 1247 case 12: 1248 case 14: 1249 case 16: 1250 return 0; 1251 default: 1252 return -EINVAL; 1253 } 1254 } 1255 1256 static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, 1257 unsigned int authsize) 1258 { 1259 return crypto_gcm_check_authsize(authsize); 1260 } 1261 1262 static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) 1263 { 1264 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 1265 1266 crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); 1267 1268 return ocs_common_aead_init(tctx); 1269 } 1270 1271 static void ocs_aead_cra_exit(struct crypto_aead *tfm) 1272 { 1273 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 1274 1275 clear_key(tctx); 1276 1277 if (tctx->sw_cipher.aead) { 1278 crypto_free_aead(tctx->sw_cipher.aead); 1279 tctx->sw_cipher.aead = NULL; 1280 } 1281 } 1282 1283 static struct skcipher_alg algs[] = { 1284 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1285 { 1286 .base.cra_name = "ecb(aes)", 1287 .base.cra_driver_name = "ecb-aes-keembay-ocs", 1288 .base.cra_priority = KMB_OCS_PRIORITY, 1289 .base.cra_flags = CRYPTO_ALG_ASYNC | 1290 CRYPTO_ALG_KERN_DRIVER_ONLY | 1291 CRYPTO_ALG_NEED_FALLBACK, 1292 .base.cra_blocksize = AES_BLOCK_SIZE, 1293 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1294 .base.cra_module = THIS_MODULE, 1295 .base.cra_alignmask = 0, 1296 1297 .min_keysize = OCS_AES_MIN_KEY_SIZE, 1298 .max_keysize = OCS_AES_MAX_KEY_SIZE, 1299 .setkey = kmb_ocs_aes_set_key, 1300 .encrypt = kmb_ocs_aes_ecb_encrypt, 1301 .decrypt = kmb_ocs_aes_ecb_decrypt, 1302 .init = ocs_aes_init_tfm, 1303 .exit = ocs_exit_tfm, 1304 }, 1305 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1306 { 1307 .base.cra_name = "cbc(aes)", 1308 .base.cra_driver_name = "cbc-aes-keembay-ocs", 1309 .base.cra_priority = KMB_OCS_PRIORITY, 1310 .base.cra_flags = CRYPTO_ALG_ASYNC | 1311 CRYPTO_ALG_KERN_DRIVER_ONLY | 1312 CRYPTO_ALG_NEED_FALLBACK, 1313 .base.cra_blocksize = AES_BLOCK_SIZE, 1314 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1315 .base.cra_module = THIS_MODULE, 1316 .base.cra_alignmask = 0, 1317 1318 .min_keysize = OCS_AES_MIN_KEY_SIZE, 1319 .max_keysize = OCS_AES_MAX_KEY_SIZE, 1320 .ivsize = AES_BLOCK_SIZE, 1321 .setkey = kmb_ocs_aes_set_key, 1322 .encrypt = kmb_ocs_aes_cbc_encrypt, 1323 .decrypt = kmb_ocs_aes_cbc_decrypt, 1324 .init = ocs_aes_init_tfm, 1325 .exit = ocs_exit_tfm, 1326 }, 1327 { 1328 .base.cra_name = "ctr(aes)", 1329 .base.cra_driver_name = "ctr-aes-keembay-ocs", 1330 .base.cra_priority = KMB_OCS_PRIORITY, 1331 .base.cra_flags = CRYPTO_ALG_ASYNC | 1332 CRYPTO_ALG_KERN_DRIVER_ONLY | 1333 CRYPTO_ALG_NEED_FALLBACK, 1334 .base.cra_blocksize = 1, 1335 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1336 .base.cra_module = THIS_MODULE, 1337 .base.cra_alignmask = 0, 1338 1339 .min_keysize = OCS_AES_MIN_KEY_SIZE, 1340 .max_keysize = OCS_AES_MAX_KEY_SIZE, 1341 .ivsize = AES_BLOCK_SIZE, 1342 .setkey = kmb_ocs_aes_set_key, 1343 .encrypt = kmb_ocs_aes_ctr_encrypt, 1344 .decrypt = kmb_ocs_aes_ctr_decrypt, 1345 .init = ocs_aes_init_tfm, 1346 .exit = ocs_exit_tfm, 1347 }, 1348 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1349 { 1350 .base.cra_name = "cts(cbc(aes))", 1351 .base.cra_driver_name = "cts-aes-keembay-ocs", 1352 .base.cra_priority = KMB_OCS_PRIORITY, 1353 .base.cra_flags = CRYPTO_ALG_ASYNC | 1354 CRYPTO_ALG_KERN_DRIVER_ONLY | 1355 CRYPTO_ALG_NEED_FALLBACK, 1356 .base.cra_blocksize = AES_BLOCK_SIZE, 1357 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1358 .base.cra_module = THIS_MODULE, 1359 .base.cra_alignmask = 0, 1360 1361 .min_keysize = OCS_AES_MIN_KEY_SIZE, 1362 .max_keysize = OCS_AES_MAX_KEY_SIZE, 1363 .ivsize = AES_BLOCK_SIZE, 1364 .setkey = kmb_ocs_aes_set_key, 1365 .encrypt = kmb_ocs_aes_cts_encrypt, 1366 .decrypt = kmb_ocs_aes_cts_decrypt, 1367 .init = ocs_aes_init_tfm, 1368 .exit = ocs_exit_tfm, 1369 }, 1370 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1371 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1372 { 1373 .base.cra_name = "ecb(sm4)", 1374 .base.cra_driver_name = "ecb-sm4-keembay-ocs", 1375 .base.cra_priority = KMB_OCS_PRIORITY, 1376 .base.cra_flags = CRYPTO_ALG_ASYNC | 1377 CRYPTO_ALG_KERN_DRIVER_ONLY, 1378 .base.cra_blocksize = AES_BLOCK_SIZE, 1379 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1380 .base.cra_module = THIS_MODULE, 1381 .base.cra_alignmask = 0, 1382 1383 .min_keysize = OCS_SM4_KEY_SIZE, 1384 .max_keysize = OCS_SM4_KEY_SIZE, 1385 .setkey = kmb_ocs_sm4_set_key, 1386 .encrypt = kmb_ocs_sm4_ecb_encrypt, 1387 .decrypt = kmb_ocs_sm4_ecb_decrypt, 1388 .init = ocs_sm4_init_tfm, 1389 .exit = ocs_exit_tfm, 1390 }, 1391 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1392 { 1393 .base.cra_name = "cbc(sm4)", 1394 .base.cra_driver_name = "cbc-sm4-keembay-ocs", 1395 .base.cra_priority = KMB_OCS_PRIORITY, 1396 .base.cra_flags = CRYPTO_ALG_ASYNC | 1397 CRYPTO_ALG_KERN_DRIVER_ONLY, 1398 .base.cra_blocksize = AES_BLOCK_SIZE, 1399 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1400 .base.cra_module = THIS_MODULE, 1401 .base.cra_alignmask = 0, 1402 1403 .min_keysize = OCS_SM4_KEY_SIZE, 1404 .max_keysize = OCS_SM4_KEY_SIZE, 1405 .ivsize = AES_BLOCK_SIZE, 1406 .setkey = kmb_ocs_sm4_set_key, 1407 .encrypt = kmb_ocs_sm4_cbc_encrypt, 1408 .decrypt = kmb_ocs_sm4_cbc_decrypt, 1409 .init = ocs_sm4_init_tfm, 1410 .exit = ocs_exit_tfm, 1411 }, 1412 { 1413 .base.cra_name = "ctr(sm4)", 1414 .base.cra_driver_name = "ctr-sm4-keembay-ocs", 1415 .base.cra_priority = KMB_OCS_PRIORITY, 1416 .base.cra_flags = CRYPTO_ALG_ASYNC | 1417 CRYPTO_ALG_KERN_DRIVER_ONLY, 1418 .base.cra_blocksize = 1, 1419 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1420 .base.cra_module = THIS_MODULE, 1421 .base.cra_alignmask = 0, 1422 1423 .min_keysize = OCS_SM4_KEY_SIZE, 1424 .max_keysize = OCS_SM4_KEY_SIZE, 1425 .ivsize = AES_BLOCK_SIZE, 1426 .setkey = kmb_ocs_sm4_set_key, 1427 .encrypt = kmb_ocs_sm4_ctr_encrypt, 1428 .decrypt = kmb_ocs_sm4_ctr_decrypt, 1429 .init = ocs_sm4_init_tfm, 1430 .exit = ocs_exit_tfm, 1431 }, 1432 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1433 { 1434 .base.cra_name = "cts(cbc(sm4))", 1435 .base.cra_driver_name = "cts-sm4-keembay-ocs", 1436 .base.cra_priority = KMB_OCS_PRIORITY, 1437 .base.cra_flags = CRYPTO_ALG_ASYNC | 1438 CRYPTO_ALG_KERN_DRIVER_ONLY, 1439 .base.cra_blocksize = AES_BLOCK_SIZE, 1440 .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1441 .base.cra_module = THIS_MODULE, 1442 .base.cra_alignmask = 0, 1443 1444 .min_keysize = OCS_SM4_KEY_SIZE, 1445 .max_keysize = OCS_SM4_KEY_SIZE, 1446 .ivsize = AES_BLOCK_SIZE, 1447 .setkey = kmb_ocs_sm4_set_key, 1448 .encrypt = kmb_ocs_sm4_cts_encrypt, 1449 .decrypt = kmb_ocs_sm4_cts_decrypt, 1450 .init = ocs_sm4_init_tfm, 1451 .exit = ocs_exit_tfm, 1452 } 1453 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1454 }; 1455 1456 static struct aead_alg algs_aead[] = { 1457 { 1458 .base = { 1459 .cra_name = "gcm(aes)", 1460 .cra_driver_name = "gcm-aes-keembay-ocs", 1461 .cra_priority = KMB_OCS_PRIORITY, 1462 .cra_flags = CRYPTO_ALG_ASYNC | 1463 CRYPTO_ALG_KERN_DRIVER_ONLY | 1464 CRYPTO_ALG_NEED_FALLBACK, 1465 .cra_blocksize = 1, 1466 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1467 .cra_alignmask = 0, 1468 .cra_module = THIS_MODULE, 1469 }, 1470 .init = ocs_aes_aead_cra_init, 1471 .exit = ocs_aead_cra_exit, 1472 .ivsize = GCM_AES_IV_SIZE, 1473 .maxauthsize = AES_BLOCK_SIZE, 1474 .setauthsize = kmb_ocs_aead_gcm_setauthsize, 1475 .setkey = kmb_ocs_aes_aead_set_key, 1476 .encrypt = kmb_ocs_aes_gcm_encrypt, 1477 .decrypt = kmb_ocs_aes_gcm_decrypt, 1478 }, 1479 { 1480 .base = { 1481 .cra_name = "ccm(aes)", 1482 .cra_driver_name = "ccm-aes-keembay-ocs", 1483 .cra_priority = KMB_OCS_PRIORITY, 1484 .cra_flags = CRYPTO_ALG_ASYNC | 1485 CRYPTO_ALG_KERN_DRIVER_ONLY | 1486 CRYPTO_ALG_NEED_FALLBACK, 1487 .cra_blocksize = 1, 1488 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1489 .cra_alignmask = 0, 1490 .cra_module = THIS_MODULE, 1491 }, 1492 .init = ocs_aes_aead_cra_init, 1493 .exit = ocs_aead_cra_exit, 1494 .ivsize = AES_BLOCK_SIZE, 1495 .maxauthsize = AES_BLOCK_SIZE, 1496 .setauthsize = kmb_ocs_aead_ccm_setauthsize, 1497 .setkey = kmb_ocs_aes_aead_set_key, 1498 .encrypt = kmb_ocs_aes_ccm_encrypt, 1499 .decrypt = kmb_ocs_aes_ccm_decrypt, 1500 }, 1501 { 1502 .base = { 1503 .cra_name = "gcm(sm4)", 1504 .cra_driver_name = "gcm-sm4-keembay-ocs", 1505 .cra_priority = KMB_OCS_PRIORITY, 1506 .cra_flags = CRYPTO_ALG_ASYNC | 1507 CRYPTO_ALG_KERN_DRIVER_ONLY, 1508 .cra_blocksize = 1, 1509 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1510 .cra_alignmask = 0, 1511 .cra_module = THIS_MODULE, 1512 }, 1513 .init = ocs_sm4_aead_cra_init, 1514 .exit = ocs_aead_cra_exit, 1515 .ivsize = GCM_AES_IV_SIZE, 1516 .maxauthsize = AES_BLOCK_SIZE, 1517 .setauthsize = kmb_ocs_aead_gcm_setauthsize, 1518 .setkey = kmb_ocs_sm4_aead_set_key, 1519 .encrypt = kmb_ocs_sm4_gcm_encrypt, 1520 .decrypt = kmb_ocs_sm4_gcm_decrypt, 1521 }, 1522 { 1523 .base = { 1524 .cra_name = "ccm(sm4)", 1525 .cra_driver_name = "ccm-sm4-keembay-ocs", 1526 .cra_priority = KMB_OCS_PRIORITY, 1527 .cra_flags = CRYPTO_ALG_ASYNC | 1528 CRYPTO_ALG_KERN_DRIVER_ONLY, 1529 .cra_blocksize = 1, 1530 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1531 .cra_alignmask = 0, 1532 .cra_module = THIS_MODULE, 1533 }, 1534 .init = ocs_sm4_aead_cra_init, 1535 .exit = ocs_aead_cra_exit, 1536 .ivsize = AES_BLOCK_SIZE, 1537 .maxauthsize = AES_BLOCK_SIZE, 1538 .setauthsize = kmb_ocs_aead_ccm_setauthsize, 1539 .setkey = kmb_ocs_sm4_aead_set_key, 1540 .encrypt = kmb_ocs_sm4_ccm_encrypt, 1541 .decrypt = kmb_ocs_sm4_ccm_decrypt, 1542 } 1543 }; 1544 1545 static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) 1546 { 1547 crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); 1548 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); 1549 } 1550 1551 static int register_aes_algs(struct ocs_aes_dev *aes_dev) 1552 { 1553 int ret; 1554 1555 /* 1556 * If any algorithm fails to register, all preceding algorithms that 1557 * were successfully registered will be automatically unregistered. 1558 */ 1559 ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); 1560 if (ret) 1561 return ret; 1562 1563 ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); 1564 if (ret) 1565 crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); 1566 1567 return ret; 1568 } 1569 1570 /* Device tree driver match. */ 1571 static const struct of_device_id kmb_ocs_aes_of_match[] = { 1572 { 1573 .compatible = "intel,keembay-ocs-aes", 1574 }, 1575 {} 1576 }; 1577 1578 static int kmb_ocs_aes_remove(struct platform_device *pdev) 1579 { 1580 struct ocs_aes_dev *aes_dev; 1581 1582 aes_dev = platform_get_drvdata(pdev); 1583 1584 unregister_aes_algs(aes_dev); 1585 1586 spin_lock(&ocs_aes.lock); 1587 list_del(&aes_dev->list); 1588 spin_unlock(&ocs_aes.lock); 1589 1590 crypto_engine_exit(aes_dev->engine); 1591 1592 return 0; 1593 } 1594 1595 static int kmb_ocs_aes_probe(struct platform_device *pdev) 1596 { 1597 struct device *dev = &pdev->dev; 1598 struct ocs_aes_dev *aes_dev; 1599 int rc; 1600 1601 aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); 1602 if (!aes_dev) 1603 return -ENOMEM; 1604 1605 aes_dev->dev = dev; 1606 1607 platform_set_drvdata(pdev, aes_dev); 1608 1609 rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1610 if (rc) { 1611 dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); 1612 return rc; 1613 } 1614 1615 /* Get base register address. */ 1616 aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); 1617 if (IS_ERR(aes_dev->base_reg)) 1618 return PTR_ERR(aes_dev->base_reg); 1619 1620 /* Get and request IRQ */ 1621 aes_dev->irq = platform_get_irq(pdev, 0); 1622 if (aes_dev->irq < 0) 1623 return aes_dev->irq; 1624 1625 rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, 1626 NULL, 0, "keembay-ocs-aes", aes_dev); 1627 if (rc < 0) { 1628 dev_err(dev, "Could not request IRQ\n"); 1629 return rc; 1630 } 1631 1632 INIT_LIST_HEAD(&aes_dev->list); 1633 spin_lock(&ocs_aes.lock); 1634 list_add_tail(&aes_dev->list, &ocs_aes.dev_list); 1635 spin_unlock(&ocs_aes.lock); 1636 1637 init_completion(&aes_dev->irq_completion); 1638 1639 /* Initialize crypto engine */ 1640 aes_dev->engine = crypto_engine_alloc_init(dev, true); 1641 if (!aes_dev->engine) { 1642 rc = -ENOMEM; 1643 goto list_del; 1644 } 1645 1646 rc = crypto_engine_start(aes_dev->engine); 1647 if (rc) { 1648 dev_err(dev, "Could not start crypto engine\n"); 1649 goto cleanup; 1650 } 1651 1652 rc = register_aes_algs(aes_dev); 1653 if (rc) { 1654 dev_err(dev, 1655 "Could not register OCS algorithms with Crypto API\n"); 1656 goto cleanup; 1657 } 1658 1659 return 0; 1660 1661 cleanup: 1662 crypto_engine_exit(aes_dev->engine); 1663 list_del: 1664 spin_lock(&ocs_aes.lock); 1665 list_del(&aes_dev->list); 1666 spin_unlock(&ocs_aes.lock); 1667 1668 return rc; 1669 } 1670 1671 /* The OCS driver is a platform device. */ 1672 static struct platform_driver kmb_ocs_aes_driver = { 1673 .probe = kmb_ocs_aes_probe, 1674 .remove = kmb_ocs_aes_remove, 1675 .driver = { 1676 .name = DRV_NAME, 1677 .of_match_table = kmb_ocs_aes_of_match, 1678 }, 1679 }; 1680 1681 module_platform_driver(kmb_ocs_aes_driver); 1682 1683 MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); 1684 MODULE_LICENSE("GPL"); 1685 1686 MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); 1687 MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); 1688 MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); 1689 MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); 1690 1691 MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); 1692 MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); 1693 MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); 1694 MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); 1695 1696 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1697 MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); 1698 MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); 1699 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1700 1701 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1702 MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); 1703 MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); 1704 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1705