1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel Keem Bay OCS AES Crypto Driver. 4 * 5 * Copyright (C) 2018-2020 Intel Corporation 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/engine.h> 10 #include <crypto/gcm.h> 11 #include <crypto/internal/aead.h> 12 #include <crypto/internal/skcipher.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/clk.h> 15 #include <linux/completion.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/err.h> 18 #include <linux/interrupt.h> 19 #include <linux/io.h> 20 #include <linux/kernel.h> 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/platform_device.h> 24 #include <linux/string.h> 25 26 #include "ocs-aes.h" 27 28 #define KMB_OCS_PRIORITY 350 29 #define DRV_NAME "keembay-ocs-aes" 30 31 #define OCS_AES_MIN_KEY_SIZE 16 32 #define OCS_AES_MAX_KEY_SIZE 32 33 #define OCS_AES_KEYSIZE_128 16 34 #define OCS_AES_KEYSIZE_192 24 35 #define OCS_AES_KEYSIZE_256 32 36 #define OCS_SM4_KEY_SIZE 16 37 38 /** 39 * struct ocs_aes_tctx - OCS AES Transform context 40 * @aes_dev: The OCS AES device. 41 * @key: AES/SM4 key. 42 * @key_len: The length (in bytes) of @key. 43 * @cipher: OCS cipher to use (either AES or SM4). 44 * @sw_cipher: The cipher to use as fallback. 45 * @use_fallback: Whether or not fallback cipher should be used. 46 */ 47 struct ocs_aes_tctx { 48 struct ocs_aes_dev *aes_dev; 49 u8 key[OCS_AES_KEYSIZE_256]; 50 unsigned int key_len; 51 enum ocs_cipher cipher; 52 union { 53 struct crypto_sync_skcipher *sk; 54 struct crypto_aead *aead; 55 } sw_cipher; 56 bool use_fallback; 57 }; 58 59 /** 60 * struct ocs_aes_rctx - OCS AES Request context. 61 * @instruction: Instruction to be executed (encrypt / decrypt). 62 * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) 63 * @src_nents: Number of source SG entries. 64 * @dst_nents: Number of destination SG entries. 65 * @src_dma_count: The number of DMA-mapped entries of the source SG. 66 * @dst_dma_count: The number of DMA-mapped entries of the destination SG. 67 * @in_place: Whether or not this is an in place request, i.e., 68 * src_sg == dst_sg. 69 * @src_dll: OCS DMA linked list for input data. 70 * @dst_dll: OCS DMA linked list for output data. 71 * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC 72 * mode). 73 * @cts_swap: Whether or not CTS swap must be performed. 74 * @aad_src_dll: OCS DMA linked list for input AAD data. 75 * @aad_dst_dll: OCS DMA linked list for output AAD data. 76 * @in_tag: Buffer to hold input encrypted tag (only used for 77 * CCM/GCM decrypt). 78 * @out_tag: Buffer to hold output encrypted / decrypted tag (only 79 * used for GCM encrypt / decrypt). 80 */ 81 struct ocs_aes_rctx { 82 /* Fields common across all modes. */ 83 enum ocs_instruction instruction; 84 enum ocs_mode mode; 85 int src_nents; 86 int dst_nents; 87 int src_dma_count; 88 int dst_dma_count; 89 bool in_place; 90 struct ocs_dll_desc src_dll; 91 struct ocs_dll_desc dst_dll; 92 93 /* CBC specific */ 94 u8 last_ct_blk[AES_BLOCK_SIZE]; 95 96 /* CTS specific */ 97 int cts_swap; 98 99 /* CCM/GCM specific */ 100 struct ocs_dll_desc aad_src_dll; 101 struct ocs_dll_desc aad_dst_dll; 102 u8 in_tag[AES_BLOCK_SIZE]; 103 104 /* GCM specific */ 105 u8 out_tag[AES_BLOCK_SIZE]; 106 }; 107 108 /* Driver data. */ 109 struct ocs_aes_drv { 110 struct list_head dev_list; 111 spinlock_t lock; /* Protects dev_list. */ 112 }; 113 114 static struct ocs_aes_drv ocs_aes = { 115 .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), 116 .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), 117 }; 118 119 static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) 120 { 121 struct ocs_aes_dev *aes_dev; 122 123 spin_lock(&ocs_aes.lock); 124 125 if (tctx->aes_dev) { 126 aes_dev = tctx->aes_dev; 127 goto exit; 128 } 129 130 /* Only a single OCS device available */ 131 aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); 132 tctx->aes_dev = aes_dev; 133 134 exit: 135 spin_unlock(&ocs_aes.lock); 136 137 return aes_dev; 138 } 139 140 /* 141 * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual 142 * key is being passed in. 143 * 144 * Return: 0 if key is valid, -EINVAL otherwise. 145 */ 146 static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) 147 { 148 if (!in_key) 149 return -EINVAL; 150 151 /* For AES, only 128-byte or 256-byte keys are supported. */ 152 if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || 153 key_len == OCS_AES_KEYSIZE_256)) 154 return 0; 155 156 /* For SM4, only 128-byte keys are supported. */ 157 if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) 158 return 0; 159 160 /* Everything else is unsupported. */ 161 return -EINVAL; 162 } 163 164 /* Save key into transformation context. */ 165 static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, 166 enum ocs_cipher cipher) 167 { 168 int ret; 169 170 ret = check_key(in_key, key_len, cipher); 171 if (ret) 172 return ret; 173 174 memcpy(tctx->key, in_key, key_len); 175 tctx->key_len = key_len; 176 tctx->cipher = cipher; 177 178 return 0; 179 } 180 181 /* Set key for symmetric cypher. */ 182 static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 183 size_t key_len, enum ocs_cipher cipher) 184 { 185 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 186 187 /* Fallback is used for AES with 192-bit key. */ 188 tctx->use_fallback = (cipher == OCS_AES && 189 key_len == OCS_AES_KEYSIZE_192); 190 191 if (!tctx->use_fallback) 192 return save_key(tctx, in_key, key_len, cipher); 193 194 crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, 195 CRYPTO_TFM_REQ_MASK); 196 crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, 197 tfm->base.crt_flags & 198 CRYPTO_TFM_REQ_MASK); 199 200 return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); 201 } 202 203 /* Set key for AEAD cipher. */ 204 static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, 205 size_t key_len, enum ocs_cipher cipher) 206 { 207 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 208 209 /* Fallback is used for AES with 192-bit key. */ 210 tctx->use_fallback = (cipher == OCS_AES && 211 key_len == OCS_AES_KEYSIZE_192); 212 213 if (!tctx->use_fallback) 214 return save_key(tctx, in_key, key_len, cipher); 215 216 crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); 217 crypto_aead_set_flags(tctx->sw_cipher.aead, 218 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 219 220 return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); 221 } 222 223 /* Swap two AES blocks in SG lists. */ 224 static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, 225 off_t blk1_offset, off_t blk2_offset) 226 { 227 u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; 228 229 /* 230 * No easy way to copy within sg list, so copy both blocks to temporary 231 * buffers first. 232 */ 233 sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); 234 sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); 235 sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); 236 sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); 237 } 238 239 /* Initialize request context to default values. */ 240 static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) 241 { 242 /* Zero everything. */ 243 memset(rctx, 0, sizeof(*rctx)); 244 245 /* Set initial value for DMA addresses. */ 246 rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; 247 rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; 248 rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; 249 rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; 250 } 251 252 static int kmb_ocs_sk_validate_input(struct skcipher_request *req, 253 enum ocs_mode mode) 254 { 255 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 256 int iv_size = crypto_skcipher_ivsize(tfm); 257 258 switch (mode) { 259 case OCS_MODE_ECB: 260 /* Ensure input length is multiple of block size */ 261 if (req->cryptlen % AES_BLOCK_SIZE != 0) 262 return -EINVAL; 263 264 return 0; 265 266 case OCS_MODE_CBC: 267 /* Ensure input length is multiple of block size */ 268 if (req->cryptlen % AES_BLOCK_SIZE != 0) 269 return -EINVAL; 270 271 /* Ensure IV is present and block size in length */ 272 if (!req->iv || iv_size != AES_BLOCK_SIZE) 273 return -EINVAL; 274 /* 275 * NOTE: Since req->cryptlen == 0 case was already handled in 276 * kmb_ocs_sk_common(), the above two conditions also guarantee 277 * that: cryptlen >= iv_size 278 */ 279 return 0; 280 281 case OCS_MODE_CTR: 282 /* Ensure IV is present and block size in length */ 283 if (!req->iv || iv_size != AES_BLOCK_SIZE) 284 return -EINVAL; 285 return 0; 286 287 case OCS_MODE_CTS: 288 /* Ensure input length >= block size */ 289 if (req->cryptlen < AES_BLOCK_SIZE) 290 return -EINVAL; 291 292 /* Ensure IV is present and block size in length */ 293 if (!req->iv || iv_size != AES_BLOCK_SIZE) 294 return -EINVAL; 295 296 return 0; 297 default: 298 return -EINVAL; 299 } 300 } 301 302 /* 303 * Called by encrypt() / decrypt() skcipher functions. 304 * 305 * Use fallback if needed, otherwise initialize context and enqueue request 306 * into engine. 307 */ 308 static int kmb_ocs_sk_common(struct skcipher_request *req, 309 enum ocs_cipher cipher, 310 enum ocs_instruction instruction, 311 enum ocs_mode mode) 312 { 313 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 314 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 315 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 316 struct ocs_aes_dev *aes_dev; 317 int rc; 318 319 if (tctx->use_fallback) { 320 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); 321 322 skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); 323 skcipher_request_set_callback(subreq, req->base.flags, NULL, 324 NULL); 325 skcipher_request_set_crypt(subreq, req->src, req->dst, 326 req->cryptlen, req->iv); 327 328 if (instruction == OCS_ENCRYPT) 329 rc = crypto_skcipher_encrypt(subreq); 330 else 331 rc = crypto_skcipher_decrypt(subreq); 332 333 skcipher_request_zero(subreq); 334 335 return rc; 336 } 337 338 /* 339 * If cryptlen == 0, no processing needed for ECB, CBC and CTR. 340 * 341 * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. 342 */ 343 if (!req->cryptlen && mode != OCS_MODE_CTS) 344 return 0; 345 346 rc = kmb_ocs_sk_validate_input(req, mode); 347 if (rc) 348 return rc; 349 350 aes_dev = kmb_ocs_aes_find_dev(tctx); 351 if (!aes_dev) 352 return -ENODEV; 353 354 if (cipher != tctx->cipher) 355 return -EINVAL; 356 357 ocs_aes_init_rctx(rctx); 358 rctx->instruction = instruction; 359 rctx->mode = mode; 360 361 return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); 362 } 363 364 static void cleanup_ocs_dma_linked_list(struct device *dev, 365 struct ocs_dll_desc *dll) 366 { 367 if (dll->vaddr) 368 dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); 369 dll->vaddr = NULL; 370 dll->size = 0; 371 dll->dma_addr = DMA_MAPPING_ERROR; 372 } 373 374 static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) 375 { 376 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 377 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 378 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 379 struct device *dev = tctx->aes_dev->dev; 380 381 if (rctx->src_dma_count) { 382 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 383 rctx->src_dma_count = 0; 384 } 385 386 if (rctx->dst_dma_count) { 387 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? 388 DMA_BIDIRECTIONAL : 389 DMA_FROM_DEVICE); 390 rctx->dst_dma_count = 0; 391 } 392 393 /* Clean up OCS DMA linked lists */ 394 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); 395 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); 396 } 397 398 static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) 399 { 400 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 401 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 402 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 403 int iv_size = crypto_skcipher_ivsize(tfm); 404 int rc; 405 406 /* 407 * For CBC decrypt, save last block (iv) to last_ct_blk buffer. 408 * 409 * Note: if we are here, we already checked that cryptlen >= iv_size 410 * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see 411 * kmb_ocs_sk_validate_input(). 412 */ 413 if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) 414 scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, 415 req->cryptlen - iv_size, iv_size, 0); 416 417 /* For CTS decrypt, swap last two blocks, if needed. */ 418 if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) 419 sg_swap_blocks(req->dst, rctx->dst_nents, 420 req->cryptlen - AES_BLOCK_SIZE, 421 req->cryptlen - (2 * AES_BLOCK_SIZE)); 422 423 /* src and dst buffers are the same, use bidirectional DMA mapping. */ 424 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, 425 rctx->dst_nents, DMA_BIDIRECTIONAL); 426 if (rctx->dst_dma_count == 0) { 427 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); 428 return -ENOMEM; 429 } 430 431 /* Create DST linked list */ 432 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 433 rctx->dst_dma_count, &rctx->dst_dll, 434 req->cryptlen, 0); 435 if (rc) 436 return rc; 437 /* 438 * If descriptor creation was successful, set the src_dll.dma_addr to 439 * the value of dst_dll.dma_addr, as we do in-place AES operation on 440 * the src. 441 */ 442 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; 443 444 return 0; 445 } 446 447 static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) 448 { 449 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 450 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 451 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 452 int rc; 453 454 rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); 455 if (rctx->src_nents < 0) 456 return -EBADMSG; 457 458 /* Map SRC SG. */ 459 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, 460 rctx->src_nents, DMA_TO_DEVICE); 461 if (rctx->src_dma_count == 0) { 462 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); 463 return -ENOMEM; 464 } 465 466 /* Create SRC linked list */ 467 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, 468 rctx->src_dma_count, &rctx->src_dll, 469 req->cryptlen, 0); 470 if (rc) 471 return rc; 472 473 /* Map DST SG. */ 474 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, 475 rctx->dst_nents, DMA_FROM_DEVICE); 476 if (rctx->dst_dma_count == 0) { 477 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); 478 return -ENOMEM; 479 } 480 481 /* Create DST linked list */ 482 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 483 rctx->dst_dma_count, &rctx->dst_dll, 484 req->cryptlen, 0); 485 if (rc) 486 return rc; 487 488 /* If this is not a CTS decrypt operation with swapping, we are done. */ 489 if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) 490 return 0; 491 492 /* 493 * Otherwise, we have to copy src to dst (as we cannot modify src). 494 * Use OCS AES bypass mode to copy src to dst via DMA. 495 * 496 * NOTE: for anything other than small data sizes this is rather 497 * inefficient. 498 */ 499 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, 500 rctx->src_dll.dma_addr, req->cryptlen); 501 if (rc) 502 return rc; 503 504 /* 505 * Now dst == src, so clean up what we did so far and use in_place 506 * logic. 507 */ 508 kmb_ocs_sk_dma_cleanup(req); 509 rctx->in_place = true; 510 511 return kmb_ocs_sk_prepare_inplace(req); 512 } 513 514 static int kmb_ocs_sk_run(struct skcipher_request *req) 515 { 516 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 517 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); 518 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 519 struct ocs_aes_dev *aes_dev = tctx->aes_dev; 520 int iv_size = crypto_skcipher_ivsize(tfm); 521 int rc; 522 523 rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 524 if (rctx->dst_nents < 0) 525 return -EBADMSG; 526 527 /* 528 * If 2 blocks or greater, and multiple of block size swap last two 529 * blocks to be compatible with other crypto API CTS implementations: 530 * OCS mode uses CBC-CS2, whereas other crypto API implementations use 531 * CBC-CS3. 532 * CBC-CS2 and CBC-CS3 defined by: 533 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf 534 */ 535 rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && 536 req->cryptlen > AES_BLOCK_SIZE && 537 req->cryptlen % AES_BLOCK_SIZE == 0); 538 539 rctx->in_place = (req->src == req->dst); 540 541 if (rctx->in_place) 542 rc = kmb_ocs_sk_prepare_inplace(req); 543 else 544 rc = kmb_ocs_sk_prepare_notinplace(req); 545 546 if (rc) 547 goto error; 548 549 rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, 550 rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, 551 req->cryptlen, req->iv, iv_size); 552 if (rc) 553 goto error; 554 555 /* Clean-up DMA before further processing output. */ 556 kmb_ocs_sk_dma_cleanup(req); 557 558 /* For CTS Encrypt, swap last 2 blocks, if needed. */ 559 if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { 560 sg_swap_blocks(req->dst, rctx->dst_nents, 561 req->cryptlen - AES_BLOCK_SIZE, 562 req->cryptlen - (2 * AES_BLOCK_SIZE)); 563 return 0; 564 } 565 566 /* For CBC copy IV to req->IV. */ 567 if (rctx->mode == OCS_MODE_CBC) { 568 /* CBC encrypt case. */ 569 if (rctx->instruction == OCS_ENCRYPT) { 570 scatterwalk_map_and_copy(req->iv, req->dst, 571 req->cryptlen - iv_size, 572 iv_size, 0); 573 return 0; 574 } 575 /* CBC decrypt case. */ 576 if (rctx->in_place) 577 memcpy(req->iv, rctx->last_ct_blk, iv_size); 578 else 579 scatterwalk_map_and_copy(req->iv, req->src, 580 req->cryptlen - iv_size, 581 iv_size, 0); 582 return 0; 583 } 584 /* For all other modes there's nothing to do. */ 585 586 return 0; 587 588 error: 589 kmb_ocs_sk_dma_cleanup(req); 590 591 return rc; 592 } 593 594 static int kmb_ocs_aead_validate_input(struct aead_request *req, 595 enum ocs_instruction instruction, 596 enum ocs_mode mode) 597 { 598 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 599 int tag_size = crypto_aead_authsize(tfm); 600 int iv_size = crypto_aead_ivsize(tfm); 601 602 /* For decrypt crytplen == len(PT) + len(tag). */ 603 if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) 604 return -EINVAL; 605 606 /* IV is mandatory. */ 607 if (!req->iv) 608 return -EINVAL; 609 610 switch (mode) { 611 case OCS_MODE_GCM: 612 if (iv_size != GCM_AES_IV_SIZE) 613 return -EINVAL; 614 615 return 0; 616 617 case OCS_MODE_CCM: 618 /* Ensure IV is present and block size in length */ 619 if (iv_size != AES_BLOCK_SIZE) 620 return -EINVAL; 621 622 return 0; 623 624 default: 625 return -EINVAL; 626 } 627 } 628 629 /* 630 * Called by encrypt() / decrypt() aead functions. 631 * 632 * Use fallback if needed, otherwise initialize context and enqueue request 633 * into engine. 634 */ 635 static int kmb_ocs_aead_common(struct aead_request *req, 636 enum ocs_cipher cipher, 637 enum ocs_instruction instruction, 638 enum ocs_mode mode) 639 { 640 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 641 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 642 struct ocs_aes_dev *dd; 643 int rc; 644 645 if (tctx->use_fallback) { 646 struct aead_request *subreq = aead_request_ctx(req); 647 648 aead_request_set_tfm(subreq, tctx->sw_cipher.aead); 649 aead_request_set_callback(subreq, req->base.flags, 650 req->base.complete, req->base.data); 651 aead_request_set_crypt(subreq, req->src, req->dst, 652 req->cryptlen, req->iv); 653 aead_request_set_ad(subreq, req->assoclen); 654 rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, 655 crypto_aead_authsize(crypto_aead_reqtfm(req))); 656 if (rc) 657 return rc; 658 659 return (instruction == OCS_ENCRYPT) ? 660 crypto_aead_encrypt(subreq) : 661 crypto_aead_decrypt(subreq); 662 } 663 664 rc = kmb_ocs_aead_validate_input(req, instruction, mode); 665 if (rc) 666 return rc; 667 668 dd = kmb_ocs_aes_find_dev(tctx); 669 if (!dd) 670 return -ENODEV; 671 672 if (cipher != tctx->cipher) 673 return -EINVAL; 674 675 ocs_aes_init_rctx(rctx); 676 rctx->instruction = instruction; 677 rctx->mode = mode; 678 679 return crypto_transfer_aead_request_to_engine(dd->engine, req); 680 } 681 682 static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) 683 { 684 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 685 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 686 struct device *dev = tctx->aes_dev->dev; 687 688 if (rctx->src_dma_count) { 689 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); 690 rctx->src_dma_count = 0; 691 } 692 693 if (rctx->dst_dma_count) { 694 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? 695 DMA_BIDIRECTIONAL : 696 DMA_FROM_DEVICE); 697 rctx->dst_dma_count = 0; 698 } 699 /* Clean up OCS DMA linked lists */ 700 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); 701 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); 702 cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); 703 cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); 704 } 705 706 /** 707 * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. 708 * @req: The AEAD request being processed. 709 * @src_dll_size: Where to store the length of the data mapped into the 710 * src_dll OCS DMA list. 711 * 712 * Do the following: 713 * - DMA map req->src and req->dst 714 * - Initialize the following OCS DMA linked lists: rctx->src_dll, 715 * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. 716 * 717 * Return: 0 on success, negative error code otherwise. 718 */ 719 static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) 720 { 721 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 722 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); 723 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 724 u32 in_size; /* The length of the data to be mapped by src_dll. */ 725 u32 out_size; /* The length of the data to be mapped by dst_dll. */ 726 u32 dst_size; /* The length of the data in dst_sg. */ 727 int rc; 728 729 /* Get number of entries in input data SG list. */ 730 rctx->src_nents = sg_nents_for_len(req->src, 731 req->assoclen + req->cryptlen); 732 if (rctx->src_nents < 0) 733 return -EBADMSG; 734 735 if (rctx->instruction == OCS_DECRYPT) { 736 /* 737 * For decrypt: 738 * - src sg list is: AAD|CT|tag 739 * - dst sg list expects: AAD|PT 740 * 741 * in_size == len(CT); out_size == len(PT) 742 */ 743 744 /* req->cryptlen includes both CT and tag. */ 745 in_size = req->cryptlen - tag_size; 746 747 /* out_size = PT size == CT size */ 748 out_size = in_size; 749 750 /* len(dst_sg) == len(AAD) + len(PT) */ 751 dst_size = req->assoclen + out_size; 752 753 /* 754 * Copy tag from source SG list to 'in_tag' buffer. 755 * 756 * Note: this needs to be done here, before DMA mapping src_sg. 757 */ 758 sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, 759 tag_size, req->assoclen + in_size); 760 761 } else { /* OCS_ENCRYPT */ 762 /* 763 * For encrypt: 764 * src sg list is: AAD|PT 765 * dst sg list expects: AAD|CT|tag 766 */ 767 /* in_size == len(PT) */ 768 in_size = req->cryptlen; 769 770 /* 771 * In CCM mode the OCS engine appends the tag to the ciphertext, 772 * but in GCM mode the tag must be read from the tag registers 773 * and appended manually below 774 */ 775 out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : 776 in_size; 777 /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ 778 dst_size = req->assoclen + in_size + tag_size; 779 } 780 *src_dll_size = in_size; 781 782 /* Get number of entries in output data SG list. */ 783 rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); 784 if (rctx->dst_nents < 0) 785 return -EBADMSG; 786 787 rctx->in_place = (req->src == req->dst) ? 1 : 0; 788 789 /* Map destination; use bidirectional mapping for in-place case. */ 790 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, 791 rctx->dst_nents, 792 rctx->in_place ? DMA_BIDIRECTIONAL : 793 DMA_FROM_DEVICE); 794 if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { 795 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); 796 return -ENOMEM; 797 } 798 799 /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ 800 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 801 rctx->dst_dma_count, 802 &rctx->aad_dst_dll, req->assoclen, 803 0); 804 if (rc) 805 return rc; 806 807 /* Create DST list: maps dst[AAD_SIZE:out_size] */ 808 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 809 rctx->dst_dma_count, &rctx->dst_dll, 810 out_size, req->assoclen); 811 if (rc) 812 return rc; 813 814 if (rctx->in_place) { 815 /* If this is not CCM encrypt, we are done. */ 816 if (!(rctx->mode == OCS_MODE_CCM && 817 rctx->instruction == OCS_ENCRYPT)) { 818 /* 819 * SRC and DST are the same, so re-use the same DMA 820 * addresses (to avoid allocating new DMA lists 821 * identical to the dst ones). 822 */ 823 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; 824 rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; 825 826 return 0; 827 } 828 /* 829 * For CCM encrypt the input and output linked lists contain 830 * different amounts of data, so, we need to create different 831 * SRC and AAD SRC lists, even for the in-place case. 832 */ 833 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 834 rctx->dst_dma_count, 835 &rctx->aad_src_dll, 836 req->assoclen, 0); 837 if (rc) 838 return rc; 839 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, 840 rctx->dst_dma_count, 841 &rctx->src_dll, in_size, 842 req->assoclen); 843 if (rc) 844 return rc; 845 846 return 0; 847 } 848 /* Not in-place case. */ 849 850 /* Map source SG. */ 851 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, 852 rctx->src_nents, DMA_TO_DEVICE); 853 if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { 854 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); 855 return -ENOMEM; 856 } 857 858 /* Create AAD SRC list. */ 859 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, 860 rctx->src_dma_count, 861 &rctx->aad_src_dll, 862 req->assoclen, 0); 863 if (rc) 864 return rc; 865 866 /* Create SRC list. */ 867 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, 868 rctx->src_dma_count, 869 &rctx->src_dll, in_size, 870 req->assoclen); 871 if (rc) 872 return rc; 873 874 if (req->assoclen == 0) 875 return 0; 876 877 /* Copy AAD from src sg to dst sg using OCS DMA. */ 878 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, 879 rctx->aad_src_dll.dma_addr, req->cryptlen); 880 if (rc) 881 dev_err(tctx->aes_dev->dev, 882 "Failed to copy source AAD to destination AAD\n"); 883 884 return rc; 885 } 886 887 static int kmb_ocs_aead_run(struct aead_request *req) 888 { 889 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 890 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); 891 struct ocs_aes_rctx *rctx = aead_request_ctx(req); 892 u32 in_size; /* The length of the data mapped by src_dll. */ 893 int rc; 894 895 rc = kmb_ocs_aead_dma_prepare(req, &in_size); 896 if (rc) 897 goto exit; 898 899 /* For CCM, we just call the OCS processing and we are done. */ 900 if (rctx->mode == OCS_MODE_CCM) { 901 rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, 902 rctx->instruction, rctx->dst_dll.dma_addr, 903 rctx->src_dll.dma_addr, in_size, 904 req->iv, 905 rctx->aad_src_dll.dma_addr, req->assoclen, 906 rctx->in_tag, tag_size); 907 goto exit; 908 } 909 /* GCM case; invoke OCS processing. */ 910 rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, 911 rctx->instruction, 912 rctx->dst_dll.dma_addr, 913 rctx->src_dll.dma_addr, in_size, 914 req->iv, 915 rctx->aad_src_dll.dma_addr, req->assoclen, 916 rctx->out_tag, tag_size); 917 if (rc) 918 goto exit; 919 920 /* For GCM decrypt, we have to compare in_tag with out_tag. */ 921 if (rctx->instruction == OCS_DECRYPT) { 922 rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? 923 -EBADMSG : 0; 924 goto exit; 925 } 926 927 /* For GCM encrypt, we must manually copy out_tag to DST sg. */ 928 929 /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ 930 kmb_ocs_aead_dma_cleanup(req); 931 932 /* Copy tag to destination sg after AAD and CT. */ 933 sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, 934 tag_size, req->assoclen + req->cryptlen); 935 936 /* Return directly as DMA cleanup already done. */ 937 return 0; 938 939 exit: 940 kmb_ocs_aead_dma_cleanup(req); 941 942 return rc; 943 } 944 945 static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, 946 void *areq) 947 { 948 struct skcipher_request *req = 949 container_of(areq, struct skcipher_request, base); 950 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 951 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 952 int err; 953 954 if (!tctx->aes_dev) { 955 err = -ENODEV; 956 goto exit; 957 } 958 959 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, 960 tctx->cipher); 961 if (err) 962 goto exit; 963 964 err = kmb_ocs_sk_run(req); 965 966 exit: 967 crypto_finalize_skcipher_request(engine, req, err); 968 969 return 0; 970 } 971 972 static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, 973 void *areq) 974 { 975 struct aead_request *req = container_of(areq, 976 struct aead_request, base); 977 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 978 int err; 979 980 if (!tctx->aes_dev) 981 return -ENODEV; 982 983 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, 984 tctx->cipher); 985 if (err) 986 goto exit; 987 988 err = kmb_ocs_aead_run(req); 989 990 exit: 991 crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); 992 993 return 0; 994 } 995 996 static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 997 unsigned int key_len) 998 { 999 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); 1000 } 1001 1002 static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, 1003 unsigned int key_len) 1004 { 1005 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); 1006 } 1007 1008 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1009 static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) 1010 { 1011 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); 1012 } 1013 1014 static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) 1015 { 1016 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); 1017 } 1018 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1019 1020 static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) 1021 { 1022 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); 1023 } 1024 1025 static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) 1026 { 1027 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); 1028 } 1029 1030 static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) 1031 { 1032 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); 1033 } 1034 1035 static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) 1036 { 1037 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); 1038 } 1039 1040 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1041 static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) 1042 { 1043 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); 1044 } 1045 1046 static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) 1047 { 1048 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); 1049 } 1050 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1051 1052 static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) 1053 { 1054 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); 1055 } 1056 1057 static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) 1058 { 1059 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); 1060 } 1061 1062 static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) 1063 { 1064 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); 1065 } 1066 1067 static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) 1068 { 1069 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); 1070 } 1071 1072 static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 1073 unsigned int key_len) 1074 { 1075 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); 1076 } 1077 1078 static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, 1079 unsigned int key_len) 1080 { 1081 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); 1082 } 1083 1084 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1085 static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) 1086 { 1087 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); 1088 } 1089 1090 static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) 1091 { 1092 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); 1093 } 1094 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1095 1096 static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) 1097 { 1098 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); 1099 } 1100 1101 static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) 1102 { 1103 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); 1104 } 1105 1106 static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) 1107 { 1108 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); 1109 } 1110 1111 static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) 1112 { 1113 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); 1114 } 1115 1116 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1117 static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) 1118 { 1119 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); 1120 } 1121 1122 static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) 1123 { 1124 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); 1125 } 1126 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1127 1128 static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) 1129 { 1130 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); 1131 } 1132 1133 static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) 1134 { 1135 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); 1136 } 1137 1138 static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) 1139 { 1140 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); 1141 } 1142 1143 static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) 1144 { 1145 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); 1146 } 1147 1148 static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) 1149 { 1150 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 1151 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 1152 struct crypto_sync_skcipher *blk; 1153 1154 /* set fallback cipher in case it will be needed */ 1155 blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); 1156 if (IS_ERR(blk)) 1157 return PTR_ERR(blk); 1158 1159 tctx->sw_cipher.sk = blk; 1160 1161 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); 1162 1163 return 0; 1164 } 1165 1166 static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) 1167 { 1168 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); 1169 1170 return 0; 1171 } 1172 1173 static inline void clear_key(struct ocs_aes_tctx *tctx) 1174 { 1175 memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); 1176 1177 /* Zero key registers if set */ 1178 if (tctx->aes_dev) 1179 ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, 1180 tctx->key, OCS_AES); 1181 } 1182 1183 static void ocs_exit_tfm(struct crypto_skcipher *tfm) 1184 { 1185 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); 1186 1187 clear_key(tctx); 1188 1189 if (tctx->sw_cipher.sk) { 1190 crypto_free_sync_skcipher(tctx->sw_cipher.sk); 1191 tctx->sw_cipher.sk = NULL; 1192 } 1193 } 1194 1195 static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) 1196 { 1197 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 1198 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 1199 struct crypto_aead *blk; 1200 1201 /* Set fallback cipher in case it will be needed */ 1202 blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); 1203 if (IS_ERR(blk)) 1204 return PTR_ERR(blk); 1205 1206 tctx->sw_cipher.aead = blk; 1207 1208 crypto_aead_set_reqsize(tfm, 1209 max(sizeof(struct ocs_aes_rctx), 1210 (sizeof(struct aead_request) + 1211 crypto_aead_reqsize(tctx->sw_cipher.aead)))); 1212 1213 return 0; 1214 } 1215 1216 static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, 1217 unsigned int authsize) 1218 { 1219 switch (authsize) { 1220 case 4: 1221 case 6: 1222 case 8: 1223 case 10: 1224 case 12: 1225 case 14: 1226 case 16: 1227 return 0; 1228 default: 1229 return -EINVAL; 1230 } 1231 } 1232 1233 static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, 1234 unsigned int authsize) 1235 { 1236 return crypto_gcm_check_authsize(authsize); 1237 } 1238 1239 static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) 1240 { 1241 crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); 1242 1243 return 0; 1244 } 1245 1246 static void ocs_aead_cra_exit(struct crypto_aead *tfm) 1247 { 1248 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); 1249 1250 clear_key(tctx); 1251 1252 if (tctx->sw_cipher.aead) { 1253 crypto_free_aead(tctx->sw_cipher.aead); 1254 tctx->sw_cipher.aead = NULL; 1255 } 1256 } 1257 1258 static struct skcipher_engine_alg algs[] = { 1259 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1260 { 1261 .base.base.cra_name = "ecb(aes)", 1262 .base.base.cra_driver_name = "ecb-aes-keembay-ocs", 1263 .base.base.cra_priority = KMB_OCS_PRIORITY, 1264 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1265 CRYPTO_ALG_KERN_DRIVER_ONLY | 1266 CRYPTO_ALG_NEED_FALLBACK, 1267 .base.base.cra_blocksize = AES_BLOCK_SIZE, 1268 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1269 .base.base.cra_module = THIS_MODULE, 1270 .base.base.cra_alignmask = 0, 1271 1272 .base.min_keysize = OCS_AES_MIN_KEY_SIZE, 1273 .base.max_keysize = OCS_AES_MAX_KEY_SIZE, 1274 .base.setkey = kmb_ocs_aes_set_key, 1275 .base.encrypt = kmb_ocs_aes_ecb_encrypt, 1276 .base.decrypt = kmb_ocs_aes_ecb_decrypt, 1277 .base.init = ocs_aes_init_tfm, 1278 .base.exit = ocs_exit_tfm, 1279 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1280 }, 1281 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1282 { 1283 .base.base.cra_name = "cbc(aes)", 1284 .base.base.cra_driver_name = "cbc-aes-keembay-ocs", 1285 .base.base.cra_priority = KMB_OCS_PRIORITY, 1286 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1287 CRYPTO_ALG_KERN_DRIVER_ONLY | 1288 CRYPTO_ALG_NEED_FALLBACK, 1289 .base.base.cra_blocksize = AES_BLOCK_SIZE, 1290 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1291 .base.base.cra_module = THIS_MODULE, 1292 .base.base.cra_alignmask = 0, 1293 1294 .base.min_keysize = OCS_AES_MIN_KEY_SIZE, 1295 .base.max_keysize = OCS_AES_MAX_KEY_SIZE, 1296 .base.ivsize = AES_BLOCK_SIZE, 1297 .base.setkey = kmb_ocs_aes_set_key, 1298 .base.encrypt = kmb_ocs_aes_cbc_encrypt, 1299 .base.decrypt = kmb_ocs_aes_cbc_decrypt, 1300 .base.init = ocs_aes_init_tfm, 1301 .base.exit = ocs_exit_tfm, 1302 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1303 }, 1304 { 1305 .base.base.cra_name = "ctr(aes)", 1306 .base.base.cra_driver_name = "ctr-aes-keembay-ocs", 1307 .base.base.cra_priority = KMB_OCS_PRIORITY, 1308 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1309 CRYPTO_ALG_KERN_DRIVER_ONLY | 1310 CRYPTO_ALG_NEED_FALLBACK, 1311 .base.base.cra_blocksize = 1, 1312 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1313 .base.base.cra_module = THIS_MODULE, 1314 .base.base.cra_alignmask = 0, 1315 1316 .base.min_keysize = OCS_AES_MIN_KEY_SIZE, 1317 .base.max_keysize = OCS_AES_MAX_KEY_SIZE, 1318 .base.ivsize = AES_BLOCK_SIZE, 1319 .base.setkey = kmb_ocs_aes_set_key, 1320 .base.encrypt = kmb_ocs_aes_ctr_encrypt, 1321 .base.decrypt = kmb_ocs_aes_ctr_decrypt, 1322 .base.init = ocs_aes_init_tfm, 1323 .base.exit = ocs_exit_tfm, 1324 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1325 }, 1326 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1327 { 1328 .base.base.cra_name = "cts(cbc(aes))", 1329 .base.base.cra_driver_name = "cts-aes-keembay-ocs", 1330 .base.base.cra_priority = KMB_OCS_PRIORITY, 1331 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1332 CRYPTO_ALG_KERN_DRIVER_ONLY | 1333 CRYPTO_ALG_NEED_FALLBACK, 1334 .base.base.cra_blocksize = AES_BLOCK_SIZE, 1335 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1336 .base.base.cra_module = THIS_MODULE, 1337 .base.base.cra_alignmask = 0, 1338 1339 .base.min_keysize = OCS_AES_MIN_KEY_SIZE, 1340 .base.max_keysize = OCS_AES_MAX_KEY_SIZE, 1341 .base.ivsize = AES_BLOCK_SIZE, 1342 .base.setkey = kmb_ocs_aes_set_key, 1343 .base.encrypt = kmb_ocs_aes_cts_encrypt, 1344 .base.decrypt = kmb_ocs_aes_cts_decrypt, 1345 .base.init = ocs_aes_init_tfm, 1346 .base.exit = ocs_exit_tfm, 1347 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1348 }, 1349 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1350 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1351 { 1352 .base.base.cra_name = "ecb(sm4)", 1353 .base.base.cra_driver_name = "ecb-sm4-keembay-ocs", 1354 .base.base.cra_priority = KMB_OCS_PRIORITY, 1355 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1356 CRYPTO_ALG_KERN_DRIVER_ONLY, 1357 .base.base.cra_blocksize = AES_BLOCK_SIZE, 1358 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1359 .base.base.cra_module = THIS_MODULE, 1360 .base.base.cra_alignmask = 0, 1361 1362 .base.min_keysize = OCS_SM4_KEY_SIZE, 1363 .base.max_keysize = OCS_SM4_KEY_SIZE, 1364 .base.setkey = kmb_ocs_sm4_set_key, 1365 .base.encrypt = kmb_ocs_sm4_ecb_encrypt, 1366 .base.decrypt = kmb_ocs_sm4_ecb_decrypt, 1367 .base.init = ocs_sm4_init_tfm, 1368 .base.exit = ocs_exit_tfm, 1369 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1370 }, 1371 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1372 { 1373 .base.base.cra_name = "cbc(sm4)", 1374 .base.base.cra_driver_name = "cbc-sm4-keembay-ocs", 1375 .base.base.cra_priority = KMB_OCS_PRIORITY, 1376 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1377 CRYPTO_ALG_KERN_DRIVER_ONLY, 1378 .base.base.cra_blocksize = AES_BLOCK_SIZE, 1379 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1380 .base.base.cra_module = THIS_MODULE, 1381 .base.base.cra_alignmask = 0, 1382 1383 .base.min_keysize = OCS_SM4_KEY_SIZE, 1384 .base.max_keysize = OCS_SM4_KEY_SIZE, 1385 .base.ivsize = AES_BLOCK_SIZE, 1386 .base.setkey = kmb_ocs_sm4_set_key, 1387 .base.encrypt = kmb_ocs_sm4_cbc_encrypt, 1388 .base.decrypt = kmb_ocs_sm4_cbc_decrypt, 1389 .base.init = ocs_sm4_init_tfm, 1390 .base.exit = ocs_exit_tfm, 1391 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1392 }, 1393 { 1394 .base.base.cra_name = "ctr(sm4)", 1395 .base.base.cra_driver_name = "ctr-sm4-keembay-ocs", 1396 .base.base.cra_priority = KMB_OCS_PRIORITY, 1397 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1398 CRYPTO_ALG_KERN_DRIVER_ONLY, 1399 .base.base.cra_blocksize = 1, 1400 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1401 .base.base.cra_module = THIS_MODULE, 1402 .base.base.cra_alignmask = 0, 1403 1404 .base.min_keysize = OCS_SM4_KEY_SIZE, 1405 .base.max_keysize = OCS_SM4_KEY_SIZE, 1406 .base.ivsize = AES_BLOCK_SIZE, 1407 .base.setkey = kmb_ocs_sm4_set_key, 1408 .base.encrypt = kmb_ocs_sm4_ctr_encrypt, 1409 .base.decrypt = kmb_ocs_sm4_ctr_decrypt, 1410 .base.init = ocs_sm4_init_tfm, 1411 .base.exit = ocs_exit_tfm, 1412 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1413 }, 1414 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1415 { 1416 .base.base.cra_name = "cts(cbc(sm4))", 1417 .base.base.cra_driver_name = "cts-sm4-keembay-ocs", 1418 .base.base.cra_priority = KMB_OCS_PRIORITY, 1419 .base.base.cra_flags = CRYPTO_ALG_ASYNC | 1420 CRYPTO_ALG_KERN_DRIVER_ONLY, 1421 .base.base.cra_blocksize = AES_BLOCK_SIZE, 1422 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx), 1423 .base.base.cra_module = THIS_MODULE, 1424 .base.base.cra_alignmask = 0, 1425 1426 .base.min_keysize = OCS_SM4_KEY_SIZE, 1427 .base.max_keysize = OCS_SM4_KEY_SIZE, 1428 .base.ivsize = AES_BLOCK_SIZE, 1429 .base.setkey = kmb_ocs_sm4_set_key, 1430 .base.encrypt = kmb_ocs_sm4_cts_encrypt, 1431 .base.decrypt = kmb_ocs_sm4_cts_decrypt, 1432 .base.init = ocs_sm4_init_tfm, 1433 .base.exit = ocs_exit_tfm, 1434 .op.do_one_request = kmb_ocs_aes_sk_do_one_request, 1435 } 1436 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1437 }; 1438 1439 static struct aead_engine_alg algs_aead[] = { 1440 { 1441 .base.base = { 1442 .cra_name = "gcm(aes)", 1443 .cra_driver_name = "gcm-aes-keembay-ocs", 1444 .cra_priority = KMB_OCS_PRIORITY, 1445 .cra_flags = CRYPTO_ALG_ASYNC | 1446 CRYPTO_ALG_KERN_DRIVER_ONLY | 1447 CRYPTO_ALG_NEED_FALLBACK, 1448 .cra_blocksize = 1, 1449 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1450 .cra_alignmask = 0, 1451 .cra_module = THIS_MODULE, 1452 }, 1453 .base.init = ocs_aes_aead_cra_init, 1454 .base.exit = ocs_aead_cra_exit, 1455 .base.ivsize = GCM_AES_IV_SIZE, 1456 .base.maxauthsize = AES_BLOCK_SIZE, 1457 .base.setauthsize = kmb_ocs_aead_gcm_setauthsize, 1458 .base.setkey = kmb_ocs_aes_aead_set_key, 1459 .base.encrypt = kmb_ocs_aes_gcm_encrypt, 1460 .base.decrypt = kmb_ocs_aes_gcm_decrypt, 1461 .op.do_one_request = kmb_ocs_aes_aead_do_one_request, 1462 }, 1463 { 1464 .base.base = { 1465 .cra_name = "ccm(aes)", 1466 .cra_driver_name = "ccm-aes-keembay-ocs", 1467 .cra_priority = KMB_OCS_PRIORITY, 1468 .cra_flags = CRYPTO_ALG_ASYNC | 1469 CRYPTO_ALG_KERN_DRIVER_ONLY | 1470 CRYPTO_ALG_NEED_FALLBACK, 1471 .cra_blocksize = 1, 1472 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1473 .cra_alignmask = 0, 1474 .cra_module = THIS_MODULE, 1475 }, 1476 .base.init = ocs_aes_aead_cra_init, 1477 .base.exit = ocs_aead_cra_exit, 1478 .base.ivsize = AES_BLOCK_SIZE, 1479 .base.maxauthsize = AES_BLOCK_SIZE, 1480 .base.setauthsize = kmb_ocs_aead_ccm_setauthsize, 1481 .base.setkey = kmb_ocs_aes_aead_set_key, 1482 .base.encrypt = kmb_ocs_aes_ccm_encrypt, 1483 .base.decrypt = kmb_ocs_aes_ccm_decrypt, 1484 .op.do_one_request = kmb_ocs_aes_aead_do_one_request, 1485 }, 1486 { 1487 .base.base = { 1488 .cra_name = "gcm(sm4)", 1489 .cra_driver_name = "gcm-sm4-keembay-ocs", 1490 .cra_priority = KMB_OCS_PRIORITY, 1491 .cra_flags = CRYPTO_ALG_ASYNC | 1492 CRYPTO_ALG_KERN_DRIVER_ONLY, 1493 .cra_blocksize = 1, 1494 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1495 .cra_alignmask = 0, 1496 .cra_module = THIS_MODULE, 1497 }, 1498 .base.init = ocs_sm4_aead_cra_init, 1499 .base.exit = ocs_aead_cra_exit, 1500 .base.ivsize = GCM_AES_IV_SIZE, 1501 .base.maxauthsize = AES_BLOCK_SIZE, 1502 .base.setauthsize = kmb_ocs_aead_gcm_setauthsize, 1503 .base.setkey = kmb_ocs_sm4_aead_set_key, 1504 .base.encrypt = kmb_ocs_sm4_gcm_encrypt, 1505 .base.decrypt = kmb_ocs_sm4_gcm_decrypt, 1506 .op.do_one_request = kmb_ocs_aes_aead_do_one_request, 1507 }, 1508 { 1509 .base.base = { 1510 .cra_name = "ccm(sm4)", 1511 .cra_driver_name = "ccm-sm4-keembay-ocs", 1512 .cra_priority = KMB_OCS_PRIORITY, 1513 .cra_flags = CRYPTO_ALG_ASYNC | 1514 CRYPTO_ALG_KERN_DRIVER_ONLY, 1515 .cra_blocksize = 1, 1516 .cra_ctxsize = sizeof(struct ocs_aes_tctx), 1517 .cra_alignmask = 0, 1518 .cra_module = THIS_MODULE, 1519 }, 1520 .base.init = ocs_sm4_aead_cra_init, 1521 .base.exit = ocs_aead_cra_exit, 1522 .base.ivsize = AES_BLOCK_SIZE, 1523 .base.maxauthsize = AES_BLOCK_SIZE, 1524 .base.setauthsize = kmb_ocs_aead_ccm_setauthsize, 1525 .base.setkey = kmb_ocs_sm4_aead_set_key, 1526 .base.encrypt = kmb_ocs_sm4_ccm_encrypt, 1527 .base.decrypt = kmb_ocs_sm4_ccm_decrypt, 1528 .op.do_one_request = kmb_ocs_aes_aead_do_one_request, 1529 } 1530 }; 1531 1532 static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) 1533 { 1534 crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); 1535 crypto_engine_unregister_skciphers(algs, ARRAY_SIZE(algs)); 1536 } 1537 1538 static int register_aes_algs(struct ocs_aes_dev *aes_dev) 1539 { 1540 int ret; 1541 1542 /* 1543 * If any algorithm fails to register, all preceding algorithms that 1544 * were successfully registered will be automatically unregistered. 1545 */ 1546 ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); 1547 if (ret) 1548 return ret; 1549 1550 ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs)); 1551 if (ret) 1552 crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); 1553 1554 return ret; 1555 } 1556 1557 /* Device tree driver match. */ 1558 static const struct of_device_id kmb_ocs_aes_of_match[] = { 1559 { 1560 .compatible = "intel,keembay-ocs-aes", 1561 }, 1562 {} 1563 }; 1564 1565 static int kmb_ocs_aes_remove(struct platform_device *pdev) 1566 { 1567 struct ocs_aes_dev *aes_dev; 1568 1569 aes_dev = platform_get_drvdata(pdev); 1570 1571 unregister_aes_algs(aes_dev); 1572 1573 spin_lock(&ocs_aes.lock); 1574 list_del(&aes_dev->list); 1575 spin_unlock(&ocs_aes.lock); 1576 1577 crypto_engine_exit(aes_dev->engine); 1578 1579 return 0; 1580 } 1581 1582 static int kmb_ocs_aes_probe(struct platform_device *pdev) 1583 { 1584 struct device *dev = &pdev->dev; 1585 struct ocs_aes_dev *aes_dev; 1586 int rc; 1587 1588 aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); 1589 if (!aes_dev) 1590 return -ENOMEM; 1591 1592 aes_dev->dev = dev; 1593 1594 platform_set_drvdata(pdev, aes_dev); 1595 1596 rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1597 if (rc) { 1598 dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); 1599 return rc; 1600 } 1601 1602 /* Get base register address. */ 1603 aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); 1604 if (IS_ERR(aes_dev->base_reg)) 1605 return PTR_ERR(aes_dev->base_reg); 1606 1607 /* Get and request IRQ */ 1608 aes_dev->irq = platform_get_irq(pdev, 0); 1609 if (aes_dev->irq < 0) 1610 return aes_dev->irq; 1611 1612 rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, 1613 NULL, 0, "keembay-ocs-aes", aes_dev); 1614 if (rc < 0) { 1615 dev_err(dev, "Could not request IRQ\n"); 1616 return rc; 1617 } 1618 1619 INIT_LIST_HEAD(&aes_dev->list); 1620 spin_lock(&ocs_aes.lock); 1621 list_add_tail(&aes_dev->list, &ocs_aes.dev_list); 1622 spin_unlock(&ocs_aes.lock); 1623 1624 init_completion(&aes_dev->irq_completion); 1625 1626 /* Initialize crypto engine */ 1627 aes_dev->engine = crypto_engine_alloc_init(dev, true); 1628 if (!aes_dev->engine) { 1629 rc = -ENOMEM; 1630 goto list_del; 1631 } 1632 1633 rc = crypto_engine_start(aes_dev->engine); 1634 if (rc) { 1635 dev_err(dev, "Could not start crypto engine\n"); 1636 goto cleanup; 1637 } 1638 1639 rc = register_aes_algs(aes_dev); 1640 if (rc) { 1641 dev_err(dev, 1642 "Could not register OCS algorithms with Crypto API\n"); 1643 goto cleanup; 1644 } 1645 1646 return 0; 1647 1648 cleanup: 1649 crypto_engine_exit(aes_dev->engine); 1650 list_del: 1651 spin_lock(&ocs_aes.lock); 1652 list_del(&aes_dev->list); 1653 spin_unlock(&ocs_aes.lock); 1654 1655 return rc; 1656 } 1657 1658 /* The OCS driver is a platform device. */ 1659 static struct platform_driver kmb_ocs_aes_driver = { 1660 .probe = kmb_ocs_aes_probe, 1661 .remove = kmb_ocs_aes_remove, 1662 .driver = { 1663 .name = DRV_NAME, 1664 .of_match_table = kmb_ocs_aes_of_match, 1665 }, 1666 }; 1667 1668 module_platform_driver(kmb_ocs_aes_driver); 1669 1670 MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); 1671 MODULE_LICENSE("GPL"); 1672 1673 MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); 1674 MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); 1675 MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); 1676 MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); 1677 1678 MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); 1679 MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); 1680 MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); 1681 MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); 1682 1683 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB 1684 MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); 1685 MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); 1686 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ 1687 1688 #ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS 1689 MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); 1690 MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); 1691 #endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ 1692