1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sun8i-ce-core.c - hardware cryptographic offloader for 4 * Allwinner H3/A64/H5/H2+/H6/R40 SoC 5 * 6 * Copyright (C) 2015-2019 Corentin Labbe <clabbe.montjoie@gmail.com> 7 * 8 * Core file which registers crypto algorithms supported by the CryptoEngine. 9 * 10 * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst 11 */ 12 13 #include <crypto/engine.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/internal/rng.h> 16 #include <crypto/internal/skcipher.h> 17 #include <linux/clk.h> 18 #include <linux/delay.h> 19 #include <linux/dma-mapping.h> 20 #include <linux/err.h> 21 #include <linux/interrupt.h> 22 #include <linux/io.h> 23 #include <linux/irq.h> 24 #include <linux/kernel.h> 25 #include <linux/module.h> 26 #include <linux/of.h> 27 #include <linux/platform_device.h> 28 #include <linux/pm_runtime.h> 29 #include <linux/reset.h> 30 31 #include "sun8i-ce.h" 32 33 /* 34 * mod clock is lower on H3 than other SoC due to some DMA timeout occurring 35 * with high value. 36 * If you want to tune mod clock, loading driver and passing selftest is 37 * insufficient, you need to test with some LUKS test (mount and write to it) 38 */ 39 static const struct ce_variant ce_h3_variant = { 40 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 41 }, 42 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 43 CE_ALG_SHA384, CE_ALG_SHA512 44 }, 45 .op_mode = { CE_OP_ECB, CE_OP_CBC 46 }, 47 .ce_clks = { 48 { "bus", 0, 200000000 }, 49 { "mod", 50000000, 0 }, 50 }, 51 .esr = ESR_H3, 52 .prng = CE_ALG_PRNG, 53 .trng = CE_ID_NOTSUPP, 54 }; 55 56 static const struct ce_variant ce_h5_variant = { 57 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 58 }, 59 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 60 CE_ID_NOTSUPP, CE_ID_NOTSUPP 61 }, 62 .op_mode = { CE_OP_ECB, CE_OP_CBC 63 }, 64 .ce_clks = { 65 { "bus", 0, 200000000 }, 66 { "mod", 300000000, 0 }, 67 }, 68 .esr = ESR_H5, 69 .prng = CE_ALG_PRNG, 70 .trng = CE_ID_NOTSUPP, 71 }; 72 73 static const struct ce_variant ce_h6_variant = { 74 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 75 }, 76 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 77 CE_ALG_SHA384, CE_ALG_SHA512 78 }, 79 .op_mode = { CE_OP_ECB, CE_OP_CBC 80 }, 81 .cipher_t_dlen_in_bytes = true, 82 .hash_t_dlen_in_bits = true, 83 .prng_t_dlen_in_bytes = true, 84 .trng_t_dlen_in_bytes = true, 85 .ce_clks = { 86 { "bus", 0, 200000000 }, 87 { "mod", 300000000, 0 }, 88 { "ram", 0, 400000000 }, 89 }, 90 .esr = ESR_H6, 91 .prng = CE_ALG_PRNG_V2, 92 .trng = CE_ALG_TRNG_V2, 93 }; 94 95 static const struct ce_variant ce_a64_variant = { 96 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 97 }, 98 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 99 CE_ID_NOTSUPP, CE_ID_NOTSUPP 100 }, 101 .op_mode = { CE_OP_ECB, CE_OP_CBC 102 }, 103 .ce_clks = { 104 { "bus", 0, 200000000 }, 105 { "mod", 300000000, 0 }, 106 }, 107 .esr = ESR_A64, 108 .prng = CE_ALG_PRNG, 109 .trng = CE_ID_NOTSUPP, 110 }; 111 112 static const struct ce_variant ce_d1_variant = { 113 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 114 }, 115 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 116 CE_ALG_SHA384, CE_ALG_SHA512 117 }, 118 .op_mode = { CE_OP_ECB, CE_OP_CBC 119 }, 120 .ce_clks = { 121 { "bus", 0, 200000000 }, 122 { "mod", 300000000, 0 }, 123 { "ram", 0, 400000000 }, 124 { "trng", 0, 0 }, 125 }, 126 .esr = ESR_D1, 127 .prng = CE_ALG_PRNG, 128 .trng = CE_ALG_TRNG, 129 }; 130 131 static const struct ce_variant ce_r40_variant = { 132 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 133 }, 134 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 135 CE_ID_NOTSUPP, CE_ID_NOTSUPP 136 }, 137 .op_mode = { CE_OP_ECB, CE_OP_CBC 138 }, 139 .ce_clks = { 140 { "bus", 0, 200000000 }, 141 { "mod", 300000000, 0 }, 142 }, 143 .esr = ESR_R40, 144 .prng = CE_ALG_PRNG, 145 .trng = CE_ID_NOTSUPP, 146 }; 147 148 /* 149 * sun8i_ce_get_engine_number() get the next channel slot 150 * This is a simple round-robin way of getting the next channel 151 * The flow 3 is reserve for xRNG operations 152 */ 153 int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce) 154 { 155 return atomic_inc_return(&ce->flow) % (MAXFLOW - 1); 156 } 157 158 int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) 159 { 160 u32 v; 161 int err = 0; 162 struct ce_task *cet = ce->chanlist[flow].tl; 163 164 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 165 ce->chanlist[flow].stat_req++; 166 #endif 167 168 mutex_lock(&ce->mlock); 169 170 v = readl(ce->base + CE_ICR); 171 v |= 1 << flow; 172 writel(v, ce->base + CE_ICR); 173 174 reinit_completion(&ce->chanlist[flow].complete); 175 writel(ce->chanlist[flow].t_phy, ce->base + CE_TDQ); 176 177 ce->chanlist[flow].status = 0; 178 /* Be sure all data is written before enabling the task */ 179 wmb(); 180 181 /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored 182 * on older SoCs, we have no reason to complicate things. 183 */ 184 v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8); 185 writel(v, ce->base + CE_TLR); 186 mutex_unlock(&ce->mlock); 187 188 wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete, 189 msecs_to_jiffies(ce->chanlist[flow].timeout)); 190 191 if (ce->chanlist[flow].status == 0) { 192 dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, 193 ce->chanlist[flow].timeout, flow); 194 err = -EFAULT; 195 } 196 /* No need to lock for this read, the channel is locked so 197 * nothing could modify the error value for this channel 198 */ 199 v = readl(ce->base + CE_ESR); 200 switch (ce->variant->esr) { 201 case ESR_H3: 202 /* Sadly, the error bit is not per flow */ 203 if (v) { 204 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); 205 err = -EFAULT; 206 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, 207 cet, sizeof(struct ce_task), false); 208 } 209 if (v & CE_ERR_ALGO_NOTSUP) 210 dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); 211 if (v & CE_ERR_DATALEN) 212 dev_err(ce->dev, "CE ERROR: data length error\n"); 213 if (v & CE_ERR_KEYSRAM) 214 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); 215 break; 216 case ESR_A64: 217 case ESR_D1: 218 case ESR_H5: 219 case ESR_R40: 220 v >>= (flow * 4); 221 v &= 0xF; 222 if (v) { 223 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); 224 err = -EFAULT; 225 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, 226 cet, sizeof(struct ce_task), false); 227 } 228 if (v & CE_ERR_ALGO_NOTSUP) 229 dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); 230 if (v & CE_ERR_DATALEN) 231 dev_err(ce->dev, "CE ERROR: data length error\n"); 232 if (v & CE_ERR_KEYSRAM) 233 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); 234 break; 235 case ESR_H6: 236 v >>= (flow * 8); 237 v &= 0xFF; 238 if (v) { 239 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); 240 err = -EFAULT; 241 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, 242 cet, sizeof(struct ce_task), false); 243 } 244 if (v & CE_ERR_ALGO_NOTSUP) 245 dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); 246 if (v & CE_ERR_DATALEN) 247 dev_err(ce->dev, "CE ERROR: data length error\n"); 248 if (v & CE_ERR_KEYSRAM) 249 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); 250 if (v & CE_ERR_ADDR_INVALID) 251 dev_err(ce->dev, "CE ERROR: address invalid\n"); 252 if (v & CE_ERR_KEYLADDER) 253 dev_err(ce->dev, "CE ERROR: key ladder configuration error\n"); 254 break; 255 } 256 257 return err; 258 } 259 260 static irqreturn_t ce_irq_handler(int irq, void *data) 261 { 262 struct sun8i_ce_dev *ce = (struct sun8i_ce_dev *)data; 263 int flow = 0; 264 u32 p; 265 266 p = readl(ce->base + CE_ISR); 267 for (flow = 0; flow < MAXFLOW; flow++) { 268 if (p & (BIT(flow))) { 269 writel(BIT(flow), ce->base + CE_ISR); 270 ce->chanlist[flow].status = 1; 271 complete(&ce->chanlist[flow].complete); 272 } 273 } 274 275 return IRQ_HANDLED; 276 } 277 278 static struct sun8i_ce_alg_template ce_algs[] = { 279 { 280 .type = CRYPTO_ALG_TYPE_SKCIPHER, 281 .ce_algo_id = CE_ID_CIPHER_AES, 282 .ce_blockmode = CE_ID_OP_CBC, 283 .alg.skcipher.base = { 284 .base = { 285 .cra_name = "cbc(aes)", 286 .cra_driver_name = "cbc-aes-sun8i-ce", 287 .cra_priority = 400, 288 .cra_blocksize = AES_BLOCK_SIZE, 289 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 290 CRYPTO_ALG_ASYNC | 291 CRYPTO_ALG_NEED_FALLBACK, 292 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 293 .cra_module = THIS_MODULE, 294 .cra_alignmask = 0xf, 295 .cra_init = sun8i_ce_cipher_init, 296 .cra_exit = sun8i_ce_cipher_exit, 297 }, 298 .min_keysize = AES_MIN_KEY_SIZE, 299 .max_keysize = AES_MAX_KEY_SIZE, 300 .ivsize = AES_BLOCK_SIZE, 301 .setkey = sun8i_ce_aes_setkey, 302 .encrypt = sun8i_ce_skencrypt, 303 .decrypt = sun8i_ce_skdecrypt, 304 }, 305 .alg.skcipher.op = { 306 .do_one_request = sun8i_ce_cipher_do_one, 307 }, 308 }, 309 { 310 .type = CRYPTO_ALG_TYPE_SKCIPHER, 311 .ce_algo_id = CE_ID_CIPHER_AES, 312 .ce_blockmode = CE_ID_OP_ECB, 313 .alg.skcipher.base = { 314 .base = { 315 .cra_name = "ecb(aes)", 316 .cra_driver_name = "ecb-aes-sun8i-ce", 317 .cra_priority = 400, 318 .cra_blocksize = AES_BLOCK_SIZE, 319 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 320 CRYPTO_ALG_ASYNC | 321 CRYPTO_ALG_NEED_FALLBACK, 322 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 323 .cra_module = THIS_MODULE, 324 .cra_alignmask = 0xf, 325 .cra_init = sun8i_ce_cipher_init, 326 .cra_exit = sun8i_ce_cipher_exit, 327 }, 328 .min_keysize = AES_MIN_KEY_SIZE, 329 .max_keysize = AES_MAX_KEY_SIZE, 330 .setkey = sun8i_ce_aes_setkey, 331 .encrypt = sun8i_ce_skencrypt, 332 .decrypt = sun8i_ce_skdecrypt, 333 }, 334 .alg.skcipher.op = { 335 .do_one_request = sun8i_ce_cipher_do_one, 336 }, 337 }, 338 { 339 .type = CRYPTO_ALG_TYPE_SKCIPHER, 340 .ce_algo_id = CE_ID_CIPHER_DES3, 341 .ce_blockmode = CE_ID_OP_CBC, 342 .alg.skcipher.base = { 343 .base = { 344 .cra_name = "cbc(des3_ede)", 345 .cra_driver_name = "cbc-des3-sun8i-ce", 346 .cra_priority = 400, 347 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 348 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 349 CRYPTO_ALG_ASYNC | 350 CRYPTO_ALG_NEED_FALLBACK, 351 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 352 .cra_module = THIS_MODULE, 353 .cra_alignmask = 0xf, 354 .cra_init = sun8i_ce_cipher_init, 355 .cra_exit = sun8i_ce_cipher_exit, 356 }, 357 .min_keysize = DES3_EDE_KEY_SIZE, 358 .max_keysize = DES3_EDE_KEY_SIZE, 359 .ivsize = DES3_EDE_BLOCK_SIZE, 360 .setkey = sun8i_ce_des3_setkey, 361 .encrypt = sun8i_ce_skencrypt, 362 .decrypt = sun8i_ce_skdecrypt, 363 }, 364 .alg.skcipher.op = { 365 .do_one_request = sun8i_ce_cipher_do_one, 366 }, 367 }, 368 { 369 .type = CRYPTO_ALG_TYPE_SKCIPHER, 370 .ce_algo_id = CE_ID_CIPHER_DES3, 371 .ce_blockmode = CE_ID_OP_ECB, 372 .alg.skcipher.base = { 373 .base = { 374 .cra_name = "ecb(des3_ede)", 375 .cra_driver_name = "ecb-des3-sun8i-ce", 376 .cra_priority = 400, 377 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 378 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 379 CRYPTO_ALG_ASYNC | 380 CRYPTO_ALG_NEED_FALLBACK, 381 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 382 .cra_module = THIS_MODULE, 383 .cra_alignmask = 0xf, 384 .cra_init = sun8i_ce_cipher_init, 385 .cra_exit = sun8i_ce_cipher_exit, 386 }, 387 .min_keysize = DES3_EDE_KEY_SIZE, 388 .max_keysize = DES3_EDE_KEY_SIZE, 389 .setkey = sun8i_ce_des3_setkey, 390 .encrypt = sun8i_ce_skencrypt, 391 .decrypt = sun8i_ce_skdecrypt, 392 }, 393 .alg.skcipher.op = { 394 .do_one_request = sun8i_ce_cipher_do_one, 395 }, 396 }, 397 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH 398 { .type = CRYPTO_ALG_TYPE_AHASH, 399 .ce_algo_id = CE_ID_HASH_MD5, 400 .alg.hash.base = { 401 .init = sun8i_ce_hash_init, 402 .update = sun8i_ce_hash_update, 403 .final = sun8i_ce_hash_final, 404 .finup = sun8i_ce_hash_finup, 405 .digest = sun8i_ce_hash_digest, 406 .export = sun8i_ce_hash_export, 407 .import = sun8i_ce_hash_import, 408 .init_tfm = sun8i_ce_hash_init_tfm, 409 .exit_tfm = sun8i_ce_hash_exit_tfm, 410 .halg = { 411 .digestsize = MD5_DIGEST_SIZE, 412 .statesize = sizeof(struct md5_state), 413 .base = { 414 .cra_name = "md5", 415 .cra_driver_name = "md5-sun8i-ce", 416 .cra_priority = 300, 417 .cra_alignmask = 3, 418 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 419 CRYPTO_ALG_ASYNC | 420 CRYPTO_ALG_NEED_FALLBACK, 421 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 422 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 423 .cra_module = THIS_MODULE, 424 } 425 } 426 }, 427 .alg.hash.op = { 428 .do_one_request = sun8i_ce_hash_run, 429 }, 430 431 }, 432 { .type = CRYPTO_ALG_TYPE_AHASH, 433 .ce_algo_id = CE_ID_HASH_SHA1, 434 .alg.hash.base = { 435 .init = sun8i_ce_hash_init, 436 .update = sun8i_ce_hash_update, 437 .final = sun8i_ce_hash_final, 438 .finup = sun8i_ce_hash_finup, 439 .digest = sun8i_ce_hash_digest, 440 .export = sun8i_ce_hash_export, 441 .import = sun8i_ce_hash_import, 442 .init_tfm = sun8i_ce_hash_init_tfm, 443 .exit_tfm = sun8i_ce_hash_exit_tfm, 444 .halg = { 445 .digestsize = SHA1_DIGEST_SIZE, 446 .statesize = sizeof(struct sha1_state), 447 .base = { 448 .cra_name = "sha1", 449 .cra_driver_name = "sha1-sun8i-ce", 450 .cra_priority = 300, 451 .cra_alignmask = 3, 452 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 453 CRYPTO_ALG_ASYNC | 454 CRYPTO_ALG_NEED_FALLBACK, 455 .cra_blocksize = SHA1_BLOCK_SIZE, 456 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 457 .cra_module = THIS_MODULE, 458 } 459 } 460 }, 461 .alg.hash.op = { 462 .do_one_request = sun8i_ce_hash_run, 463 }, 464 }, 465 { .type = CRYPTO_ALG_TYPE_AHASH, 466 .ce_algo_id = CE_ID_HASH_SHA224, 467 .alg.hash.base = { 468 .init = sun8i_ce_hash_init, 469 .update = sun8i_ce_hash_update, 470 .final = sun8i_ce_hash_final, 471 .finup = sun8i_ce_hash_finup, 472 .digest = sun8i_ce_hash_digest, 473 .export = sun8i_ce_hash_export, 474 .import = sun8i_ce_hash_import, 475 .init_tfm = sun8i_ce_hash_init_tfm, 476 .exit_tfm = sun8i_ce_hash_exit_tfm, 477 .halg = { 478 .digestsize = SHA224_DIGEST_SIZE, 479 .statesize = sizeof(struct sha256_state), 480 .base = { 481 .cra_name = "sha224", 482 .cra_driver_name = "sha224-sun8i-ce", 483 .cra_priority = 300, 484 .cra_alignmask = 3, 485 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 486 CRYPTO_ALG_ASYNC | 487 CRYPTO_ALG_NEED_FALLBACK, 488 .cra_blocksize = SHA224_BLOCK_SIZE, 489 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 490 .cra_module = THIS_MODULE, 491 } 492 } 493 }, 494 .alg.hash.op = { 495 .do_one_request = sun8i_ce_hash_run, 496 }, 497 }, 498 { .type = CRYPTO_ALG_TYPE_AHASH, 499 .ce_algo_id = CE_ID_HASH_SHA256, 500 .alg.hash.base = { 501 .init = sun8i_ce_hash_init, 502 .update = sun8i_ce_hash_update, 503 .final = sun8i_ce_hash_final, 504 .finup = sun8i_ce_hash_finup, 505 .digest = sun8i_ce_hash_digest, 506 .export = sun8i_ce_hash_export, 507 .import = sun8i_ce_hash_import, 508 .init_tfm = sun8i_ce_hash_init_tfm, 509 .exit_tfm = sun8i_ce_hash_exit_tfm, 510 .halg = { 511 .digestsize = SHA256_DIGEST_SIZE, 512 .statesize = sizeof(struct sha256_state), 513 .base = { 514 .cra_name = "sha256", 515 .cra_driver_name = "sha256-sun8i-ce", 516 .cra_priority = 300, 517 .cra_alignmask = 3, 518 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 519 CRYPTO_ALG_ASYNC | 520 CRYPTO_ALG_NEED_FALLBACK, 521 .cra_blocksize = SHA256_BLOCK_SIZE, 522 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 523 .cra_module = THIS_MODULE, 524 } 525 } 526 }, 527 .alg.hash.op = { 528 .do_one_request = sun8i_ce_hash_run, 529 }, 530 }, 531 { .type = CRYPTO_ALG_TYPE_AHASH, 532 .ce_algo_id = CE_ID_HASH_SHA384, 533 .alg.hash.base = { 534 .init = sun8i_ce_hash_init, 535 .update = sun8i_ce_hash_update, 536 .final = sun8i_ce_hash_final, 537 .finup = sun8i_ce_hash_finup, 538 .digest = sun8i_ce_hash_digest, 539 .export = sun8i_ce_hash_export, 540 .import = sun8i_ce_hash_import, 541 .init_tfm = sun8i_ce_hash_init_tfm, 542 .exit_tfm = sun8i_ce_hash_exit_tfm, 543 .halg = { 544 .digestsize = SHA384_DIGEST_SIZE, 545 .statesize = sizeof(struct sha512_state), 546 .base = { 547 .cra_name = "sha384", 548 .cra_driver_name = "sha384-sun8i-ce", 549 .cra_priority = 300, 550 .cra_alignmask = 3, 551 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 552 CRYPTO_ALG_ASYNC | 553 CRYPTO_ALG_NEED_FALLBACK, 554 .cra_blocksize = SHA384_BLOCK_SIZE, 555 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 556 .cra_module = THIS_MODULE, 557 } 558 } 559 }, 560 .alg.hash.op = { 561 .do_one_request = sun8i_ce_hash_run, 562 }, 563 }, 564 { .type = CRYPTO_ALG_TYPE_AHASH, 565 .ce_algo_id = CE_ID_HASH_SHA512, 566 .alg.hash.base = { 567 .init = sun8i_ce_hash_init, 568 .update = sun8i_ce_hash_update, 569 .final = sun8i_ce_hash_final, 570 .finup = sun8i_ce_hash_finup, 571 .digest = sun8i_ce_hash_digest, 572 .export = sun8i_ce_hash_export, 573 .import = sun8i_ce_hash_import, 574 .init_tfm = sun8i_ce_hash_init_tfm, 575 .exit_tfm = sun8i_ce_hash_exit_tfm, 576 .halg = { 577 .digestsize = SHA512_DIGEST_SIZE, 578 .statesize = sizeof(struct sha512_state), 579 .base = { 580 .cra_name = "sha512", 581 .cra_driver_name = "sha512-sun8i-ce", 582 .cra_priority = 300, 583 .cra_alignmask = 3, 584 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 585 CRYPTO_ALG_ASYNC | 586 CRYPTO_ALG_NEED_FALLBACK, 587 .cra_blocksize = SHA512_BLOCK_SIZE, 588 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 589 .cra_module = THIS_MODULE, 590 } 591 } 592 }, 593 .alg.hash.op = { 594 .do_one_request = sun8i_ce_hash_run, 595 }, 596 }, 597 #endif 598 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG 599 { 600 .type = CRYPTO_ALG_TYPE_RNG, 601 .alg.rng = { 602 .base = { 603 .cra_name = "stdrng", 604 .cra_driver_name = "sun8i-ce-prng", 605 .cra_priority = 300, 606 .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx), 607 .cra_module = THIS_MODULE, 608 .cra_init = sun8i_ce_prng_init, 609 .cra_exit = sun8i_ce_prng_exit, 610 }, 611 .generate = sun8i_ce_prng_generate, 612 .seed = sun8i_ce_prng_seed, 613 .seedsize = PRNG_SEED_SIZE, 614 } 615 }, 616 #endif 617 }; 618 619 static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) 620 { 621 struct sun8i_ce_dev *ce __maybe_unused = seq->private; 622 unsigned int i; 623 624 for (i = 0; i < MAXFLOW; i++) 625 seq_printf(seq, "Channel %d: nreq %lu\n", i, 626 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 627 ce->chanlist[i].stat_req); 628 #else 629 0ul); 630 #endif 631 632 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 633 if (!ce_algs[i].ce) 634 continue; 635 switch (ce_algs[i].type) { 636 case CRYPTO_ALG_TYPE_SKCIPHER: 637 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", 638 ce_algs[i].alg.skcipher.base.base.cra_driver_name, 639 ce_algs[i].alg.skcipher.base.base.cra_name, 640 ce_algs[i].stat_req, ce_algs[i].stat_fb); 641 seq_printf(seq, "\tLast fallback is: %s\n", 642 ce_algs[i].fbname); 643 seq_printf(seq, "\tFallback due to 0 length: %lu\n", 644 ce_algs[i].stat_fb_len0); 645 seq_printf(seq, "\tFallback due to length !mod16: %lu\n", 646 ce_algs[i].stat_fb_mod16); 647 seq_printf(seq, "\tFallback due to length < IV: %lu\n", 648 ce_algs[i].stat_fb_leniv); 649 seq_printf(seq, "\tFallback due to source alignment: %lu\n", 650 ce_algs[i].stat_fb_srcali); 651 seq_printf(seq, "\tFallback due to dest alignment: %lu\n", 652 ce_algs[i].stat_fb_dstali); 653 seq_printf(seq, "\tFallback due to source length: %lu\n", 654 ce_algs[i].stat_fb_srclen); 655 seq_printf(seq, "\tFallback due to dest length: %lu\n", 656 ce_algs[i].stat_fb_dstlen); 657 seq_printf(seq, "\tFallback due to SG numbers: %lu\n", 658 ce_algs[i].stat_fb_maxsg); 659 break; 660 case CRYPTO_ALG_TYPE_AHASH: 661 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", 662 ce_algs[i].alg.hash.base.halg.base.cra_driver_name, 663 ce_algs[i].alg.hash.base.halg.base.cra_name, 664 ce_algs[i].stat_req, ce_algs[i].stat_fb); 665 seq_printf(seq, "\tLast fallback is: %s\n", 666 ce_algs[i].fbname); 667 seq_printf(seq, "\tFallback due to 0 length: %lu\n", 668 ce_algs[i].stat_fb_len0); 669 seq_printf(seq, "\tFallback due to length: %lu\n", 670 ce_algs[i].stat_fb_srclen); 671 seq_printf(seq, "\tFallback due to alignment: %lu\n", 672 ce_algs[i].stat_fb_srcali); 673 seq_printf(seq, "\tFallback due to SG numbers: %lu\n", 674 ce_algs[i].stat_fb_maxsg); 675 break; 676 case CRYPTO_ALG_TYPE_RNG: 677 seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n", 678 ce_algs[i].alg.rng.base.cra_driver_name, 679 ce_algs[i].alg.rng.base.cra_name, 680 ce_algs[i].stat_req, ce_algs[i].stat_bytes); 681 break; 682 } 683 } 684 #if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \ 685 defined(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG) 686 seq_printf(seq, "HWRNG %lu %lu\n", 687 ce->hwrng_stat_req, ce->hwrng_stat_bytes); 688 #endif 689 return 0; 690 } 691 692 DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs); 693 694 static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i) 695 { 696 while (i >= 0) { 697 crypto_engine_exit(ce->chanlist[i].engine); 698 if (ce->chanlist[i].tl) 699 dma_free_coherent(ce->dev, sizeof(struct ce_task), 700 ce->chanlist[i].tl, 701 ce->chanlist[i].t_phy); 702 i--; 703 } 704 } 705 706 /* 707 * Allocate the channel list structure 708 */ 709 static int sun8i_ce_allocate_chanlist(struct sun8i_ce_dev *ce) 710 { 711 int i, err; 712 713 ce->chanlist = devm_kcalloc(ce->dev, MAXFLOW, 714 sizeof(struct sun8i_ce_flow), GFP_KERNEL); 715 if (!ce->chanlist) 716 return -ENOMEM; 717 718 for (i = 0; i < MAXFLOW; i++) { 719 init_completion(&ce->chanlist[i].complete); 720 721 ce->chanlist[i].engine = crypto_engine_alloc_init(ce->dev, true); 722 if (!ce->chanlist[i].engine) { 723 dev_err(ce->dev, "Cannot allocate engine\n"); 724 i--; 725 err = -ENOMEM; 726 goto error_engine; 727 } 728 err = crypto_engine_start(ce->chanlist[i].engine); 729 if (err) { 730 dev_err(ce->dev, "Cannot start engine\n"); 731 goto error_engine; 732 } 733 ce->chanlist[i].tl = dma_alloc_coherent(ce->dev, 734 sizeof(struct ce_task), 735 &ce->chanlist[i].t_phy, 736 GFP_KERNEL); 737 if (!ce->chanlist[i].tl) { 738 dev_err(ce->dev, "Cannot get DMA memory for task %d\n", 739 i); 740 err = -ENOMEM; 741 goto error_engine; 742 } 743 ce->chanlist[i].bounce_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, 744 GFP_KERNEL | GFP_DMA); 745 if (!ce->chanlist[i].bounce_iv) { 746 err = -ENOMEM; 747 goto error_engine; 748 } 749 ce->chanlist[i].backup_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, 750 GFP_KERNEL); 751 if (!ce->chanlist[i].backup_iv) { 752 err = -ENOMEM; 753 goto error_engine; 754 } 755 } 756 return 0; 757 error_engine: 758 sun8i_ce_free_chanlist(ce, i); 759 return err; 760 } 761 762 /* 763 * Power management strategy: The device is suspended unless a TFM exists for 764 * one of the algorithms proposed by this driver. 765 */ 766 static int sun8i_ce_pm_suspend(struct device *dev) 767 { 768 struct sun8i_ce_dev *ce = dev_get_drvdata(dev); 769 int i; 770 771 reset_control_assert(ce->reset); 772 for (i = 0; i < CE_MAX_CLOCKS; i++) 773 clk_disable_unprepare(ce->ceclks[i]); 774 return 0; 775 } 776 777 static int sun8i_ce_pm_resume(struct device *dev) 778 { 779 struct sun8i_ce_dev *ce = dev_get_drvdata(dev); 780 int err, i; 781 782 for (i = 0; i < CE_MAX_CLOCKS; i++) { 783 if (!ce->variant->ce_clks[i].name) 784 continue; 785 err = clk_prepare_enable(ce->ceclks[i]); 786 if (err) { 787 dev_err(ce->dev, "Cannot prepare_enable %s\n", 788 ce->variant->ce_clks[i].name); 789 goto error; 790 } 791 } 792 err = reset_control_deassert(ce->reset); 793 if (err) { 794 dev_err(ce->dev, "Cannot deassert reset control\n"); 795 goto error; 796 } 797 return 0; 798 error: 799 sun8i_ce_pm_suspend(dev); 800 return err; 801 } 802 803 static const struct dev_pm_ops sun8i_ce_pm_ops = { 804 SET_RUNTIME_PM_OPS(sun8i_ce_pm_suspend, sun8i_ce_pm_resume, NULL) 805 }; 806 807 static int sun8i_ce_pm_init(struct sun8i_ce_dev *ce) 808 { 809 int err; 810 811 pm_runtime_use_autosuspend(ce->dev); 812 pm_runtime_set_autosuspend_delay(ce->dev, 2000); 813 814 err = pm_runtime_set_suspended(ce->dev); 815 if (err) 816 return err; 817 pm_runtime_enable(ce->dev); 818 return err; 819 } 820 821 static void sun8i_ce_pm_exit(struct sun8i_ce_dev *ce) 822 { 823 pm_runtime_disable(ce->dev); 824 } 825 826 static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) 827 { 828 unsigned long cr; 829 int err, i; 830 831 for (i = 0; i < CE_MAX_CLOCKS; i++) { 832 if (!ce->variant->ce_clks[i].name) 833 continue; 834 ce->ceclks[i] = devm_clk_get(ce->dev, ce->variant->ce_clks[i].name); 835 if (IS_ERR(ce->ceclks[i])) { 836 err = PTR_ERR(ce->ceclks[i]); 837 dev_err(ce->dev, "Cannot get %s CE clock err=%d\n", 838 ce->variant->ce_clks[i].name, err); 839 return err; 840 } 841 cr = clk_get_rate(ce->ceclks[i]); 842 if (!cr) 843 return -EINVAL; 844 if (ce->variant->ce_clks[i].freq > 0 && 845 cr != ce->variant->ce_clks[i].freq) { 846 dev_info(ce->dev, "Set %s clock to %lu (%lu Mhz) from %lu (%lu Mhz)\n", 847 ce->variant->ce_clks[i].name, 848 ce->variant->ce_clks[i].freq, 849 ce->variant->ce_clks[i].freq / 1000000, 850 cr, cr / 1000000); 851 err = clk_set_rate(ce->ceclks[i], ce->variant->ce_clks[i].freq); 852 if (err) 853 dev_err(ce->dev, "Fail to set %s clk speed to %lu hz\n", 854 ce->variant->ce_clks[i].name, 855 ce->variant->ce_clks[i].freq); 856 } 857 if (ce->variant->ce_clks[i].max_freq > 0 && 858 cr > ce->variant->ce_clks[i].max_freq) 859 dev_warn(ce->dev, "Frequency for %s (%lu hz) is higher than datasheet's recommendation (%lu hz)", 860 ce->variant->ce_clks[i].name, cr, 861 ce->variant->ce_clks[i].max_freq); 862 } 863 return 0; 864 } 865 866 static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) 867 { 868 int ce_method, err, id; 869 unsigned int i; 870 871 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 872 ce_algs[i].ce = ce; 873 switch (ce_algs[i].type) { 874 case CRYPTO_ALG_TYPE_SKCIPHER: 875 id = ce_algs[i].ce_algo_id; 876 ce_method = ce->variant->alg_cipher[id]; 877 if (ce_method == CE_ID_NOTSUPP) { 878 dev_dbg(ce->dev, 879 "DEBUG: Algo of %s not supported\n", 880 ce_algs[i].alg.skcipher.base.base.cra_name); 881 ce_algs[i].ce = NULL; 882 break; 883 } 884 id = ce_algs[i].ce_blockmode; 885 ce_method = ce->variant->op_mode[id]; 886 if (ce_method == CE_ID_NOTSUPP) { 887 dev_dbg(ce->dev, "DEBUG: Blockmode of %s not supported\n", 888 ce_algs[i].alg.skcipher.base.base.cra_name); 889 ce_algs[i].ce = NULL; 890 break; 891 } 892 dev_info(ce->dev, "Register %s\n", 893 ce_algs[i].alg.skcipher.base.base.cra_name); 894 err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher); 895 if (err) { 896 dev_err(ce->dev, "ERROR: Fail to register %s\n", 897 ce_algs[i].alg.skcipher.base.base.cra_name); 898 ce_algs[i].ce = NULL; 899 return err; 900 } 901 break; 902 case CRYPTO_ALG_TYPE_AHASH: 903 id = ce_algs[i].ce_algo_id; 904 ce_method = ce->variant->alg_hash[id]; 905 if (ce_method == CE_ID_NOTSUPP) { 906 dev_info(ce->dev, 907 "DEBUG: Algo of %s not supported\n", 908 ce_algs[i].alg.hash.base.halg.base.cra_name); 909 ce_algs[i].ce = NULL; 910 break; 911 } 912 dev_info(ce->dev, "Register %s\n", 913 ce_algs[i].alg.hash.base.halg.base.cra_name); 914 err = crypto_engine_register_ahash(&ce_algs[i].alg.hash); 915 if (err) { 916 dev_err(ce->dev, "ERROR: Fail to register %s\n", 917 ce_algs[i].alg.hash.base.halg.base.cra_name); 918 ce_algs[i].ce = NULL; 919 return err; 920 } 921 break; 922 case CRYPTO_ALG_TYPE_RNG: 923 if (ce->variant->prng == CE_ID_NOTSUPP) { 924 dev_info(ce->dev, 925 "DEBUG: Algo of %s not supported\n", 926 ce_algs[i].alg.rng.base.cra_name); 927 ce_algs[i].ce = NULL; 928 break; 929 } 930 dev_info(ce->dev, "Register %s\n", 931 ce_algs[i].alg.rng.base.cra_name); 932 err = crypto_register_rng(&ce_algs[i].alg.rng); 933 if (err) { 934 dev_err(ce->dev, "Fail to register %s\n", 935 ce_algs[i].alg.rng.base.cra_name); 936 ce_algs[i].ce = NULL; 937 } 938 break; 939 default: 940 ce_algs[i].ce = NULL; 941 dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); 942 } 943 } 944 return 0; 945 } 946 947 static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) 948 { 949 unsigned int i; 950 951 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 952 if (!ce_algs[i].ce) 953 continue; 954 switch (ce_algs[i].type) { 955 case CRYPTO_ALG_TYPE_SKCIPHER: 956 dev_info(ce->dev, "Unregister %d %s\n", i, 957 ce_algs[i].alg.skcipher.base.base.cra_name); 958 crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher); 959 break; 960 case CRYPTO_ALG_TYPE_AHASH: 961 dev_info(ce->dev, "Unregister %d %s\n", i, 962 ce_algs[i].alg.hash.base.halg.base.cra_name); 963 crypto_engine_unregister_ahash(&ce_algs[i].alg.hash); 964 break; 965 case CRYPTO_ALG_TYPE_RNG: 966 dev_info(ce->dev, "Unregister %d %s\n", i, 967 ce_algs[i].alg.rng.base.cra_name); 968 crypto_unregister_rng(&ce_algs[i].alg.rng); 969 break; 970 } 971 } 972 } 973 974 static int sun8i_ce_probe(struct platform_device *pdev) 975 { 976 struct sun8i_ce_dev *ce; 977 int err, irq; 978 u32 v; 979 980 ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); 981 if (!ce) 982 return -ENOMEM; 983 984 ce->dev = &pdev->dev; 985 platform_set_drvdata(pdev, ce); 986 987 ce->variant = of_device_get_match_data(&pdev->dev); 988 if (!ce->variant) { 989 dev_err(&pdev->dev, "Missing Crypto Engine variant\n"); 990 return -EINVAL; 991 } 992 993 ce->base = devm_platform_ioremap_resource(pdev, 0); 994 if (IS_ERR(ce->base)) 995 return PTR_ERR(ce->base); 996 997 err = sun8i_ce_get_clks(ce); 998 if (err) 999 return err; 1000 1001 /* Get Non Secure IRQ */ 1002 irq = platform_get_irq(pdev, 0); 1003 if (irq < 0) 1004 return irq; 1005 1006 ce->reset = devm_reset_control_get(&pdev->dev, NULL); 1007 if (IS_ERR(ce->reset)) 1008 return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), 1009 "No reset control found\n"); 1010 1011 mutex_init(&ce->mlock); 1012 mutex_init(&ce->rnglock); 1013 1014 err = sun8i_ce_allocate_chanlist(ce); 1015 if (err) 1016 return err; 1017 1018 err = sun8i_ce_pm_init(ce); 1019 if (err) 1020 goto error_pm; 1021 1022 err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, 1023 "sun8i-ce-ns", ce); 1024 if (err) { 1025 dev_err(ce->dev, "Cannot request CryptoEngine Non-secure IRQ (err=%d)\n", err); 1026 goto error_irq; 1027 } 1028 1029 err = sun8i_ce_register_algs(ce); 1030 if (err) 1031 goto error_alg; 1032 1033 err = pm_runtime_resume_and_get(ce->dev); 1034 if (err < 0) 1035 goto error_alg; 1036 1037 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG 1038 sun8i_ce_hwrng_register(ce); 1039 #endif 1040 1041 v = readl(ce->base + CE_CTR); 1042 v >>= CE_DIE_ID_SHIFT; 1043 v &= CE_DIE_ID_MASK; 1044 dev_info(&pdev->dev, "CryptoEngine Die ID %x\n", v); 1045 1046 pm_runtime_put_sync(ce->dev); 1047 1048 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { 1049 struct dentry *dbgfs_dir __maybe_unused; 1050 struct dentry *dbgfs_stats __maybe_unused; 1051 1052 /* Ignore error of debugfs */ 1053 dbgfs_dir = debugfs_create_dir("sun8i-ce", NULL); 1054 dbgfs_stats = debugfs_create_file("stats", 0444, 1055 dbgfs_dir, ce, 1056 &sun8i_ce_debugfs_fops); 1057 1058 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 1059 ce->dbgfs_dir = dbgfs_dir; 1060 ce->dbgfs_stats = dbgfs_stats; 1061 #endif 1062 } 1063 1064 return 0; 1065 error_alg: 1066 sun8i_ce_unregister_algs(ce); 1067 error_irq: 1068 sun8i_ce_pm_exit(ce); 1069 error_pm: 1070 sun8i_ce_free_chanlist(ce, MAXFLOW - 1); 1071 return err; 1072 } 1073 1074 static int sun8i_ce_remove(struct platform_device *pdev) 1075 { 1076 struct sun8i_ce_dev *ce = platform_get_drvdata(pdev); 1077 1078 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG 1079 sun8i_ce_hwrng_unregister(ce); 1080 #endif 1081 1082 sun8i_ce_unregister_algs(ce); 1083 1084 #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 1085 debugfs_remove_recursive(ce->dbgfs_dir); 1086 #endif 1087 1088 sun8i_ce_free_chanlist(ce, MAXFLOW - 1); 1089 1090 sun8i_ce_pm_exit(ce); 1091 return 0; 1092 } 1093 1094 static const struct of_device_id sun8i_ce_crypto_of_match_table[] = { 1095 { .compatible = "allwinner,sun8i-h3-crypto", 1096 .data = &ce_h3_variant }, 1097 { .compatible = "allwinner,sun8i-r40-crypto", 1098 .data = &ce_r40_variant }, 1099 { .compatible = "allwinner,sun20i-d1-crypto", 1100 .data = &ce_d1_variant }, 1101 { .compatible = "allwinner,sun50i-a64-crypto", 1102 .data = &ce_a64_variant }, 1103 { .compatible = "allwinner,sun50i-h5-crypto", 1104 .data = &ce_h5_variant }, 1105 { .compatible = "allwinner,sun50i-h6-crypto", 1106 .data = &ce_h6_variant }, 1107 {} 1108 }; 1109 MODULE_DEVICE_TABLE(of, sun8i_ce_crypto_of_match_table); 1110 1111 static struct platform_driver sun8i_ce_driver = { 1112 .probe = sun8i_ce_probe, 1113 .remove = sun8i_ce_remove, 1114 .driver = { 1115 .name = "sun8i-ce", 1116 .pm = &sun8i_ce_pm_ops, 1117 .of_match_table = sun8i_ce_crypto_of_match_table, 1118 }, 1119 }; 1120 1121 module_platform_driver(sun8i_ce_driver); 1122 1123 MODULE_DESCRIPTION("Allwinner Crypto Engine cryptographic offloader"); 1124 MODULE_LICENSE("GPL"); 1125 MODULE_AUTHOR("Corentin Labbe <clabbe.montjoie@gmail.com>"); 1126