1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc. 3 */ 4 5 #include <linux/module.h> 6 #include <linux/kernel.h> 7 #include <linux/pci.h> 8 #include <linux/pci_ids.h> 9 #include <linux/crypto.h> 10 #include <linux/spinlock.h> 11 #include <crypto/algapi.h> 12 #include <crypto/aes.h> 13 14 #include <linux/io.h> 15 #include <linux/delay.h> 16 17 #include "geode-aes.h" 18 19 /* Static structures */ 20 21 static void __iomem *_iobase; 22 static spinlock_t lock; 23 24 /* Write a 128 bit field (either a writable key or IV) */ 25 static inline void 26 _writefield(u32 offset, void *value) 27 { 28 int i; 29 30 for (i = 0; i < 4; i++) 31 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4)); 32 } 33 34 /* Read a 128 bit field (either a writable key or IV) */ 35 static inline void 36 _readfield(u32 offset, void *value) 37 { 38 int i; 39 40 for (i = 0; i < 4; i++) 41 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4)); 42 } 43 44 static int 45 do_crypt(void *src, void *dst, int len, u32 flags) 46 { 47 u32 status; 48 u32 counter = AES_OP_TIMEOUT; 49 50 iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG); 51 iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG); 52 iowrite32(len, _iobase + AES_LENA_REG); 53 54 /* Start the operation */ 55 iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG); 56 57 do { 58 status = ioread32(_iobase + AES_INTR_REG); 59 cpu_relax(); 60 } while (!(status & AES_INTRA_PENDING) && --counter); 61 62 /* Clear the event */ 63 iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG); 64 return counter ? 0 : 1; 65 } 66 67 static unsigned int 68 geode_aes_crypt(struct geode_aes_op *op) 69 { 70 u32 flags = 0; 71 unsigned long iflags; 72 int ret; 73 74 if (op->len == 0) 75 return 0; 76 77 /* If the source and destination is the same, then 78 * we need to turn on the coherent flags, otherwise 79 * we don't need to worry 80 */ 81 82 flags |= (AES_CTRL_DCA | AES_CTRL_SCA); 83 84 if (op->dir == AES_DIR_ENCRYPT) 85 flags |= AES_CTRL_ENCRYPT; 86 87 /* Start the critical section */ 88 89 spin_lock_irqsave(&lock, iflags); 90 91 if (op->mode == AES_MODE_CBC) { 92 flags |= AES_CTRL_CBC; 93 _writefield(AES_WRITEIV0_REG, op->iv); 94 } 95 96 if (!(op->flags & AES_FLAGS_HIDDENKEY)) { 97 flags |= AES_CTRL_WRKEY; 98 _writefield(AES_WRITEKEY0_REG, op->key); 99 } 100 101 ret = do_crypt(op->src, op->dst, op->len, flags); 102 BUG_ON(ret); 103 104 if (op->mode == AES_MODE_CBC) 105 _readfield(AES_WRITEIV0_REG, op->iv); 106 107 spin_unlock_irqrestore(&lock, iflags); 108 109 return op->len; 110 } 111 112 /* CRYPTO-API Functions */ 113 114 static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key, 115 unsigned int len) 116 { 117 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 118 unsigned int ret; 119 120 op->keylen = len; 121 122 if (len == AES_KEYSIZE_128) { 123 memcpy(op->key, key, len); 124 return 0; 125 } 126 127 if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { 128 /* not supported at all */ 129 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 130 return -EINVAL; 131 } 132 133 /* 134 * The requested key size is not supported by HW, do a fallback 135 */ 136 op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 137 op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); 138 139 ret = crypto_cipher_setkey(op->fallback.cip, key, len); 140 if (ret) { 141 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; 142 tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK); 143 } 144 return ret; 145 } 146 147 static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key, 148 unsigned int len) 149 { 150 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 151 unsigned int ret; 152 153 op->keylen = len; 154 155 if (len == AES_KEYSIZE_128) { 156 memcpy(op->key, key, len); 157 return 0; 158 } 159 160 if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { 161 /* not supported at all */ 162 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 163 return -EINVAL; 164 } 165 166 /* 167 * The requested key size is not supported by HW, do a fallback 168 */ 169 op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 170 op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); 171 172 ret = crypto_blkcipher_setkey(op->fallback.blk, key, len); 173 if (ret) { 174 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; 175 tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK); 176 } 177 return ret; 178 } 179 180 static int fallback_blk_dec(struct blkcipher_desc *desc, 181 struct scatterlist *dst, struct scatterlist *src, 182 unsigned int nbytes) 183 { 184 unsigned int ret; 185 struct crypto_blkcipher *tfm; 186 struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); 187 188 tfm = desc->tfm; 189 desc->tfm = op->fallback.blk; 190 191 ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes); 192 193 desc->tfm = tfm; 194 return ret; 195 } 196 static int fallback_blk_enc(struct blkcipher_desc *desc, 197 struct scatterlist *dst, struct scatterlist *src, 198 unsigned int nbytes) 199 { 200 unsigned int ret; 201 struct crypto_blkcipher *tfm; 202 struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); 203 204 tfm = desc->tfm; 205 desc->tfm = op->fallback.blk; 206 207 ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes); 208 209 desc->tfm = tfm; 210 return ret; 211 } 212 213 static void 214 geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 215 { 216 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 217 218 if (unlikely(op->keylen != AES_KEYSIZE_128)) { 219 crypto_cipher_encrypt_one(op->fallback.cip, out, in); 220 return; 221 } 222 223 op->src = (void *) in; 224 op->dst = (void *) out; 225 op->mode = AES_MODE_ECB; 226 op->flags = 0; 227 op->len = AES_BLOCK_SIZE; 228 op->dir = AES_DIR_ENCRYPT; 229 230 geode_aes_crypt(op); 231 } 232 233 234 static void 235 geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 236 { 237 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 238 239 if (unlikely(op->keylen != AES_KEYSIZE_128)) { 240 crypto_cipher_decrypt_one(op->fallback.cip, out, in); 241 return; 242 } 243 244 op->src = (void *) in; 245 op->dst = (void *) out; 246 op->mode = AES_MODE_ECB; 247 op->flags = 0; 248 op->len = AES_BLOCK_SIZE; 249 op->dir = AES_DIR_DECRYPT; 250 251 geode_aes_crypt(op); 252 } 253 254 static int fallback_init_cip(struct crypto_tfm *tfm) 255 { 256 const char *name = crypto_tfm_alg_name(tfm); 257 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 258 259 op->fallback.cip = crypto_alloc_cipher(name, 0, 260 CRYPTO_ALG_NEED_FALLBACK); 261 262 if (IS_ERR(op->fallback.cip)) { 263 printk(KERN_ERR "Error allocating fallback algo %s\n", name); 264 return PTR_ERR(op->fallback.cip); 265 } 266 267 return 0; 268 } 269 270 static void fallback_exit_cip(struct crypto_tfm *tfm) 271 { 272 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 273 274 crypto_free_cipher(op->fallback.cip); 275 op->fallback.cip = NULL; 276 } 277 278 static struct crypto_alg geode_alg = { 279 .cra_name = "aes", 280 .cra_driver_name = "geode-aes", 281 .cra_priority = 300, 282 .cra_alignmask = 15, 283 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | 284 CRYPTO_ALG_NEED_FALLBACK, 285 .cra_init = fallback_init_cip, 286 .cra_exit = fallback_exit_cip, 287 .cra_blocksize = AES_BLOCK_SIZE, 288 .cra_ctxsize = sizeof(struct geode_aes_op), 289 .cra_module = THIS_MODULE, 290 .cra_u = { 291 .cipher = { 292 .cia_min_keysize = AES_MIN_KEY_SIZE, 293 .cia_max_keysize = AES_MAX_KEY_SIZE, 294 .cia_setkey = geode_setkey_cip, 295 .cia_encrypt = geode_encrypt, 296 .cia_decrypt = geode_decrypt 297 } 298 } 299 }; 300 301 static int 302 geode_cbc_decrypt(struct blkcipher_desc *desc, 303 struct scatterlist *dst, struct scatterlist *src, 304 unsigned int nbytes) 305 { 306 struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); 307 struct blkcipher_walk walk; 308 int err, ret; 309 310 if (unlikely(op->keylen != AES_KEYSIZE_128)) 311 return fallback_blk_dec(desc, dst, src, nbytes); 312 313 blkcipher_walk_init(&walk, dst, src, nbytes); 314 err = blkcipher_walk_virt(desc, &walk); 315 op->iv = walk.iv; 316 317 while ((nbytes = walk.nbytes)) { 318 op->src = walk.src.virt.addr, 319 op->dst = walk.dst.virt.addr; 320 op->mode = AES_MODE_CBC; 321 op->len = nbytes - (nbytes % AES_BLOCK_SIZE); 322 op->dir = AES_DIR_DECRYPT; 323 324 ret = geode_aes_crypt(op); 325 326 nbytes -= ret; 327 err = blkcipher_walk_done(desc, &walk, nbytes); 328 } 329 330 return err; 331 } 332 333 static int 334 geode_cbc_encrypt(struct blkcipher_desc *desc, 335 struct scatterlist *dst, struct scatterlist *src, 336 unsigned int nbytes) 337 { 338 struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); 339 struct blkcipher_walk walk; 340 int err, ret; 341 342 if (unlikely(op->keylen != AES_KEYSIZE_128)) 343 return fallback_blk_enc(desc, dst, src, nbytes); 344 345 blkcipher_walk_init(&walk, dst, src, nbytes); 346 err = blkcipher_walk_virt(desc, &walk); 347 op->iv = walk.iv; 348 349 while ((nbytes = walk.nbytes)) { 350 op->src = walk.src.virt.addr, 351 op->dst = walk.dst.virt.addr; 352 op->mode = AES_MODE_CBC; 353 op->len = nbytes - (nbytes % AES_BLOCK_SIZE); 354 op->dir = AES_DIR_ENCRYPT; 355 356 ret = geode_aes_crypt(op); 357 nbytes -= ret; 358 err = blkcipher_walk_done(desc, &walk, nbytes); 359 } 360 361 return err; 362 } 363 364 static int fallback_init_blk(struct crypto_tfm *tfm) 365 { 366 const char *name = crypto_tfm_alg_name(tfm); 367 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 368 369 op->fallback.blk = crypto_alloc_blkcipher(name, 0, 370 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 371 372 if (IS_ERR(op->fallback.blk)) { 373 printk(KERN_ERR "Error allocating fallback algo %s\n", name); 374 return PTR_ERR(op->fallback.blk); 375 } 376 377 return 0; 378 } 379 380 static void fallback_exit_blk(struct crypto_tfm *tfm) 381 { 382 struct geode_aes_op *op = crypto_tfm_ctx(tfm); 383 384 crypto_free_blkcipher(op->fallback.blk); 385 op->fallback.blk = NULL; 386 } 387 388 static struct crypto_alg geode_cbc_alg = { 389 .cra_name = "cbc(aes)", 390 .cra_driver_name = "cbc-aes-geode", 391 .cra_priority = 400, 392 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 393 CRYPTO_ALG_KERN_DRIVER_ONLY | 394 CRYPTO_ALG_NEED_FALLBACK, 395 .cra_init = fallback_init_blk, 396 .cra_exit = fallback_exit_blk, 397 .cra_blocksize = AES_BLOCK_SIZE, 398 .cra_ctxsize = sizeof(struct geode_aes_op), 399 .cra_alignmask = 15, 400 .cra_type = &crypto_blkcipher_type, 401 .cra_module = THIS_MODULE, 402 .cra_u = { 403 .blkcipher = { 404 .min_keysize = AES_MIN_KEY_SIZE, 405 .max_keysize = AES_MAX_KEY_SIZE, 406 .setkey = geode_setkey_blk, 407 .encrypt = geode_cbc_encrypt, 408 .decrypt = geode_cbc_decrypt, 409 .ivsize = AES_BLOCK_SIZE, 410 } 411 } 412 }; 413 414 static int 415 geode_ecb_decrypt(struct blkcipher_desc *desc, 416 struct scatterlist *dst, struct scatterlist *src, 417 unsigned int nbytes) 418 { 419 struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); 420 struct blkcipher_walk walk; 421 int err, ret; 422 423 if (unlikely(op->keylen != AES_KEYSIZE_128)) 424 return fallback_blk_dec(desc, dst, src, nbytes); 425 426 blkcipher_walk_init(&walk, dst, src, nbytes); 427 err = blkcipher_walk_virt(desc, &walk); 428 429 while ((nbytes = walk.nbytes)) { 430 op->src = walk.src.virt.addr, 431 op->dst = walk.dst.virt.addr; 432 op->mode = AES_MODE_ECB; 433 op->len = nbytes - (nbytes % AES_BLOCK_SIZE); 434 op->dir = AES_DIR_DECRYPT; 435 436 ret = geode_aes_crypt(op); 437 nbytes -= ret; 438 err = blkcipher_walk_done(desc, &walk, nbytes); 439 } 440 441 return err; 442 } 443 444 static int 445 geode_ecb_encrypt(struct blkcipher_desc *desc, 446 struct scatterlist *dst, struct scatterlist *src, 447 unsigned int nbytes) 448 { 449 struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); 450 struct blkcipher_walk walk; 451 int err, ret; 452 453 if (unlikely(op->keylen != AES_KEYSIZE_128)) 454 return fallback_blk_enc(desc, dst, src, nbytes); 455 456 blkcipher_walk_init(&walk, dst, src, nbytes); 457 err = blkcipher_walk_virt(desc, &walk); 458 459 while ((nbytes = walk.nbytes)) { 460 op->src = walk.src.virt.addr, 461 op->dst = walk.dst.virt.addr; 462 op->mode = AES_MODE_ECB; 463 op->len = nbytes - (nbytes % AES_BLOCK_SIZE); 464 op->dir = AES_DIR_ENCRYPT; 465 466 ret = geode_aes_crypt(op); 467 nbytes -= ret; 468 ret = blkcipher_walk_done(desc, &walk, nbytes); 469 } 470 471 return err; 472 } 473 474 static struct crypto_alg geode_ecb_alg = { 475 .cra_name = "ecb(aes)", 476 .cra_driver_name = "ecb-aes-geode", 477 .cra_priority = 400, 478 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 479 CRYPTO_ALG_KERN_DRIVER_ONLY | 480 CRYPTO_ALG_NEED_FALLBACK, 481 .cra_init = fallback_init_blk, 482 .cra_exit = fallback_exit_blk, 483 .cra_blocksize = AES_BLOCK_SIZE, 484 .cra_ctxsize = sizeof(struct geode_aes_op), 485 .cra_alignmask = 15, 486 .cra_type = &crypto_blkcipher_type, 487 .cra_module = THIS_MODULE, 488 .cra_u = { 489 .blkcipher = { 490 .min_keysize = AES_MIN_KEY_SIZE, 491 .max_keysize = AES_MAX_KEY_SIZE, 492 .setkey = geode_setkey_blk, 493 .encrypt = geode_ecb_encrypt, 494 .decrypt = geode_ecb_decrypt, 495 } 496 } 497 }; 498 499 static void geode_aes_remove(struct pci_dev *dev) 500 { 501 crypto_unregister_alg(&geode_alg); 502 crypto_unregister_alg(&geode_ecb_alg); 503 crypto_unregister_alg(&geode_cbc_alg); 504 505 pci_iounmap(dev, _iobase); 506 _iobase = NULL; 507 508 pci_release_regions(dev); 509 pci_disable_device(dev); 510 } 511 512 513 static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) 514 { 515 int ret; 516 517 ret = pci_enable_device(dev); 518 if (ret) 519 return ret; 520 521 ret = pci_request_regions(dev, "geode-aes"); 522 if (ret) 523 goto eenable; 524 525 _iobase = pci_iomap(dev, 0, 0); 526 527 if (_iobase == NULL) { 528 ret = -ENOMEM; 529 goto erequest; 530 } 531 532 spin_lock_init(&lock); 533 534 /* Clear any pending activity */ 535 iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG); 536 537 ret = crypto_register_alg(&geode_alg); 538 if (ret) 539 goto eiomap; 540 541 ret = crypto_register_alg(&geode_ecb_alg); 542 if (ret) 543 goto ealg; 544 545 ret = crypto_register_alg(&geode_cbc_alg); 546 if (ret) 547 goto eecb; 548 549 dev_notice(&dev->dev, "GEODE AES engine enabled.\n"); 550 return 0; 551 552 eecb: 553 crypto_unregister_alg(&geode_ecb_alg); 554 555 ealg: 556 crypto_unregister_alg(&geode_alg); 557 558 eiomap: 559 pci_iounmap(dev, _iobase); 560 561 erequest: 562 pci_release_regions(dev); 563 564 eenable: 565 pci_disable_device(dev); 566 567 dev_err(&dev->dev, "GEODE AES initialization failed.\n"); 568 return ret; 569 } 570 571 static struct pci_device_id geode_aes_tbl[] = { 572 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), }, 573 { 0, } 574 }; 575 576 MODULE_DEVICE_TABLE(pci, geode_aes_tbl); 577 578 static struct pci_driver geode_aes_driver = { 579 .name = "Geode LX AES", 580 .id_table = geode_aes_tbl, 581 .probe = geode_aes_probe, 582 .remove = geode_aes_remove, 583 }; 584 585 module_pci_driver(geode_aes_driver); 586 587 MODULE_AUTHOR("Advanced Micro Devices, Inc."); 588 MODULE_DESCRIPTION("Geode LX Hardware AES driver"); 589 MODULE_LICENSE("GPL"); 590