1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. 2 * 3 * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net> 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/of.h> 11 #include <linux/of_device.h> 12 #include <linux/cpumask.h> 13 #include <linux/slab.h> 14 #include <linux/interrupt.h> 15 #include <linux/crypto.h> 16 #include <crypto/md5.h> 17 #include <crypto/sha.h> 18 #include <crypto/aes.h> 19 #include <crypto/des.h> 20 #include <linux/mutex.h> 21 #include <linux/delay.h> 22 #include <linux/sched.h> 23 24 #include <crypto/internal/hash.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/algapi.h> 27 28 #include <asm/hypervisor.h> 29 #include <asm/mdesc.h> 30 31 #include "n2_core.h" 32 33 #define DRV_MODULE_NAME "n2_crypto" 34 #define DRV_MODULE_VERSION "0.2" 35 #define DRV_MODULE_RELDATE "July 28, 2011" 36 37 static char version[] = 38 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 39 40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 41 MODULE_DESCRIPTION("Niagara2 Crypto driver"); 42 MODULE_LICENSE("GPL"); 43 MODULE_VERSION(DRV_MODULE_VERSION); 44 45 #define N2_CRA_PRIORITY 200 46 47 static DEFINE_MUTEX(spu_lock); 48 49 struct spu_queue { 50 cpumask_t sharing; 51 unsigned long qhandle; 52 53 spinlock_t lock; 54 u8 q_type; 55 void *q; 56 unsigned long head; 57 unsigned long tail; 58 struct list_head jobs; 59 60 unsigned long devino; 61 62 char irq_name[32]; 63 unsigned int irq; 64 65 struct list_head list; 66 }; 67 68 static struct spu_queue **cpu_to_cwq; 69 static struct spu_queue **cpu_to_mau; 70 71 static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) 72 { 73 if (q->q_type == HV_NCS_QTYPE_MAU) { 74 off += MAU_ENTRY_SIZE; 75 if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) 76 off = 0; 77 } else { 78 off += CWQ_ENTRY_SIZE; 79 if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) 80 off = 0; 81 } 82 return off; 83 } 84 85 struct n2_request_common { 86 struct list_head entry; 87 unsigned int offset; 88 }; 89 #define OFFSET_NOT_RUNNING (~(unsigned int)0) 90 91 /* An async job request records the final tail value it used in 92 * n2_request_common->offset, test to see if that offset is in 93 * the range old_head, new_head, inclusive. 94 */ 95 static inline bool job_finished(struct spu_queue *q, unsigned int offset, 96 unsigned long old_head, unsigned long new_head) 97 { 98 if (old_head <= new_head) { 99 if (offset > old_head && offset <= new_head) 100 return true; 101 } else { 102 if (offset > old_head || offset <= new_head) 103 return true; 104 } 105 return false; 106 } 107 108 /* When the HEAD marker is unequal to the actual HEAD, we get 109 * a virtual device INO interrupt. We should process the 110 * completed CWQ entries and adjust the HEAD marker to clear 111 * the IRQ. 112 */ 113 static irqreturn_t cwq_intr(int irq, void *dev_id) 114 { 115 unsigned long off, new_head, hv_ret; 116 struct spu_queue *q = dev_id; 117 118 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n", 119 smp_processor_id(), q->qhandle); 120 121 spin_lock(&q->lock); 122 123 hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head); 124 125 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n", 126 smp_processor_id(), new_head, hv_ret); 127 128 for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { 129 /* XXX ... XXX */ 130 } 131 132 hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head); 133 if (hv_ret == HV_EOK) 134 q->head = new_head; 135 136 spin_unlock(&q->lock); 137 138 return IRQ_HANDLED; 139 } 140 141 static irqreturn_t mau_intr(int irq, void *dev_id) 142 { 143 struct spu_queue *q = dev_id; 144 unsigned long head, hv_ret; 145 146 spin_lock(&q->lock); 147 148 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n", 149 smp_processor_id(), q->qhandle); 150 151 hv_ret = sun4v_ncs_gethead(q->qhandle, &head); 152 153 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n", 154 smp_processor_id(), head, hv_ret); 155 156 sun4v_ncs_sethead_marker(q->qhandle, head); 157 158 spin_unlock(&q->lock); 159 160 return IRQ_HANDLED; 161 } 162 163 static void *spu_queue_next(struct spu_queue *q, void *cur) 164 { 165 return q->q + spu_next_offset(q, cur - q->q); 166 } 167 168 static int spu_queue_num_free(struct spu_queue *q) 169 { 170 unsigned long head = q->head; 171 unsigned long tail = q->tail; 172 unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); 173 unsigned long diff; 174 175 if (head > tail) 176 diff = head - tail; 177 else 178 diff = (end - tail) + head; 179 180 return (diff / CWQ_ENTRY_SIZE) - 1; 181 } 182 183 static void *spu_queue_alloc(struct spu_queue *q, int num_entries) 184 { 185 int avail = spu_queue_num_free(q); 186 187 if (avail >= num_entries) 188 return q->q + q->tail; 189 190 return NULL; 191 } 192 193 static unsigned long spu_queue_submit(struct spu_queue *q, void *last) 194 { 195 unsigned long hv_ret, new_tail; 196 197 new_tail = spu_next_offset(q, last - q->q); 198 199 hv_ret = sun4v_ncs_settail(q->qhandle, new_tail); 200 if (hv_ret == HV_EOK) 201 q->tail = new_tail; 202 return hv_ret; 203 } 204 205 static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, 206 int enc_type, int auth_type, 207 unsigned int hash_len, 208 bool sfas, bool sob, bool eob, bool encrypt, 209 int opcode) 210 { 211 u64 word = (len - 1) & CONTROL_LEN; 212 213 word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); 214 word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); 215 word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); 216 if (sfas) 217 word |= CONTROL_STORE_FINAL_AUTH_STATE; 218 if (sob) 219 word |= CONTROL_START_OF_BLOCK; 220 if (eob) 221 word |= CONTROL_END_OF_BLOCK; 222 if (encrypt) 223 word |= CONTROL_ENCRYPT; 224 if (hmac_key_len) 225 word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; 226 if (hash_len) 227 word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; 228 229 return word; 230 } 231 232 #if 0 233 static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) 234 { 235 if (this_len >= 64 || 236 qp->head != qp->tail) 237 return true; 238 return false; 239 } 240 #endif 241 242 struct n2_ahash_alg { 243 struct list_head entry; 244 const char *hash_zero; 245 const u32 *hash_init; 246 u8 hw_op_hashsz; 247 u8 digest_size; 248 u8 auth_type; 249 u8 hmac_type; 250 struct ahash_alg alg; 251 }; 252 253 static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) 254 { 255 struct crypto_alg *alg = tfm->__crt_alg; 256 struct ahash_alg *ahash_alg; 257 258 ahash_alg = container_of(alg, struct ahash_alg, halg.base); 259 260 return container_of(ahash_alg, struct n2_ahash_alg, alg); 261 } 262 263 struct n2_hmac_alg { 264 const char *child_alg; 265 struct n2_ahash_alg derived; 266 }; 267 268 static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) 269 { 270 struct crypto_alg *alg = tfm->__crt_alg; 271 struct ahash_alg *ahash_alg; 272 273 ahash_alg = container_of(alg, struct ahash_alg, halg.base); 274 275 return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); 276 } 277 278 struct n2_hash_ctx { 279 struct crypto_ahash *fallback_tfm; 280 }; 281 282 #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ 283 284 struct n2_hmac_ctx { 285 struct n2_hash_ctx base; 286 287 struct crypto_shash *child_shash; 288 289 int hash_key_len; 290 unsigned char hash_key[N2_HASH_KEY_MAX]; 291 }; 292 293 struct n2_hash_req_ctx { 294 union { 295 struct md5_state md5; 296 struct sha1_state sha1; 297 struct sha256_state sha256; 298 } u; 299 300 struct ahash_request fallback_req; 301 }; 302 303 static int n2_hash_async_init(struct ahash_request *req) 304 { 305 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 306 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 307 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 308 309 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 310 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 311 312 return crypto_ahash_init(&rctx->fallback_req); 313 } 314 315 static int n2_hash_async_update(struct ahash_request *req) 316 { 317 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 318 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 319 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 320 321 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 322 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 323 rctx->fallback_req.nbytes = req->nbytes; 324 rctx->fallback_req.src = req->src; 325 326 return crypto_ahash_update(&rctx->fallback_req); 327 } 328 329 static int n2_hash_async_final(struct ahash_request *req) 330 { 331 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 332 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 333 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 334 335 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 336 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 337 rctx->fallback_req.result = req->result; 338 339 return crypto_ahash_final(&rctx->fallback_req); 340 } 341 342 static int n2_hash_async_finup(struct ahash_request *req) 343 { 344 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 346 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 347 348 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 349 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 350 rctx->fallback_req.nbytes = req->nbytes; 351 rctx->fallback_req.src = req->src; 352 rctx->fallback_req.result = req->result; 353 354 return crypto_ahash_finup(&rctx->fallback_req); 355 } 356 357 static int n2_hash_cra_init(struct crypto_tfm *tfm) 358 { 359 const char *fallback_driver_name = crypto_tfm_alg_name(tfm); 360 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 361 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 362 struct crypto_ahash *fallback_tfm; 363 int err; 364 365 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 366 CRYPTO_ALG_NEED_FALLBACK); 367 if (IS_ERR(fallback_tfm)) { 368 pr_warning("Fallback driver '%s' could not be loaded!\n", 369 fallback_driver_name); 370 err = PTR_ERR(fallback_tfm); 371 goto out; 372 } 373 374 crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 375 crypto_ahash_reqsize(fallback_tfm))); 376 377 ctx->fallback_tfm = fallback_tfm; 378 return 0; 379 380 out: 381 return err; 382 } 383 384 static void n2_hash_cra_exit(struct crypto_tfm *tfm) 385 { 386 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 387 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 388 389 crypto_free_ahash(ctx->fallback_tfm); 390 } 391 392 static int n2_hmac_cra_init(struct crypto_tfm *tfm) 393 { 394 const char *fallback_driver_name = crypto_tfm_alg_name(tfm); 395 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 396 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); 397 struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); 398 struct crypto_ahash *fallback_tfm; 399 struct crypto_shash *child_shash; 400 int err; 401 402 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 403 CRYPTO_ALG_NEED_FALLBACK); 404 if (IS_ERR(fallback_tfm)) { 405 pr_warning("Fallback driver '%s' could not be loaded!\n", 406 fallback_driver_name); 407 err = PTR_ERR(fallback_tfm); 408 goto out; 409 } 410 411 child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0); 412 if (IS_ERR(child_shash)) { 413 pr_warning("Child shash '%s' could not be loaded!\n", 414 n2alg->child_alg); 415 err = PTR_ERR(child_shash); 416 goto out_free_fallback; 417 } 418 419 crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 420 crypto_ahash_reqsize(fallback_tfm))); 421 422 ctx->child_shash = child_shash; 423 ctx->base.fallback_tfm = fallback_tfm; 424 return 0; 425 426 out_free_fallback: 427 crypto_free_ahash(fallback_tfm); 428 429 out: 430 return err; 431 } 432 433 static void n2_hmac_cra_exit(struct crypto_tfm *tfm) 434 { 435 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 436 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); 437 438 crypto_free_ahash(ctx->base.fallback_tfm); 439 crypto_free_shash(ctx->child_shash); 440 } 441 442 static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, 443 unsigned int keylen) 444 { 445 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); 446 struct crypto_shash *child_shash = ctx->child_shash; 447 struct crypto_ahash *fallback_tfm; 448 SHASH_DESC_ON_STACK(shash, child_shash); 449 int err, bs, ds; 450 451 fallback_tfm = ctx->base.fallback_tfm; 452 err = crypto_ahash_setkey(fallback_tfm, key, keylen); 453 if (err) 454 return err; 455 456 shash->tfm = child_shash; 457 shash->flags = crypto_ahash_get_flags(tfm) & 458 CRYPTO_TFM_REQ_MAY_SLEEP; 459 460 bs = crypto_shash_blocksize(child_shash); 461 ds = crypto_shash_digestsize(child_shash); 462 BUG_ON(ds > N2_HASH_KEY_MAX); 463 if (keylen > bs) { 464 err = crypto_shash_digest(shash, key, keylen, 465 ctx->hash_key); 466 if (err) 467 return err; 468 keylen = ds; 469 } else if (keylen <= N2_HASH_KEY_MAX) 470 memcpy(ctx->hash_key, key, keylen); 471 472 ctx->hash_key_len = keylen; 473 474 return err; 475 } 476 477 static unsigned long wait_for_tail(struct spu_queue *qp) 478 { 479 unsigned long head, hv_ret; 480 481 do { 482 hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); 483 if (hv_ret != HV_EOK) { 484 pr_err("Hypervisor error on gethead\n"); 485 break; 486 } 487 if (head == qp->tail) { 488 qp->head = head; 489 break; 490 } 491 } while (1); 492 return hv_ret; 493 } 494 495 static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, 496 struct cwq_initial_entry *ent) 497 { 498 unsigned long hv_ret = spu_queue_submit(qp, ent); 499 500 if (hv_ret == HV_EOK) 501 hv_ret = wait_for_tail(qp); 502 503 return hv_ret; 504 } 505 506 static int n2_do_async_digest(struct ahash_request *req, 507 unsigned int auth_type, unsigned int digest_size, 508 unsigned int result_size, void *hash_loc, 509 unsigned long auth_key, unsigned int auth_key_len) 510 { 511 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 512 struct cwq_initial_entry *ent; 513 struct crypto_hash_walk walk; 514 struct spu_queue *qp; 515 unsigned long flags; 516 int err = -ENODEV; 517 int nbytes, cpu; 518 519 /* The total effective length of the operation may not 520 * exceed 2^16. 521 */ 522 if (unlikely(req->nbytes > (1 << 16))) { 523 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 524 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 525 526 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 527 rctx->fallback_req.base.flags = 528 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 529 rctx->fallback_req.nbytes = req->nbytes; 530 rctx->fallback_req.src = req->src; 531 rctx->fallback_req.result = req->result; 532 533 return crypto_ahash_digest(&rctx->fallback_req); 534 } 535 536 nbytes = crypto_hash_walk_first(req, &walk); 537 538 cpu = get_cpu(); 539 qp = cpu_to_cwq[cpu]; 540 if (!qp) 541 goto out; 542 543 spin_lock_irqsave(&qp->lock, flags); 544 545 /* XXX can do better, improve this later by doing a by-hand scatterlist 546 * XXX walk, etc. 547 */ 548 ent = qp->q + qp->tail; 549 550 ent->control = control_word_base(nbytes, auth_key_len, 0, 551 auth_type, digest_size, 552 false, true, false, false, 553 OPCODE_INPLACE_BIT | 554 OPCODE_AUTH_MAC); 555 ent->src_addr = __pa(walk.data); 556 ent->auth_key_addr = auth_key; 557 ent->auth_iv_addr = __pa(hash_loc); 558 ent->final_auth_state_addr = 0UL; 559 ent->enc_key_addr = 0UL; 560 ent->enc_iv_addr = 0UL; 561 ent->dest_addr = __pa(hash_loc); 562 563 nbytes = crypto_hash_walk_done(&walk, 0); 564 while (nbytes > 0) { 565 ent = spu_queue_next(qp, ent); 566 567 ent->control = (nbytes - 1); 568 ent->src_addr = __pa(walk.data); 569 ent->auth_key_addr = 0UL; 570 ent->auth_iv_addr = 0UL; 571 ent->final_auth_state_addr = 0UL; 572 ent->enc_key_addr = 0UL; 573 ent->enc_iv_addr = 0UL; 574 ent->dest_addr = 0UL; 575 576 nbytes = crypto_hash_walk_done(&walk, 0); 577 } 578 ent->control |= CONTROL_END_OF_BLOCK; 579 580 if (submit_and_wait_for_tail(qp, ent) != HV_EOK) 581 err = -EINVAL; 582 else 583 err = 0; 584 585 spin_unlock_irqrestore(&qp->lock, flags); 586 587 if (!err) 588 memcpy(req->result, hash_loc, result_size); 589 out: 590 put_cpu(); 591 592 return err; 593 } 594 595 static int n2_hash_async_digest(struct ahash_request *req) 596 { 597 struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm); 598 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 599 int ds; 600 601 ds = n2alg->digest_size; 602 if (unlikely(req->nbytes == 0)) { 603 memcpy(req->result, n2alg->hash_zero, ds); 604 return 0; 605 } 606 memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); 607 608 return n2_do_async_digest(req, n2alg->auth_type, 609 n2alg->hw_op_hashsz, ds, 610 &rctx->u, 0UL, 0); 611 } 612 613 static int n2_hmac_async_digest(struct ahash_request *req) 614 { 615 struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm); 616 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 617 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 618 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); 619 int ds; 620 621 ds = n2alg->derived.digest_size; 622 if (unlikely(req->nbytes == 0) || 623 unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { 624 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 625 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 626 627 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 628 rctx->fallback_req.base.flags = 629 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 630 rctx->fallback_req.nbytes = req->nbytes; 631 rctx->fallback_req.src = req->src; 632 rctx->fallback_req.result = req->result; 633 634 return crypto_ahash_digest(&rctx->fallback_req); 635 } 636 memcpy(&rctx->u, n2alg->derived.hash_init, 637 n2alg->derived.hw_op_hashsz); 638 639 return n2_do_async_digest(req, n2alg->derived.hmac_type, 640 n2alg->derived.hw_op_hashsz, ds, 641 &rctx->u, 642 __pa(&ctx->hash_key), 643 ctx->hash_key_len); 644 } 645 646 struct n2_cipher_context { 647 int key_len; 648 int enc_type; 649 union { 650 u8 aes[AES_MAX_KEY_SIZE]; 651 u8 des[DES_KEY_SIZE]; 652 u8 des3[3 * DES_KEY_SIZE]; 653 u8 arc4[258]; /* S-box, X, Y */ 654 } key; 655 }; 656 657 #define N2_CHUNK_ARR_LEN 16 658 659 struct n2_crypto_chunk { 660 struct list_head entry; 661 unsigned long iv_paddr : 44; 662 unsigned long arr_len : 20; 663 unsigned long dest_paddr; 664 unsigned long dest_final; 665 struct { 666 unsigned long src_paddr : 44; 667 unsigned long src_len : 20; 668 } arr[N2_CHUNK_ARR_LEN]; 669 }; 670 671 struct n2_request_context { 672 struct ablkcipher_walk walk; 673 struct list_head chunk_list; 674 struct n2_crypto_chunk chunk; 675 u8 temp_iv[16]; 676 }; 677 678 /* The SPU allows some level of flexibility for partial cipher blocks 679 * being specified in a descriptor. 680 * 681 * It merely requires that every descriptor's length field is at least 682 * as large as the cipher block size. This means that a cipher block 683 * can span at most 2 descriptors. However, this does not allow a 684 * partial block to span into the final descriptor as that would 685 * violate the rule (since every descriptor's length must be at lest 686 * the block size). So, for example, assuming an 8 byte block size: 687 * 688 * 0xe --> 0xa --> 0x8 689 * 690 * is a valid length sequence, whereas: 691 * 692 * 0xe --> 0xb --> 0x7 693 * 694 * is not a valid sequence. 695 */ 696 697 struct n2_cipher_alg { 698 struct list_head entry; 699 u8 enc_type; 700 struct crypto_alg alg; 701 }; 702 703 static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm) 704 { 705 struct crypto_alg *alg = tfm->__crt_alg; 706 707 return container_of(alg, struct n2_cipher_alg, alg); 708 } 709 710 struct n2_cipher_request_context { 711 struct ablkcipher_walk walk; 712 }; 713 714 static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 715 unsigned int keylen) 716 { 717 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 718 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 719 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 720 721 ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); 722 723 switch (keylen) { 724 case AES_KEYSIZE_128: 725 ctx->enc_type |= ENC_TYPE_ALG_AES128; 726 break; 727 case AES_KEYSIZE_192: 728 ctx->enc_type |= ENC_TYPE_ALG_AES192; 729 break; 730 case AES_KEYSIZE_256: 731 ctx->enc_type |= ENC_TYPE_ALG_AES256; 732 break; 733 default: 734 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 735 return -EINVAL; 736 } 737 738 ctx->key_len = keylen; 739 memcpy(ctx->key.aes, key, keylen); 740 return 0; 741 } 742 743 static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 744 unsigned int keylen) 745 { 746 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 747 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 748 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 749 u32 tmp[DES_EXPKEY_WORDS]; 750 int err; 751 752 ctx->enc_type = n2alg->enc_type; 753 754 if (keylen != DES_KEY_SIZE) { 755 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 756 return -EINVAL; 757 } 758 759 err = des_ekey(tmp, key); 760 if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 761 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; 762 return -EINVAL; 763 } 764 765 ctx->key_len = keylen; 766 memcpy(ctx->key.des, key, keylen); 767 return 0; 768 } 769 770 static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 771 unsigned int keylen) 772 { 773 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 774 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 775 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 776 777 ctx->enc_type = n2alg->enc_type; 778 779 if (keylen != (3 * DES_KEY_SIZE)) { 780 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 781 return -EINVAL; 782 } 783 ctx->key_len = keylen; 784 memcpy(ctx->key.des3, key, keylen); 785 return 0; 786 } 787 788 static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 789 unsigned int keylen) 790 { 791 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 792 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 793 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 794 u8 *s = ctx->key.arc4; 795 u8 *x = s + 256; 796 u8 *y = x + 1; 797 int i, j, k; 798 799 ctx->enc_type = n2alg->enc_type; 800 801 j = k = 0; 802 *x = 0; 803 *y = 0; 804 for (i = 0; i < 256; i++) 805 s[i] = i; 806 for (i = 0; i < 256; i++) { 807 u8 a = s[i]; 808 j = (j + key[k] + a) & 0xff; 809 s[i] = s[j]; 810 s[j] = a; 811 if (++k >= keylen) 812 k = 0; 813 } 814 815 return 0; 816 } 817 818 static inline int cipher_descriptor_len(int nbytes, unsigned int block_size) 819 { 820 int this_len = nbytes; 821 822 this_len -= (nbytes & (block_size - 1)); 823 return this_len > (1 << 16) ? (1 << 16) : this_len; 824 } 825 826 static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp, 827 struct spu_queue *qp, bool encrypt) 828 { 829 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 830 struct cwq_initial_entry *ent; 831 bool in_place; 832 int i; 833 834 ent = spu_queue_alloc(qp, cp->arr_len); 835 if (!ent) { 836 pr_info("queue_alloc() of %d fails\n", 837 cp->arr_len); 838 return -EBUSY; 839 } 840 841 in_place = (cp->dest_paddr == cp->arr[0].src_paddr); 842 843 ent->control = control_word_base(cp->arr[0].src_len, 844 0, ctx->enc_type, 0, 0, 845 false, true, false, encrypt, 846 OPCODE_ENCRYPT | 847 (in_place ? OPCODE_INPLACE_BIT : 0)); 848 ent->src_addr = cp->arr[0].src_paddr; 849 ent->auth_key_addr = 0UL; 850 ent->auth_iv_addr = 0UL; 851 ent->final_auth_state_addr = 0UL; 852 ent->enc_key_addr = __pa(&ctx->key); 853 ent->enc_iv_addr = cp->iv_paddr; 854 ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); 855 856 for (i = 1; i < cp->arr_len; i++) { 857 ent = spu_queue_next(qp, ent); 858 859 ent->control = cp->arr[i].src_len - 1; 860 ent->src_addr = cp->arr[i].src_paddr; 861 ent->auth_key_addr = 0UL; 862 ent->auth_iv_addr = 0UL; 863 ent->final_auth_state_addr = 0UL; 864 ent->enc_key_addr = 0UL; 865 ent->enc_iv_addr = 0UL; 866 ent->dest_addr = 0UL; 867 } 868 ent->control |= CONTROL_END_OF_BLOCK; 869 870 return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; 871 } 872 873 static int n2_compute_chunks(struct ablkcipher_request *req) 874 { 875 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 876 struct ablkcipher_walk *walk = &rctx->walk; 877 struct n2_crypto_chunk *chunk; 878 unsigned long dest_prev; 879 unsigned int tot_len; 880 bool prev_in_place; 881 int err, nbytes; 882 883 ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes); 884 err = ablkcipher_walk_phys(req, walk); 885 if (err) 886 return err; 887 888 INIT_LIST_HEAD(&rctx->chunk_list); 889 890 chunk = &rctx->chunk; 891 INIT_LIST_HEAD(&chunk->entry); 892 893 chunk->iv_paddr = 0UL; 894 chunk->arr_len = 0; 895 chunk->dest_paddr = 0UL; 896 897 prev_in_place = false; 898 dest_prev = ~0UL; 899 tot_len = 0; 900 901 while ((nbytes = walk->nbytes) != 0) { 902 unsigned long dest_paddr, src_paddr; 903 bool in_place; 904 int this_len; 905 906 src_paddr = (page_to_phys(walk->src.page) + 907 walk->src.offset); 908 dest_paddr = (page_to_phys(walk->dst.page) + 909 walk->dst.offset); 910 in_place = (src_paddr == dest_paddr); 911 this_len = cipher_descriptor_len(nbytes, walk->blocksize); 912 913 if (chunk->arr_len != 0) { 914 if (in_place != prev_in_place || 915 (!prev_in_place && 916 dest_paddr != dest_prev) || 917 chunk->arr_len == N2_CHUNK_ARR_LEN || 918 tot_len + this_len > (1 << 16)) { 919 chunk->dest_final = dest_prev; 920 list_add_tail(&chunk->entry, 921 &rctx->chunk_list); 922 chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC); 923 if (!chunk) { 924 err = -ENOMEM; 925 break; 926 } 927 INIT_LIST_HEAD(&chunk->entry); 928 } 929 } 930 if (chunk->arr_len == 0) { 931 chunk->dest_paddr = dest_paddr; 932 tot_len = 0; 933 } 934 chunk->arr[chunk->arr_len].src_paddr = src_paddr; 935 chunk->arr[chunk->arr_len].src_len = this_len; 936 chunk->arr_len++; 937 938 dest_prev = dest_paddr + this_len; 939 prev_in_place = in_place; 940 tot_len += this_len; 941 942 err = ablkcipher_walk_done(req, walk, nbytes - this_len); 943 if (err) 944 break; 945 } 946 if (!err && chunk->arr_len != 0) { 947 chunk->dest_final = dest_prev; 948 list_add_tail(&chunk->entry, &rctx->chunk_list); 949 } 950 951 return err; 952 } 953 954 static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv) 955 { 956 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 957 struct n2_crypto_chunk *c, *tmp; 958 959 if (final_iv) 960 memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); 961 962 ablkcipher_walk_complete(&rctx->walk); 963 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 964 list_del(&c->entry); 965 if (unlikely(c != &rctx->chunk)) 966 kfree(c); 967 } 968 969 } 970 971 static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt) 972 { 973 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 974 struct crypto_tfm *tfm = req->base.tfm; 975 int err = n2_compute_chunks(req); 976 struct n2_crypto_chunk *c, *tmp; 977 unsigned long flags, hv_ret; 978 struct spu_queue *qp; 979 980 if (err) 981 return err; 982 983 qp = cpu_to_cwq[get_cpu()]; 984 err = -ENODEV; 985 if (!qp) 986 goto out; 987 988 spin_lock_irqsave(&qp->lock, flags); 989 990 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 991 err = __n2_crypt_chunk(tfm, c, qp, encrypt); 992 if (err) 993 break; 994 list_del(&c->entry); 995 if (unlikely(c != &rctx->chunk)) 996 kfree(c); 997 } 998 if (!err) { 999 hv_ret = wait_for_tail(qp); 1000 if (hv_ret != HV_EOK) 1001 err = -EINVAL; 1002 } 1003 1004 spin_unlock_irqrestore(&qp->lock, flags); 1005 1006 out: 1007 put_cpu(); 1008 1009 n2_chunk_complete(req, NULL); 1010 return err; 1011 } 1012 1013 static int n2_encrypt_ecb(struct ablkcipher_request *req) 1014 { 1015 return n2_do_ecb(req, true); 1016 } 1017 1018 static int n2_decrypt_ecb(struct ablkcipher_request *req) 1019 { 1020 return n2_do_ecb(req, false); 1021 } 1022 1023 static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt) 1024 { 1025 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 1026 struct crypto_tfm *tfm = req->base.tfm; 1027 unsigned long flags, hv_ret, iv_paddr; 1028 int err = n2_compute_chunks(req); 1029 struct n2_crypto_chunk *c, *tmp; 1030 struct spu_queue *qp; 1031 void *final_iv_addr; 1032 1033 final_iv_addr = NULL; 1034 1035 if (err) 1036 return err; 1037 1038 qp = cpu_to_cwq[get_cpu()]; 1039 err = -ENODEV; 1040 if (!qp) 1041 goto out; 1042 1043 spin_lock_irqsave(&qp->lock, flags); 1044 1045 if (encrypt) { 1046 iv_paddr = __pa(rctx->walk.iv); 1047 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, 1048 entry) { 1049 c->iv_paddr = iv_paddr; 1050 err = __n2_crypt_chunk(tfm, c, qp, true); 1051 if (err) 1052 break; 1053 iv_paddr = c->dest_final - rctx->walk.blocksize; 1054 list_del(&c->entry); 1055 if (unlikely(c != &rctx->chunk)) 1056 kfree(c); 1057 } 1058 final_iv_addr = __va(iv_paddr); 1059 } else { 1060 list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, 1061 entry) { 1062 if (c == &rctx->chunk) { 1063 iv_paddr = __pa(rctx->walk.iv); 1064 } else { 1065 iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + 1066 tmp->arr[tmp->arr_len-1].src_len - 1067 rctx->walk.blocksize); 1068 } 1069 if (!final_iv_addr) { 1070 unsigned long pa; 1071 1072 pa = (c->arr[c->arr_len-1].src_paddr + 1073 c->arr[c->arr_len-1].src_len - 1074 rctx->walk.blocksize); 1075 final_iv_addr = rctx->temp_iv; 1076 memcpy(rctx->temp_iv, __va(pa), 1077 rctx->walk.blocksize); 1078 } 1079 c->iv_paddr = iv_paddr; 1080 err = __n2_crypt_chunk(tfm, c, qp, false); 1081 if (err) 1082 break; 1083 list_del(&c->entry); 1084 if (unlikely(c != &rctx->chunk)) 1085 kfree(c); 1086 } 1087 } 1088 if (!err) { 1089 hv_ret = wait_for_tail(qp); 1090 if (hv_ret != HV_EOK) 1091 err = -EINVAL; 1092 } 1093 1094 spin_unlock_irqrestore(&qp->lock, flags); 1095 1096 out: 1097 put_cpu(); 1098 1099 n2_chunk_complete(req, err ? NULL : final_iv_addr); 1100 return err; 1101 } 1102 1103 static int n2_encrypt_chaining(struct ablkcipher_request *req) 1104 { 1105 return n2_do_chaining(req, true); 1106 } 1107 1108 static int n2_decrypt_chaining(struct ablkcipher_request *req) 1109 { 1110 return n2_do_chaining(req, false); 1111 } 1112 1113 struct n2_cipher_tmpl { 1114 const char *name; 1115 const char *drv_name; 1116 u8 block_size; 1117 u8 enc_type; 1118 struct ablkcipher_alg ablkcipher; 1119 }; 1120 1121 static const struct n2_cipher_tmpl cipher_tmpls[] = { 1122 /* ARC4: only ECB is supported (chaining bits ignored) */ 1123 { .name = "ecb(arc4)", 1124 .drv_name = "ecb-arc4", 1125 .block_size = 1, 1126 .enc_type = (ENC_TYPE_ALG_RC4_STREAM | 1127 ENC_TYPE_CHAINING_ECB), 1128 .ablkcipher = { 1129 .min_keysize = 1, 1130 .max_keysize = 256, 1131 .setkey = n2_arc4_setkey, 1132 .encrypt = n2_encrypt_ecb, 1133 .decrypt = n2_decrypt_ecb, 1134 }, 1135 }, 1136 1137 /* DES: ECB CBC and CFB are supported */ 1138 { .name = "ecb(des)", 1139 .drv_name = "ecb-des", 1140 .block_size = DES_BLOCK_SIZE, 1141 .enc_type = (ENC_TYPE_ALG_DES | 1142 ENC_TYPE_CHAINING_ECB), 1143 .ablkcipher = { 1144 .min_keysize = DES_KEY_SIZE, 1145 .max_keysize = DES_KEY_SIZE, 1146 .setkey = n2_des_setkey, 1147 .encrypt = n2_encrypt_ecb, 1148 .decrypt = n2_decrypt_ecb, 1149 }, 1150 }, 1151 { .name = "cbc(des)", 1152 .drv_name = "cbc-des", 1153 .block_size = DES_BLOCK_SIZE, 1154 .enc_type = (ENC_TYPE_ALG_DES | 1155 ENC_TYPE_CHAINING_CBC), 1156 .ablkcipher = { 1157 .ivsize = DES_BLOCK_SIZE, 1158 .min_keysize = DES_KEY_SIZE, 1159 .max_keysize = DES_KEY_SIZE, 1160 .setkey = n2_des_setkey, 1161 .encrypt = n2_encrypt_chaining, 1162 .decrypt = n2_decrypt_chaining, 1163 }, 1164 }, 1165 { .name = "cfb(des)", 1166 .drv_name = "cfb-des", 1167 .block_size = DES_BLOCK_SIZE, 1168 .enc_type = (ENC_TYPE_ALG_DES | 1169 ENC_TYPE_CHAINING_CFB), 1170 .ablkcipher = { 1171 .min_keysize = DES_KEY_SIZE, 1172 .max_keysize = DES_KEY_SIZE, 1173 .setkey = n2_des_setkey, 1174 .encrypt = n2_encrypt_chaining, 1175 .decrypt = n2_decrypt_chaining, 1176 }, 1177 }, 1178 1179 /* 3DES: ECB CBC and CFB are supported */ 1180 { .name = "ecb(des3_ede)", 1181 .drv_name = "ecb-3des", 1182 .block_size = DES_BLOCK_SIZE, 1183 .enc_type = (ENC_TYPE_ALG_3DES | 1184 ENC_TYPE_CHAINING_ECB), 1185 .ablkcipher = { 1186 .min_keysize = 3 * DES_KEY_SIZE, 1187 .max_keysize = 3 * DES_KEY_SIZE, 1188 .setkey = n2_3des_setkey, 1189 .encrypt = n2_encrypt_ecb, 1190 .decrypt = n2_decrypt_ecb, 1191 }, 1192 }, 1193 { .name = "cbc(des3_ede)", 1194 .drv_name = "cbc-3des", 1195 .block_size = DES_BLOCK_SIZE, 1196 .enc_type = (ENC_TYPE_ALG_3DES | 1197 ENC_TYPE_CHAINING_CBC), 1198 .ablkcipher = { 1199 .ivsize = DES_BLOCK_SIZE, 1200 .min_keysize = 3 * DES_KEY_SIZE, 1201 .max_keysize = 3 * DES_KEY_SIZE, 1202 .setkey = n2_3des_setkey, 1203 .encrypt = n2_encrypt_chaining, 1204 .decrypt = n2_decrypt_chaining, 1205 }, 1206 }, 1207 { .name = "cfb(des3_ede)", 1208 .drv_name = "cfb-3des", 1209 .block_size = DES_BLOCK_SIZE, 1210 .enc_type = (ENC_TYPE_ALG_3DES | 1211 ENC_TYPE_CHAINING_CFB), 1212 .ablkcipher = { 1213 .min_keysize = 3 * DES_KEY_SIZE, 1214 .max_keysize = 3 * DES_KEY_SIZE, 1215 .setkey = n2_3des_setkey, 1216 .encrypt = n2_encrypt_chaining, 1217 .decrypt = n2_decrypt_chaining, 1218 }, 1219 }, 1220 /* AES: ECB CBC and CTR are supported */ 1221 { .name = "ecb(aes)", 1222 .drv_name = "ecb-aes", 1223 .block_size = AES_BLOCK_SIZE, 1224 .enc_type = (ENC_TYPE_ALG_AES128 | 1225 ENC_TYPE_CHAINING_ECB), 1226 .ablkcipher = { 1227 .min_keysize = AES_MIN_KEY_SIZE, 1228 .max_keysize = AES_MAX_KEY_SIZE, 1229 .setkey = n2_aes_setkey, 1230 .encrypt = n2_encrypt_ecb, 1231 .decrypt = n2_decrypt_ecb, 1232 }, 1233 }, 1234 { .name = "cbc(aes)", 1235 .drv_name = "cbc-aes", 1236 .block_size = AES_BLOCK_SIZE, 1237 .enc_type = (ENC_TYPE_ALG_AES128 | 1238 ENC_TYPE_CHAINING_CBC), 1239 .ablkcipher = { 1240 .ivsize = AES_BLOCK_SIZE, 1241 .min_keysize = AES_MIN_KEY_SIZE, 1242 .max_keysize = AES_MAX_KEY_SIZE, 1243 .setkey = n2_aes_setkey, 1244 .encrypt = n2_encrypt_chaining, 1245 .decrypt = n2_decrypt_chaining, 1246 }, 1247 }, 1248 { .name = "ctr(aes)", 1249 .drv_name = "ctr-aes", 1250 .block_size = AES_BLOCK_SIZE, 1251 .enc_type = (ENC_TYPE_ALG_AES128 | 1252 ENC_TYPE_CHAINING_COUNTER), 1253 .ablkcipher = { 1254 .ivsize = AES_BLOCK_SIZE, 1255 .min_keysize = AES_MIN_KEY_SIZE, 1256 .max_keysize = AES_MAX_KEY_SIZE, 1257 .setkey = n2_aes_setkey, 1258 .encrypt = n2_encrypt_chaining, 1259 .decrypt = n2_encrypt_chaining, 1260 }, 1261 }, 1262 1263 }; 1264 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls) 1265 1266 static LIST_HEAD(cipher_algs); 1267 1268 struct n2_hash_tmpl { 1269 const char *name; 1270 const char *hash_zero; 1271 const u32 *hash_init; 1272 u8 hw_op_hashsz; 1273 u8 digest_size; 1274 u8 block_size; 1275 u8 auth_type; 1276 u8 hmac_type; 1277 }; 1278 1279 static const char md5_zero[MD5_DIGEST_SIZE] = { 1280 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 1281 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, 1282 }; 1283 static const u32 md5_init[MD5_HASH_WORDS] = { 1284 cpu_to_le32(0x67452301), 1285 cpu_to_le32(0xefcdab89), 1286 cpu_to_le32(0x98badcfe), 1287 cpu_to_le32(0x10325476), 1288 }; 1289 static const char sha1_zero[SHA1_DIGEST_SIZE] = { 1290 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 1291 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 1292 0x07, 0x09 1293 }; 1294 static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { 1295 SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 1296 }; 1297 static const char sha256_zero[SHA256_DIGEST_SIZE] = { 1298 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 1299 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 1300 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 1301 0x1b, 0x78, 0x52, 0xb8, 0x55 1302 }; 1303 static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = { 1304 SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, 1305 SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, 1306 }; 1307 static const char sha224_zero[SHA224_DIGEST_SIZE] = { 1308 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, 1309 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, 1310 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, 1311 0x2f 1312 }; 1313 static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = { 1314 SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, 1315 SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, 1316 }; 1317 1318 static const struct n2_hash_tmpl hash_tmpls[] = { 1319 { .name = "md5", 1320 .hash_zero = md5_zero, 1321 .hash_init = md5_init, 1322 .auth_type = AUTH_TYPE_MD5, 1323 .hmac_type = AUTH_TYPE_HMAC_MD5, 1324 .hw_op_hashsz = MD5_DIGEST_SIZE, 1325 .digest_size = MD5_DIGEST_SIZE, 1326 .block_size = MD5_HMAC_BLOCK_SIZE }, 1327 { .name = "sha1", 1328 .hash_zero = sha1_zero, 1329 .hash_init = sha1_init, 1330 .auth_type = AUTH_TYPE_SHA1, 1331 .hmac_type = AUTH_TYPE_HMAC_SHA1, 1332 .hw_op_hashsz = SHA1_DIGEST_SIZE, 1333 .digest_size = SHA1_DIGEST_SIZE, 1334 .block_size = SHA1_BLOCK_SIZE }, 1335 { .name = "sha256", 1336 .hash_zero = sha256_zero, 1337 .hash_init = sha256_init, 1338 .auth_type = AUTH_TYPE_SHA256, 1339 .hmac_type = AUTH_TYPE_HMAC_SHA256, 1340 .hw_op_hashsz = SHA256_DIGEST_SIZE, 1341 .digest_size = SHA256_DIGEST_SIZE, 1342 .block_size = SHA256_BLOCK_SIZE }, 1343 { .name = "sha224", 1344 .hash_zero = sha224_zero, 1345 .hash_init = sha224_init, 1346 .auth_type = AUTH_TYPE_SHA256, 1347 .hmac_type = AUTH_TYPE_RESERVED, 1348 .hw_op_hashsz = SHA256_DIGEST_SIZE, 1349 .digest_size = SHA224_DIGEST_SIZE, 1350 .block_size = SHA224_BLOCK_SIZE }, 1351 }; 1352 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) 1353 1354 static LIST_HEAD(ahash_algs); 1355 static LIST_HEAD(hmac_algs); 1356 1357 static int algs_registered; 1358 1359 static void __n2_unregister_algs(void) 1360 { 1361 struct n2_cipher_alg *cipher, *cipher_tmp; 1362 struct n2_ahash_alg *alg, *alg_tmp; 1363 struct n2_hmac_alg *hmac, *hmac_tmp; 1364 1365 list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) { 1366 crypto_unregister_alg(&cipher->alg); 1367 list_del(&cipher->entry); 1368 kfree(cipher); 1369 } 1370 list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { 1371 crypto_unregister_ahash(&hmac->derived.alg); 1372 list_del(&hmac->derived.entry); 1373 kfree(hmac); 1374 } 1375 list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { 1376 crypto_unregister_ahash(&alg->alg); 1377 list_del(&alg->entry); 1378 kfree(alg); 1379 } 1380 } 1381 1382 static int n2_cipher_cra_init(struct crypto_tfm *tfm) 1383 { 1384 tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context); 1385 return 0; 1386 } 1387 1388 static int __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl) 1389 { 1390 struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1391 struct crypto_alg *alg; 1392 int err; 1393 1394 if (!p) 1395 return -ENOMEM; 1396 1397 alg = &p->alg; 1398 1399 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1400 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name); 1401 alg->cra_priority = N2_CRA_PRIORITY; 1402 alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1403 CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; 1404 alg->cra_blocksize = tmpl->block_size; 1405 p->enc_type = tmpl->enc_type; 1406 alg->cra_ctxsize = sizeof(struct n2_cipher_context); 1407 alg->cra_type = &crypto_ablkcipher_type; 1408 alg->cra_u.ablkcipher = tmpl->ablkcipher; 1409 alg->cra_init = n2_cipher_cra_init; 1410 alg->cra_module = THIS_MODULE; 1411 1412 list_add(&p->entry, &cipher_algs); 1413 err = crypto_register_alg(alg); 1414 if (err) { 1415 pr_err("%s alg registration failed\n", alg->cra_name); 1416 list_del(&p->entry); 1417 kfree(p); 1418 } else { 1419 pr_info("%s alg registered\n", alg->cra_name); 1420 } 1421 return err; 1422 } 1423 1424 static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) 1425 { 1426 struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1427 struct ahash_alg *ahash; 1428 struct crypto_alg *base; 1429 int err; 1430 1431 if (!p) 1432 return -ENOMEM; 1433 1434 p->child_alg = n2ahash->alg.halg.base.cra_name; 1435 memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); 1436 INIT_LIST_HEAD(&p->derived.entry); 1437 1438 ahash = &p->derived.alg; 1439 ahash->digest = n2_hmac_async_digest; 1440 ahash->setkey = n2_hmac_async_setkey; 1441 1442 base = &ahash->halg.base; 1443 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg); 1444 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg); 1445 1446 base->cra_ctxsize = sizeof(struct n2_hmac_ctx); 1447 base->cra_init = n2_hmac_cra_init; 1448 base->cra_exit = n2_hmac_cra_exit; 1449 1450 list_add(&p->derived.entry, &hmac_algs); 1451 err = crypto_register_ahash(ahash); 1452 if (err) { 1453 pr_err("%s alg registration failed\n", base->cra_name); 1454 list_del(&p->derived.entry); 1455 kfree(p); 1456 } else { 1457 pr_info("%s alg registered\n", base->cra_name); 1458 } 1459 return err; 1460 } 1461 1462 static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) 1463 { 1464 struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1465 struct hash_alg_common *halg; 1466 struct crypto_alg *base; 1467 struct ahash_alg *ahash; 1468 int err; 1469 1470 if (!p) 1471 return -ENOMEM; 1472 1473 p->hash_zero = tmpl->hash_zero; 1474 p->hash_init = tmpl->hash_init; 1475 p->auth_type = tmpl->auth_type; 1476 p->hmac_type = tmpl->hmac_type; 1477 p->hw_op_hashsz = tmpl->hw_op_hashsz; 1478 p->digest_size = tmpl->digest_size; 1479 1480 ahash = &p->alg; 1481 ahash->init = n2_hash_async_init; 1482 ahash->update = n2_hash_async_update; 1483 ahash->final = n2_hash_async_final; 1484 ahash->finup = n2_hash_async_finup; 1485 ahash->digest = n2_hash_async_digest; 1486 1487 halg = &ahash->halg; 1488 halg->digestsize = tmpl->digest_size; 1489 1490 base = &halg->base; 1491 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1492 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name); 1493 base->cra_priority = N2_CRA_PRIORITY; 1494 base->cra_flags = CRYPTO_ALG_TYPE_AHASH | 1495 CRYPTO_ALG_KERN_DRIVER_ONLY | 1496 CRYPTO_ALG_NEED_FALLBACK; 1497 base->cra_blocksize = tmpl->block_size; 1498 base->cra_ctxsize = sizeof(struct n2_hash_ctx); 1499 base->cra_module = THIS_MODULE; 1500 base->cra_init = n2_hash_cra_init; 1501 base->cra_exit = n2_hash_cra_exit; 1502 1503 list_add(&p->entry, &ahash_algs); 1504 err = crypto_register_ahash(ahash); 1505 if (err) { 1506 pr_err("%s alg registration failed\n", base->cra_name); 1507 list_del(&p->entry); 1508 kfree(p); 1509 } else { 1510 pr_info("%s alg registered\n", base->cra_name); 1511 } 1512 if (!err && p->hmac_type != AUTH_TYPE_RESERVED) 1513 err = __n2_register_one_hmac(p); 1514 return err; 1515 } 1516 1517 static int n2_register_algs(void) 1518 { 1519 int i, err = 0; 1520 1521 mutex_lock(&spu_lock); 1522 if (algs_registered++) 1523 goto out; 1524 1525 for (i = 0; i < NUM_HASH_TMPLS; i++) { 1526 err = __n2_register_one_ahash(&hash_tmpls[i]); 1527 if (err) { 1528 __n2_unregister_algs(); 1529 goto out; 1530 } 1531 } 1532 for (i = 0; i < NUM_CIPHER_TMPLS; i++) { 1533 err = __n2_register_one_cipher(&cipher_tmpls[i]); 1534 if (err) { 1535 __n2_unregister_algs(); 1536 goto out; 1537 } 1538 } 1539 1540 out: 1541 mutex_unlock(&spu_lock); 1542 return err; 1543 } 1544 1545 static void n2_unregister_algs(void) 1546 { 1547 mutex_lock(&spu_lock); 1548 if (!--algs_registered) 1549 __n2_unregister_algs(); 1550 mutex_unlock(&spu_lock); 1551 } 1552 1553 /* To map CWQ queues to interrupt sources, the hypervisor API provides 1554 * a devino. This isn't very useful to us because all of the 1555 * interrupts listed in the device_node have been translated to 1556 * Linux virtual IRQ cookie numbers. 1557 * 1558 * So we have to back-translate, going through the 'intr' and 'ino' 1559 * property tables of the n2cp MDESC node, matching it with the OF 1560 * 'interrupts' property entries, in order to to figure out which 1561 * devino goes to which already-translated IRQ. 1562 */ 1563 static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, 1564 unsigned long dev_ino) 1565 { 1566 const unsigned int *dev_intrs; 1567 unsigned int intr; 1568 int i; 1569 1570 for (i = 0; i < ip->num_intrs; i++) { 1571 if (ip->ino_table[i].ino == dev_ino) 1572 break; 1573 } 1574 if (i == ip->num_intrs) 1575 return -ENODEV; 1576 1577 intr = ip->ino_table[i].intr; 1578 1579 dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL); 1580 if (!dev_intrs) 1581 return -ENODEV; 1582 1583 for (i = 0; i < dev->archdata.num_irqs; i++) { 1584 if (dev_intrs[i] == intr) 1585 return i; 1586 } 1587 1588 return -ENODEV; 1589 } 1590 1591 static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip, 1592 const char *irq_name, struct spu_queue *p, 1593 irq_handler_t handler) 1594 { 1595 unsigned long herr; 1596 int index; 1597 1598 herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino); 1599 if (herr) 1600 return -EINVAL; 1601 1602 index = find_devino_index(dev, ip, p->devino); 1603 if (index < 0) 1604 return index; 1605 1606 p->irq = dev->archdata.irqs[index]; 1607 1608 sprintf(p->irq_name, "%s-%d", irq_name, index); 1609 1610 return request_irq(p->irq, handler, 0, p->irq_name, p); 1611 } 1612 1613 static struct kmem_cache *queue_cache[2]; 1614 1615 static void *new_queue(unsigned long q_type) 1616 { 1617 return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL); 1618 } 1619 1620 static void free_queue(void *p, unsigned long q_type) 1621 { 1622 return kmem_cache_free(queue_cache[q_type - 1], p); 1623 } 1624 1625 static int queue_cache_init(void) 1626 { 1627 if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 1628 queue_cache[HV_NCS_QTYPE_MAU - 1] = 1629 kmem_cache_create("mau_queue", 1630 (MAU_NUM_ENTRIES * 1631 MAU_ENTRY_SIZE), 1632 MAU_ENTRY_SIZE, 0, NULL); 1633 if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 1634 return -ENOMEM; 1635 1636 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) 1637 queue_cache[HV_NCS_QTYPE_CWQ - 1] = 1638 kmem_cache_create("cwq_queue", 1639 (CWQ_NUM_ENTRIES * 1640 CWQ_ENTRY_SIZE), 1641 CWQ_ENTRY_SIZE, 0, NULL); 1642 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { 1643 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 1644 return -ENOMEM; 1645 } 1646 return 0; 1647 } 1648 1649 static void queue_cache_destroy(void) 1650 { 1651 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 1652 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]); 1653 } 1654 1655 static int spu_queue_register(struct spu_queue *p, unsigned long q_type) 1656 { 1657 cpumask_var_t old_allowed; 1658 unsigned long hv_ret; 1659 1660 if (cpumask_empty(&p->sharing)) 1661 return -EINVAL; 1662 1663 if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL)) 1664 return -ENOMEM; 1665 1666 cpumask_copy(old_allowed, ¤t->cpus_allowed); 1667 1668 set_cpus_allowed_ptr(current, &p->sharing); 1669 1670 hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q), 1671 CWQ_NUM_ENTRIES, &p->qhandle); 1672 if (!hv_ret) 1673 sun4v_ncs_sethead_marker(p->qhandle, 0); 1674 1675 set_cpus_allowed_ptr(current, old_allowed); 1676 1677 free_cpumask_var(old_allowed); 1678 1679 return (hv_ret ? -EINVAL : 0); 1680 } 1681 1682 static int spu_queue_setup(struct spu_queue *p) 1683 { 1684 int err; 1685 1686 p->q = new_queue(p->q_type); 1687 if (!p->q) 1688 return -ENOMEM; 1689 1690 err = spu_queue_register(p, p->q_type); 1691 if (err) { 1692 free_queue(p->q, p->q_type); 1693 p->q = NULL; 1694 } 1695 1696 return err; 1697 } 1698 1699 static void spu_queue_destroy(struct spu_queue *p) 1700 { 1701 unsigned long hv_ret; 1702 1703 if (!p->q) 1704 return; 1705 1706 hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle); 1707 1708 if (!hv_ret) 1709 free_queue(p->q, p->q_type); 1710 } 1711 1712 static void spu_list_destroy(struct list_head *list) 1713 { 1714 struct spu_queue *p, *n; 1715 1716 list_for_each_entry_safe(p, n, list, list) { 1717 int i; 1718 1719 for (i = 0; i < NR_CPUS; i++) { 1720 if (cpu_to_cwq[i] == p) 1721 cpu_to_cwq[i] = NULL; 1722 } 1723 1724 if (p->irq) { 1725 free_irq(p->irq, p); 1726 p->irq = 0; 1727 } 1728 spu_queue_destroy(p); 1729 list_del(&p->list); 1730 kfree(p); 1731 } 1732 } 1733 1734 /* Walk the backward arcs of a CWQ 'exec-unit' node, 1735 * gathering cpu membership information. 1736 */ 1737 static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, 1738 struct platform_device *dev, 1739 u64 node, struct spu_queue *p, 1740 struct spu_queue **table) 1741 { 1742 u64 arc; 1743 1744 mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { 1745 u64 tgt = mdesc_arc_target(mdesc, arc); 1746 const char *name = mdesc_node_name(mdesc, tgt); 1747 const u64 *id; 1748 1749 if (strcmp(name, "cpu")) 1750 continue; 1751 id = mdesc_get_property(mdesc, tgt, "id", NULL); 1752 if (table[*id] != NULL) { 1753 dev_err(&dev->dev, "%s: SPU cpu slot already set.\n", 1754 dev->dev.of_node->full_name); 1755 return -EINVAL; 1756 } 1757 cpu_set(*id, p->sharing); 1758 table[*id] = p; 1759 } 1760 return 0; 1761 } 1762 1763 /* Process an 'exec-unit' MDESC node of type 'cwq'. */ 1764 static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, 1765 struct platform_device *dev, struct mdesc_handle *mdesc, 1766 u64 node, const char *iname, unsigned long q_type, 1767 irq_handler_t handler, struct spu_queue **table) 1768 { 1769 struct spu_queue *p; 1770 int err; 1771 1772 p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); 1773 if (!p) { 1774 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n", 1775 dev->dev.of_node->full_name); 1776 return -ENOMEM; 1777 } 1778 1779 cpus_clear(p->sharing); 1780 spin_lock_init(&p->lock); 1781 p->q_type = q_type; 1782 INIT_LIST_HEAD(&p->jobs); 1783 list_add(&p->list, list); 1784 1785 err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); 1786 if (err) 1787 return err; 1788 1789 err = spu_queue_setup(p); 1790 if (err) 1791 return err; 1792 1793 return spu_map_ino(dev, ip, iname, p, handler); 1794 } 1795 1796 static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev, 1797 struct spu_mdesc_info *ip, struct list_head *list, 1798 const char *exec_name, unsigned long q_type, 1799 irq_handler_t handler, struct spu_queue **table) 1800 { 1801 int err = 0; 1802 u64 node; 1803 1804 mdesc_for_each_node_by_name(mdesc, node, "exec-unit") { 1805 const char *type; 1806 1807 type = mdesc_get_property(mdesc, node, "type", NULL); 1808 if (!type || strcmp(type, exec_name)) 1809 continue; 1810 1811 err = handle_exec_unit(ip, list, dev, mdesc, node, 1812 exec_name, q_type, handler, table); 1813 if (err) { 1814 spu_list_destroy(list); 1815 break; 1816 } 1817 } 1818 1819 return err; 1820 } 1821 1822 static int get_irq_props(struct mdesc_handle *mdesc, u64 node, 1823 struct spu_mdesc_info *ip) 1824 { 1825 const u64 *ino; 1826 int ino_len; 1827 int i; 1828 1829 ino = mdesc_get_property(mdesc, node, "ino", &ino_len); 1830 if (!ino) { 1831 printk("NO 'ino'\n"); 1832 return -ENODEV; 1833 } 1834 1835 ip->num_intrs = ino_len / sizeof(u64); 1836 ip->ino_table = kzalloc((sizeof(struct ino_blob) * 1837 ip->num_intrs), 1838 GFP_KERNEL); 1839 if (!ip->ino_table) 1840 return -ENOMEM; 1841 1842 for (i = 0; i < ip->num_intrs; i++) { 1843 struct ino_blob *b = &ip->ino_table[i]; 1844 b->intr = i + 1; 1845 b->ino = ino[i]; 1846 } 1847 1848 return 0; 1849 } 1850 1851 static int grab_mdesc_irq_props(struct mdesc_handle *mdesc, 1852 struct platform_device *dev, 1853 struct spu_mdesc_info *ip, 1854 const char *node_name) 1855 { 1856 const unsigned int *reg; 1857 u64 node; 1858 1859 reg = of_get_property(dev->dev.of_node, "reg", NULL); 1860 if (!reg) 1861 return -ENODEV; 1862 1863 mdesc_for_each_node_by_name(mdesc, node, "virtual-device") { 1864 const char *name; 1865 const u64 *chdl; 1866 1867 name = mdesc_get_property(mdesc, node, "name", NULL); 1868 if (!name || strcmp(name, node_name)) 1869 continue; 1870 chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL); 1871 if (!chdl || (*chdl != *reg)) 1872 continue; 1873 ip->cfg_handle = *chdl; 1874 return get_irq_props(mdesc, node, ip); 1875 } 1876 1877 return -ENODEV; 1878 } 1879 1880 static unsigned long n2_spu_hvapi_major; 1881 static unsigned long n2_spu_hvapi_minor; 1882 1883 static int n2_spu_hvapi_register(void) 1884 { 1885 int err; 1886 1887 n2_spu_hvapi_major = 2; 1888 n2_spu_hvapi_minor = 0; 1889 1890 err = sun4v_hvapi_register(HV_GRP_NCS, 1891 n2_spu_hvapi_major, 1892 &n2_spu_hvapi_minor); 1893 1894 if (!err) 1895 pr_info("Registered NCS HVAPI version %lu.%lu\n", 1896 n2_spu_hvapi_major, 1897 n2_spu_hvapi_minor); 1898 1899 return err; 1900 } 1901 1902 static void n2_spu_hvapi_unregister(void) 1903 { 1904 sun4v_hvapi_unregister(HV_GRP_NCS); 1905 } 1906 1907 static int global_ref; 1908 1909 static int grab_global_resources(void) 1910 { 1911 int err = 0; 1912 1913 mutex_lock(&spu_lock); 1914 1915 if (global_ref++) 1916 goto out; 1917 1918 err = n2_spu_hvapi_register(); 1919 if (err) 1920 goto out; 1921 1922 err = queue_cache_init(); 1923 if (err) 1924 goto out_hvapi_release; 1925 1926 err = -ENOMEM; 1927 cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS, 1928 GFP_KERNEL); 1929 if (!cpu_to_cwq) 1930 goto out_queue_cache_destroy; 1931 1932 cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS, 1933 GFP_KERNEL); 1934 if (!cpu_to_mau) 1935 goto out_free_cwq_table; 1936 1937 err = 0; 1938 1939 out: 1940 if (err) 1941 global_ref--; 1942 mutex_unlock(&spu_lock); 1943 return err; 1944 1945 out_free_cwq_table: 1946 kfree(cpu_to_cwq); 1947 cpu_to_cwq = NULL; 1948 1949 out_queue_cache_destroy: 1950 queue_cache_destroy(); 1951 1952 out_hvapi_release: 1953 n2_spu_hvapi_unregister(); 1954 goto out; 1955 } 1956 1957 static void release_global_resources(void) 1958 { 1959 mutex_lock(&spu_lock); 1960 if (!--global_ref) { 1961 kfree(cpu_to_cwq); 1962 cpu_to_cwq = NULL; 1963 1964 kfree(cpu_to_mau); 1965 cpu_to_mau = NULL; 1966 1967 queue_cache_destroy(); 1968 n2_spu_hvapi_unregister(); 1969 } 1970 mutex_unlock(&spu_lock); 1971 } 1972 1973 static struct n2_crypto *alloc_n2cp(void) 1974 { 1975 struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL); 1976 1977 if (np) 1978 INIT_LIST_HEAD(&np->cwq_list); 1979 1980 return np; 1981 } 1982 1983 static void free_n2cp(struct n2_crypto *np) 1984 { 1985 if (np->cwq_info.ino_table) { 1986 kfree(np->cwq_info.ino_table); 1987 np->cwq_info.ino_table = NULL; 1988 } 1989 1990 kfree(np); 1991 } 1992 1993 static void n2_spu_driver_version(void) 1994 { 1995 static int n2_spu_version_printed; 1996 1997 if (n2_spu_version_printed++ == 0) 1998 pr_info("%s", version); 1999 } 2000 2001 static int n2_crypto_probe(struct platform_device *dev) 2002 { 2003 struct mdesc_handle *mdesc; 2004 const char *full_name; 2005 struct n2_crypto *np; 2006 int err; 2007 2008 n2_spu_driver_version(); 2009 2010 full_name = dev->dev.of_node->full_name; 2011 pr_info("Found N2CP at %s\n", full_name); 2012 2013 np = alloc_n2cp(); 2014 if (!np) { 2015 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n", 2016 full_name); 2017 return -ENOMEM; 2018 } 2019 2020 err = grab_global_resources(); 2021 if (err) { 2022 dev_err(&dev->dev, "%s: Unable to grab " 2023 "global resources.\n", full_name); 2024 goto out_free_n2cp; 2025 } 2026 2027 mdesc = mdesc_grab(); 2028 2029 if (!mdesc) { 2030 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 2031 full_name); 2032 err = -ENODEV; 2033 goto out_free_global; 2034 } 2035 err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); 2036 if (err) { 2037 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 2038 full_name); 2039 mdesc_release(mdesc); 2040 goto out_free_global; 2041 } 2042 2043 err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list, 2044 "cwq", HV_NCS_QTYPE_CWQ, cwq_intr, 2045 cpu_to_cwq); 2046 mdesc_release(mdesc); 2047 2048 if (err) { 2049 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n", 2050 full_name); 2051 goto out_free_global; 2052 } 2053 2054 err = n2_register_algs(); 2055 if (err) { 2056 dev_err(&dev->dev, "%s: Unable to register algorithms.\n", 2057 full_name); 2058 goto out_free_spu_list; 2059 } 2060 2061 dev_set_drvdata(&dev->dev, np); 2062 2063 return 0; 2064 2065 out_free_spu_list: 2066 spu_list_destroy(&np->cwq_list); 2067 2068 out_free_global: 2069 release_global_resources(); 2070 2071 out_free_n2cp: 2072 free_n2cp(np); 2073 2074 return err; 2075 } 2076 2077 static int n2_crypto_remove(struct platform_device *dev) 2078 { 2079 struct n2_crypto *np = dev_get_drvdata(&dev->dev); 2080 2081 n2_unregister_algs(); 2082 2083 spu_list_destroy(&np->cwq_list); 2084 2085 release_global_resources(); 2086 2087 free_n2cp(np); 2088 2089 return 0; 2090 } 2091 2092 static struct n2_mau *alloc_ncp(void) 2093 { 2094 struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL); 2095 2096 if (mp) 2097 INIT_LIST_HEAD(&mp->mau_list); 2098 2099 return mp; 2100 } 2101 2102 static void free_ncp(struct n2_mau *mp) 2103 { 2104 if (mp->mau_info.ino_table) { 2105 kfree(mp->mau_info.ino_table); 2106 mp->mau_info.ino_table = NULL; 2107 } 2108 2109 kfree(mp); 2110 } 2111 2112 static int n2_mau_probe(struct platform_device *dev) 2113 { 2114 struct mdesc_handle *mdesc; 2115 const char *full_name; 2116 struct n2_mau *mp; 2117 int err; 2118 2119 n2_spu_driver_version(); 2120 2121 full_name = dev->dev.of_node->full_name; 2122 pr_info("Found NCP at %s\n", full_name); 2123 2124 mp = alloc_ncp(); 2125 if (!mp) { 2126 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n", 2127 full_name); 2128 return -ENOMEM; 2129 } 2130 2131 err = grab_global_resources(); 2132 if (err) { 2133 dev_err(&dev->dev, "%s: Unable to grab " 2134 "global resources.\n", full_name); 2135 goto out_free_ncp; 2136 } 2137 2138 mdesc = mdesc_grab(); 2139 2140 if (!mdesc) { 2141 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 2142 full_name); 2143 err = -ENODEV; 2144 goto out_free_global; 2145 } 2146 2147 err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); 2148 if (err) { 2149 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 2150 full_name); 2151 mdesc_release(mdesc); 2152 goto out_free_global; 2153 } 2154 2155 err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list, 2156 "mau", HV_NCS_QTYPE_MAU, mau_intr, 2157 cpu_to_mau); 2158 mdesc_release(mdesc); 2159 2160 if (err) { 2161 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n", 2162 full_name); 2163 goto out_free_global; 2164 } 2165 2166 dev_set_drvdata(&dev->dev, mp); 2167 2168 return 0; 2169 2170 out_free_global: 2171 release_global_resources(); 2172 2173 out_free_ncp: 2174 free_ncp(mp); 2175 2176 return err; 2177 } 2178 2179 static int n2_mau_remove(struct platform_device *dev) 2180 { 2181 struct n2_mau *mp = dev_get_drvdata(&dev->dev); 2182 2183 spu_list_destroy(&mp->mau_list); 2184 2185 release_global_resources(); 2186 2187 free_ncp(mp); 2188 2189 return 0; 2190 } 2191 2192 static struct of_device_id n2_crypto_match[] = { 2193 { 2194 .name = "n2cp", 2195 .compatible = "SUNW,n2-cwq", 2196 }, 2197 { 2198 .name = "n2cp", 2199 .compatible = "SUNW,vf-cwq", 2200 }, 2201 { 2202 .name = "n2cp", 2203 .compatible = "SUNW,kt-cwq", 2204 }, 2205 {}, 2206 }; 2207 2208 MODULE_DEVICE_TABLE(of, n2_crypto_match); 2209 2210 static struct platform_driver n2_crypto_driver = { 2211 .driver = { 2212 .name = "n2cp", 2213 .owner = THIS_MODULE, 2214 .of_match_table = n2_crypto_match, 2215 }, 2216 .probe = n2_crypto_probe, 2217 .remove = n2_crypto_remove, 2218 }; 2219 2220 static struct of_device_id n2_mau_match[] = { 2221 { 2222 .name = "ncp", 2223 .compatible = "SUNW,n2-mau", 2224 }, 2225 { 2226 .name = "ncp", 2227 .compatible = "SUNW,vf-mau", 2228 }, 2229 { 2230 .name = "ncp", 2231 .compatible = "SUNW,kt-mau", 2232 }, 2233 {}, 2234 }; 2235 2236 MODULE_DEVICE_TABLE(of, n2_mau_match); 2237 2238 static struct platform_driver n2_mau_driver = { 2239 .driver = { 2240 .name = "ncp", 2241 .owner = THIS_MODULE, 2242 .of_match_table = n2_mau_match, 2243 }, 2244 .probe = n2_mau_probe, 2245 .remove = n2_mau_remove, 2246 }; 2247 2248 static int __init n2_init(void) 2249 { 2250 int err = platform_driver_register(&n2_crypto_driver); 2251 2252 if (!err) { 2253 err = platform_driver_register(&n2_mau_driver); 2254 if (err) 2255 platform_driver_unregister(&n2_crypto_driver); 2256 } 2257 return err; 2258 } 2259 2260 static void __exit n2_exit(void) 2261 { 2262 platform_driver_unregister(&n2_mau_driver); 2263 platform_driver_unregister(&n2_crypto_driver); 2264 } 2265 2266 module_init(n2_init); 2267 module_exit(n2_exit); 2268