1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. 2 * 3 * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net> 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/of.h> 11 #include <linux/of_device.h> 12 #include <linux/cpumask.h> 13 #include <linux/slab.h> 14 #include <linux/interrupt.h> 15 #include <linux/crypto.h> 16 #include <crypto/md5.h> 17 #include <crypto/sha.h> 18 #include <crypto/aes.h> 19 #include <crypto/des.h> 20 #include <linux/mutex.h> 21 #include <linux/delay.h> 22 #include <linux/sched.h> 23 24 #include <crypto/internal/hash.h> 25 #include <crypto/scatterwalk.h> 26 #include <crypto/algapi.h> 27 28 #include <asm/hypervisor.h> 29 #include <asm/mdesc.h> 30 31 #include "n2_core.h" 32 33 #define DRV_MODULE_NAME "n2_crypto" 34 #define DRV_MODULE_VERSION "0.2" 35 #define DRV_MODULE_RELDATE "July 28, 2011" 36 37 static const char version[] = 38 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 39 40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 41 MODULE_DESCRIPTION("Niagara2 Crypto driver"); 42 MODULE_LICENSE("GPL"); 43 MODULE_VERSION(DRV_MODULE_VERSION); 44 45 #define N2_CRA_PRIORITY 200 46 47 static DEFINE_MUTEX(spu_lock); 48 49 struct spu_queue { 50 cpumask_t sharing; 51 unsigned long qhandle; 52 53 spinlock_t lock; 54 u8 q_type; 55 void *q; 56 unsigned long head; 57 unsigned long tail; 58 struct list_head jobs; 59 60 unsigned long devino; 61 62 char irq_name[32]; 63 unsigned int irq; 64 65 struct list_head list; 66 }; 67 68 static struct spu_queue **cpu_to_cwq; 69 static struct spu_queue **cpu_to_mau; 70 71 static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) 72 { 73 if (q->q_type == HV_NCS_QTYPE_MAU) { 74 off += MAU_ENTRY_SIZE; 75 if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) 76 off = 0; 77 } else { 78 off += CWQ_ENTRY_SIZE; 79 if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) 80 off = 0; 81 } 82 return off; 83 } 84 85 struct n2_request_common { 86 struct list_head entry; 87 unsigned int offset; 88 }; 89 #define OFFSET_NOT_RUNNING (~(unsigned int)0) 90 91 /* An async job request records the final tail value it used in 92 * n2_request_common->offset, test to see if that offset is in 93 * the range old_head, new_head, inclusive. 94 */ 95 static inline bool job_finished(struct spu_queue *q, unsigned int offset, 96 unsigned long old_head, unsigned long new_head) 97 { 98 if (old_head <= new_head) { 99 if (offset > old_head && offset <= new_head) 100 return true; 101 } else { 102 if (offset > old_head || offset <= new_head) 103 return true; 104 } 105 return false; 106 } 107 108 /* When the HEAD marker is unequal to the actual HEAD, we get 109 * a virtual device INO interrupt. We should process the 110 * completed CWQ entries and adjust the HEAD marker to clear 111 * the IRQ. 112 */ 113 static irqreturn_t cwq_intr(int irq, void *dev_id) 114 { 115 unsigned long off, new_head, hv_ret; 116 struct spu_queue *q = dev_id; 117 118 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n", 119 smp_processor_id(), q->qhandle); 120 121 spin_lock(&q->lock); 122 123 hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head); 124 125 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n", 126 smp_processor_id(), new_head, hv_ret); 127 128 for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { 129 /* XXX ... XXX */ 130 } 131 132 hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head); 133 if (hv_ret == HV_EOK) 134 q->head = new_head; 135 136 spin_unlock(&q->lock); 137 138 return IRQ_HANDLED; 139 } 140 141 static irqreturn_t mau_intr(int irq, void *dev_id) 142 { 143 struct spu_queue *q = dev_id; 144 unsigned long head, hv_ret; 145 146 spin_lock(&q->lock); 147 148 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n", 149 smp_processor_id(), q->qhandle); 150 151 hv_ret = sun4v_ncs_gethead(q->qhandle, &head); 152 153 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n", 154 smp_processor_id(), head, hv_ret); 155 156 sun4v_ncs_sethead_marker(q->qhandle, head); 157 158 spin_unlock(&q->lock); 159 160 return IRQ_HANDLED; 161 } 162 163 static void *spu_queue_next(struct spu_queue *q, void *cur) 164 { 165 return q->q + spu_next_offset(q, cur - q->q); 166 } 167 168 static int spu_queue_num_free(struct spu_queue *q) 169 { 170 unsigned long head = q->head; 171 unsigned long tail = q->tail; 172 unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); 173 unsigned long diff; 174 175 if (head > tail) 176 diff = head - tail; 177 else 178 diff = (end - tail) + head; 179 180 return (diff / CWQ_ENTRY_SIZE) - 1; 181 } 182 183 static void *spu_queue_alloc(struct spu_queue *q, int num_entries) 184 { 185 int avail = spu_queue_num_free(q); 186 187 if (avail >= num_entries) 188 return q->q + q->tail; 189 190 return NULL; 191 } 192 193 static unsigned long spu_queue_submit(struct spu_queue *q, void *last) 194 { 195 unsigned long hv_ret, new_tail; 196 197 new_tail = spu_next_offset(q, last - q->q); 198 199 hv_ret = sun4v_ncs_settail(q->qhandle, new_tail); 200 if (hv_ret == HV_EOK) 201 q->tail = new_tail; 202 return hv_ret; 203 } 204 205 static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, 206 int enc_type, int auth_type, 207 unsigned int hash_len, 208 bool sfas, bool sob, bool eob, bool encrypt, 209 int opcode) 210 { 211 u64 word = (len - 1) & CONTROL_LEN; 212 213 word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); 214 word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); 215 word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); 216 if (sfas) 217 word |= CONTROL_STORE_FINAL_AUTH_STATE; 218 if (sob) 219 word |= CONTROL_START_OF_BLOCK; 220 if (eob) 221 word |= CONTROL_END_OF_BLOCK; 222 if (encrypt) 223 word |= CONTROL_ENCRYPT; 224 if (hmac_key_len) 225 word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; 226 if (hash_len) 227 word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; 228 229 return word; 230 } 231 232 #if 0 233 static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) 234 { 235 if (this_len >= 64 || 236 qp->head != qp->tail) 237 return true; 238 return false; 239 } 240 #endif 241 242 struct n2_ahash_alg { 243 struct list_head entry; 244 const u8 *hash_zero; 245 const u32 *hash_init; 246 u8 hw_op_hashsz; 247 u8 digest_size; 248 u8 auth_type; 249 u8 hmac_type; 250 struct ahash_alg alg; 251 }; 252 253 static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) 254 { 255 struct crypto_alg *alg = tfm->__crt_alg; 256 struct ahash_alg *ahash_alg; 257 258 ahash_alg = container_of(alg, struct ahash_alg, halg.base); 259 260 return container_of(ahash_alg, struct n2_ahash_alg, alg); 261 } 262 263 struct n2_hmac_alg { 264 const char *child_alg; 265 struct n2_ahash_alg derived; 266 }; 267 268 static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) 269 { 270 struct crypto_alg *alg = tfm->__crt_alg; 271 struct ahash_alg *ahash_alg; 272 273 ahash_alg = container_of(alg, struct ahash_alg, halg.base); 274 275 return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); 276 } 277 278 struct n2_hash_ctx { 279 struct crypto_ahash *fallback_tfm; 280 }; 281 282 #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ 283 284 struct n2_hmac_ctx { 285 struct n2_hash_ctx base; 286 287 struct crypto_shash *child_shash; 288 289 int hash_key_len; 290 unsigned char hash_key[N2_HASH_KEY_MAX]; 291 }; 292 293 struct n2_hash_req_ctx { 294 union { 295 struct md5_state md5; 296 struct sha1_state sha1; 297 struct sha256_state sha256; 298 } u; 299 300 struct ahash_request fallback_req; 301 }; 302 303 static int n2_hash_async_init(struct ahash_request *req) 304 { 305 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 306 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 307 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 308 309 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 310 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 311 312 return crypto_ahash_init(&rctx->fallback_req); 313 } 314 315 static int n2_hash_async_update(struct ahash_request *req) 316 { 317 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 318 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 319 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 320 321 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 322 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 323 rctx->fallback_req.nbytes = req->nbytes; 324 rctx->fallback_req.src = req->src; 325 326 return crypto_ahash_update(&rctx->fallback_req); 327 } 328 329 static int n2_hash_async_final(struct ahash_request *req) 330 { 331 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 332 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 333 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 334 335 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 336 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 337 rctx->fallback_req.result = req->result; 338 339 return crypto_ahash_final(&rctx->fallback_req); 340 } 341 342 static int n2_hash_async_finup(struct ahash_request *req) 343 { 344 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 346 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 347 348 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 349 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 350 rctx->fallback_req.nbytes = req->nbytes; 351 rctx->fallback_req.src = req->src; 352 rctx->fallback_req.result = req->result; 353 354 return crypto_ahash_finup(&rctx->fallback_req); 355 } 356 357 static int n2_hash_cra_init(struct crypto_tfm *tfm) 358 { 359 const char *fallback_driver_name = crypto_tfm_alg_name(tfm); 360 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 361 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 362 struct crypto_ahash *fallback_tfm; 363 int err; 364 365 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 366 CRYPTO_ALG_NEED_FALLBACK); 367 if (IS_ERR(fallback_tfm)) { 368 pr_warning("Fallback driver '%s' could not be loaded!\n", 369 fallback_driver_name); 370 err = PTR_ERR(fallback_tfm); 371 goto out; 372 } 373 374 crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 375 crypto_ahash_reqsize(fallback_tfm))); 376 377 ctx->fallback_tfm = fallback_tfm; 378 return 0; 379 380 out: 381 return err; 382 } 383 384 static void n2_hash_cra_exit(struct crypto_tfm *tfm) 385 { 386 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 387 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 388 389 crypto_free_ahash(ctx->fallback_tfm); 390 } 391 392 static int n2_hmac_cra_init(struct crypto_tfm *tfm) 393 { 394 const char *fallback_driver_name = crypto_tfm_alg_name(tfm); 395 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 396 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); 397 struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); 398 struct crypto_ahash *fallback_tfm; 399 struct crypto_shash *child_shash; 400 int err; 401 402 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 403 CRYPTO_ALG_NEED_FALLBACK); 404 if (IS_ERR(fallback_tfm)) { 405 pr_warning("Fallback driver '%s' could not be loaded!\n", 406 fallback_driver_name); 407 err = PTR_ERR(fallback_tfm); 408 goto out; 409 } 410 411 child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0); 412 if (IS_ERR(child_shash)) { 413 pr_warning("Child shash '%s' could not be loaded!\n", 414 n2alg->child_alg); 415 err = PTR_ERR(child_shash); 416 goto out_free_fallback; 417 } 418 419 crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 420 crypto_ahash_reqsize(fallback_tfm))); 421 422 ctx->child_shash = child_shash; 423 ctx->base.fallback_tfm = fallback_tfm; 424 return 0; 425 426 out_free_fallback: 427 crypto_free_ahash(fallback_tfm); 428 429 out: 430 return err; 431 } 432 433 static void n2_hmac_cra_exit(struct crypto_tfm *tfm) 434 { 435 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 436 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); 437 438 crypto_free_ahash(ctx->base.fallback_tfm); 439 crypto_free_shash(ctx->child_shash); 440 } 441 442 static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, 443 unsigned int keylen) 444 { 445 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); 446 struct crypto_shash *child_shash = ctx->child_shash; 447 struct crypto_ahash *fallback_tfm; 448 SHASH_DESC_ON_STACK(shash, child_shash); 449 int err, bs, ds; 450 451 fallback_tfm = ctx->base.fallback_tfm; 452 err = crypto_ahash_setkey(fallback_tfm, key, keylen); 453 if (err) 454 return err; 455 456 shash->tfm = child_shash; 457 shash->flags = crypto_ahash_get_flags(tfm) & 458 CRYPTO_TFM_REQ_MAY_SLEEP; 459 460 bs = crypto_shash_blocksize(child_shash); 461 ds = crypto_shash_digestsize(child_shash); 462 BUG_ON(ds > N2_HASH_KEY_MAX); 463 if (keylen > bs) { 464 err = crypto_shash_digest(shash, key, keylen, 465 ctx->hash_key); 466 if (err) 467 return err; 468 keylen = ds; 469 } else if (keylen <= N2_HASH_KEY_MAX) 470 memcpy(ctx->hash_key, key, keylen); 471 472 ctx->hash_key_len = keylen; 473 474 return err; 475 } 476 477 static unsigned long wait_for_tail(struct spu_queue *qp) 478 { 479 unsigned long head, hv_ret; 480 481 do { 482 hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); 483 if (hv_ret != HV_EOK) { 484 pr_err("Hypervisor error on gethead\n"); 485 break; 486 } 487 if (head == qp->tail) { 488 qp->head = head; 489 break; 490 } 491 } while (1); 492 return hv_ret; 493 } 494 495 static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, 496 struct cwq_initial_entry *ent) 497 { 498 unsigned long hv_ret = spu_queue_submit(qp, ent); 499 500 if (hv_ret == HV_EOK) 501 hv_ret = wait_for_tail(qp); 502 503 return hv_ret; 504 } 505 506 static int n2_do_async_digest(struct ahash_request *req, 507 unsigned int auth_type, unsigned int digest_size, 508 unsigned int result_size, void *hash_loc, 509 unsigned long auth_key, unsigned int auth_key_len) 510 { 511 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 512 struct cwq_initial_entry *ent; 513 struct crypto_hash_walk walk; 514 struct spu_queue *qp; 515 unsigned long flags; 516 int err = -ENODEV; 517 int nbytes, cpu; 518 519 /* The total effective length of the operation may not 520 * exceed 2^16. 521 */ 522 if (unlikely(req->nbytes > (1 << 16))) { 523 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 524 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 525 526 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 527 rctx->fallback_req.base.flags = 528 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 529 rctx->fallback_req.nbytes = req->nbytes; 530 rctx->fallback_req.src = req->src; 531 rctx->fallback_req.result = req->result; 532 533 return crypto_ahash_digest(&rctx->fallback_req); 534 } 535 536 nbytes = crypto_hash_walk_first(req, &walk); 537 538 cpu = get_cpu(); 539 qp = cpu_to_cwq[cpu]; 540 if (!qp) 541 goto out; 542 543 spin_lock_irqsave(&qp->lock, flags); 544 545 /* XXX can do better, improve this later by doing a by-hand scatterlist 546 * XXX walk, etc. 547 */ 548 ent = qp->q + qp->tail; 549 550 ent->control = control_word_base(nbytes, auth_key_len, 0, 551 auth_type, digest_size, 552 false, true, false, false, 553 OPCODE_INPLACE_BIT | 554 OPCODE_AUTH_MAC); 555 ent->src_addr = __pa(walk.data); 556 ent->auth_key_addr = auth_key; 557 ent->auth_iv_addr = __pa(hash_loc); 558 ent->final_auth_state_addr = 0UL; 559 ent->enc_key_addr = 0UL; 560 ent->enc_iv_addr = 0UL; 561 ent->dest_addr = __pa(hash_loc); 562 563 nbytes = crypto_hash_walk_done(&walk, 0); 564 while (nbytes > 0) { 565 ent = spu_queue_next(qp, ent); 566 567 ent->control = (nbytes - 1); 568 ent->src_addr = __pa(walk.data); 569 ent->auth_key_addr = 0UL; 570 ent->auth_iv_addr = 0UL; 571 ent->final_auth_state_addr = 0UL; 572 ent->enc_key_addr = 0UL; 573 ent->enc_iv_addr = 0UL; 574 ent->dest_addr = 0UL; 575 576 nbytes = crypto_hash_walk_done(&walk, 0); 577 } 578 ent->control |= CONTROL_END_OF_BLOCK; 579 580 if (submit_and_wait_for_tail(qp, ent) != HV_EOK) 581 err = -EINVAL; 582 else 583 err = 0; 584 585 spin_unlock_irqrestore(&qp->lock, flags); 586 587 if (!err) 588 memcpy(req->result, hash_loc, result_size); 589 out: 590 put_cpu(); 591 592 return err; 593 } 594 595 static int n2_hash_async_digest(struct ahash_request *req) 596 { 597 struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm); 598 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 599 int ds; 600 601 ds = n2alg->digest_size; 602 if (unlikely(req->nbytes == 0)) { 603 memcpy(req->result, n2alg->hash_zero, ds); 604 return 0; 605 } 606 memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); 607 608 return n2_do_async_digest(req, n2alg->auth_type, 609 n2alg->hw_op_hashsz, ds, 610 &rctx->u, 0UL, 0); 611 } 612 613 static int n2_hmac_async_digest(struct ahash_request *req) 614 { 615 struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm); 616 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 617 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 618 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); 619 int ds; 620 621 ds = n2alg->derived.digest_size; 622 if (unlikely(req->nbytes == 0) || 623 unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { 624 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 625 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 626 627 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 628 rctx->fallback_req.base.flags = 629 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 630 rctx->fallback_req.nbytes = req->nbytes; 631 rctx->fallback_req.src = req->src; 632 rctx->fallback_req.result = req->result; 633 634 return crypto_ahash_digest(&rctx->fallback_req); 635 } 636 memcpy(&rctx->u, n2alg->derived.hash_init, 637 n2alg->derived.hw_op_hashsz); 638 639 return n2_do_async_digest(req, n2alg->derived.hmac_type, 640 n2alg->derived.hw_op_hashsz, ds, 641 &rctx->u, 642 __pa(&ctx->hash_key), 643 ctx->hash_key_len); 644 } 645 646 struct n2_cipher_context { 647 int key_len; 648 int enc_type; 649 union { 650 u8 aes[AES_MAX_KEY_SIZE]; 651 u8 des[DES_KEY_SIZE]; 652 u8 des3[3 * DES_KEY_SIZE]; 653 u8 arc4[258]; /* S-box, X, Y */ 654 } key; 655 }; 656 657 #define N2_CHUNK_ARR_LEN 16 658 659 struct n2_crypto_chunk { 660 struct list_head entry; 661 unsigned long iv_paddr : 44; 662 unsigned long arr_len : 20; 663 unsigned long dest_paddr; 664 unsigned long dest_final; 665 struct { 666 unsigned long src_paddr : 44; 667 unsigned long src_len : 20; 668 } arr[N2_CHUNK_ARR_LEN]; 669 }; 670 671 struct n2_request_context { 672 struct ablkcipher_walk walk; 673 struct list_head chunk_list; 674 struct n2_crypto_chunk chunk; 675 u8 temp_iv[16]; 676 }; 677 678 /* The SPU allows some level of flexibility for partial cipher blocks 679 * being specified in a descriptor. 680 * 681 * It merely requires that every descriptor's length field is at least 682 * as large as the cipher block size. This means that a cipher block 683 * can span at most 2 descriptors. However, this does not allow a 684 * partial block to span into the final descriptor as that would 685 * violate the rule (since every descriptor's length must be at lest 686 * the block size). So, for example, assuming an 8 byte block size: 687 * 688 * 0xe --> 0xa --> 0x8 689 * 690 * is a valid length sequence, whereas: 691 * 692 * 0xe --> 0xb --> 0x7 693 * 694 * is not a valid sequence. 695 */ 696 697 struct n2_cipher_alg { 698 struct list_head entry; 699 u8 enc_type; 700 struct crypto_alg alg; 701 }; 702 703 static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm) 704 { 705 struct crypto_alg *alg = tfm->__crt_alg; 706 707 return container_of(alg, struct n2_cipher_alg, alg); 708 } 709 710 struct n2_cipher_request_context { 711 struct ablkcipher_walk walk; 712 }; 713 714 static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 715 unsigned int keylen) 716 { 717 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 718 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 719 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 720 721 ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); 722 723 switch (keylen) { 724 case AES_KEYSIZE_128: 725 ctx->enc_type |= ENC_TYPE_ALG_AES128; 726 break; 727 case AES_KEYSIZE_192: 728 ctx->enc_type |= ENC_TYPE_ALG_AES192; 729 break; 730 case AES_KEYSIZE_256: 731 ctx->enc_type |= ENC_TYPE_ALG_AES256; 732 break; 733 default: 734 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 735 return -EINVAL; 736 } 737 738 ctx->key_len = keylen; 739 memcpy(ctx->key.aes, key, keylen); 740 return 0; 741 } 742 743 static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 744 unsigned int keylen) 745 { 746 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 747 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 748 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 749 u32 tmp[DES_EXPKEY_WORDS]; 750 int err; 751 752 ctx->enc_type = n2alg->enc_type; 753 754 if (keylen != DES_KEY_SIZE) { 755 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 756 return -EINVAL; 757 } 758 759 err = des_ekey(tmp, key); 760 if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 761 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; 762 return -EINVAL; 763 } 764 765 ctx->key_len = keylen; 766 memcpy(ctx->key.des, key, keylen); 767 return 0; 768 } 769 770 static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 771 unsigned int keylen) 772 { 773 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 774 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 775 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 776 777 ctx->enc_type = n2alg->enc_type; 778 779 if (keylen != (3 * DES_KEY_SIZE)) { 780 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 781 return -EINVAL; 782 } 783 ctx->key_len = keylen; 784 memcpy(ctx->key.des3, key, keylen); 785 return 0; 786 } 787 788 static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 789 unsigned int keylen) 790 { 791 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 792 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 793 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 794 u8 *s = ctx->key.arc4; 795 u8 *x = s + 256; 796 u8 *y = x + 1; 797 int i, j, k; 798 799 ctx->enc_type = n2alg->enc_type; 800 801 j = k = 0; 802 *x = 0; 803 *y = 0; 804 for (i = 0; i < 256; i++) 805 s[i] = i; 806 for (i = 0; i < 256; i++) { 807 u8 a = s[i]; 808 j = (j + key[k] + a) & 0xff; 809 s[i] = s[j]; 810 s[j] = a; 811 if (++k >= keylen) 812 k = 0; 813 } 814 815 return 0; 816 } 817 818 static inline int cipher_descriptor_len(int nbytes, unsigned int block_size) 819 { 820 int this_len = nbytes; 821 822 this_len -= (nbytes & (block_size - 1)); 823 return this_len > (1 << 16) ? (1 << 16) : this_len; 824 } 825 826 static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp, 827 struct spu_queue *qp, bool encrypt) 828 { 829 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 830 struct cwq_initial_entry *ent; 831 bool in_place; 832 int i; 833 834 ent = spu_queue_alloc(qp, cp->arr_len); 835 if (!ent) { 836 pr_info("queue_alloc() of %d fails\n", 837 cp->arr_len); 838 return -EBUSY; 839 } 840 841 in_place = (cp->dest_paddr == cp->arr[0].src_paddr); 842 843 ent->control = control_word_base(cp->arr[0].src_len, 844 0, ctx->enc_type, 0, 0, 845 false, true, false, encrypt, 846 OPCODE_ENCRYPT | 847 (in_place ? OPCODE_INPLACE_BIT : 0)); 848 ent->src_addr = cp->arr[0].src_paddr; 849 ent->auth_key_addr = 0UL; 850 ent->auth_iv_addr = 0UL; 851 ent->final_auth_state_addr = 0UL; 852 ent->enc_key_addr = __pa(&ctx->key); 853 ent->enc_iv_addr = cp->iv_paddr; 854 ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); 855 856 for (i = 1; i < cp->arr_len; i++) { 857 ent = spu_queue_next(qp, ent); 858 859 ent->control = cp->arr[i].src_len - 1; 860 ent->src_addr = cp->arr[i].src_paddr; 861 ent->auth_key_addr = 0UL; 862 ent->auth_iv_addr = 0UL; 863 ent->final_auth_state_addr = 0UL; 864 ent->enc_key_addr = 0UL; 865 ent->enc_iv_addr = 0UL; 866 ent->dest_addr = 0UL; 867 } 868 ent->control |= CONTROL_END_OF_BLOCK; 869 870 return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; 871 } 872 873 static int n2_compute_chunks(struct ablkcipher_request *req) 874 { 875 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 876 struct ablkcipher_walk *walk = &rctx->walk; 877 struct n2_crypto_chunk *chunk; 878 unsigned long dest_prev; 879 unsigned int tot_len; 880 bool prev_in_place; 881 int err, nbytes; 882 883 ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes); 884 err = ablkcipher_walk_phys(req, walk); 885 if (err) 886 return err; 887 888 INIT_LIST_HEAD(&rctx->chunk_list); 889 890 chunk = &rctx->chunk; 891 INIT_LIST_HEAD(&chunk->entry); 892 893 chunk->iv_paddr = 0UL; 894 chunk->arr_len = 0; 895 chunk->dest_paddr = 0UL; 896 897 prev_in_place = false; 898 dest_prev = ~0UL; 899 tot_len = 0; 900 901 while ((nbytes = walk->nbytes) != 0) { 902 unsigned long dest_paddr, src_paddr; 903 bool in_place; 904 int this_len; 905 906 src_paddr = (page_to_phys(walk->src.page) + 907 walk->src.offset); 908 dest_paddr = (page_to_phys(walk->dst.page) + 909 walk->dst.offset); 910 in_place = (src_paddr == dest_paddr); 911 this_len = cipher_descriptor_len(nbytes, walk->blocksize); 912 913 if (chunk->arr_len != 0) { 914 if (in_place != prev_in_place || 915 (!prev_in_place && 916 dest_paddr != dest_prev) || 917 chunk->arr_len == N2_CHUNK_ARR_LEN || 918 tot_len + this_len > (1 << 16)) { 919 chunk->dest_final = dest_prev; 920 list_add_tail(&chunk->entry, 921 &rctx->chunk_list); 922 chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC); 923 if (!chunk) { 924 err = -ENOMEM; 925 break; 926 } 927 INIT_LIST_HEAD(&chunk->entry); 928 } 929 } 930 if (chunk->arr_len == 0) { 931 chunk->dest_paddr = dest_paddr; 932 tot_len = 0; 933 } 934 chunk->arr[chunk->arr_len].src_paddr = src_paddr; 935 chunk->arr[chunk->arr_len].src_len = this_len; 936 chunk->arr_len++; 937 938 dest_prev = dest_paddr + this_len; 939 prev_in_place = in_place; 940 tot_len += this_len; 941 942 err = ablkcipher_walk_done(req, walk, nbytes - this_len); 943 if (err) 944 break; 945 } 946 if (!err && chunk->arr_len != 0) { 947 chunk->dest_final = dest_prev; 948 list_add_tail(&chunk->entry, &rctx->chunk_list); 949 } 950 951 return err; 952 } 953 954 static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv) 955 { 956 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 957 struct n2_crypto_chunk *c, *tmp; 958 959 if (final_iv) 960 memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); 961 962 ablkcipher_walk_complete(&rctx->walk); 963 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 964 list_del(&c->entry); 965 if (unlikely(c != &rctx->chunk)) 966 kfree(c); 967 } 968 969 } 970 971 static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt) 972 { 973 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 974 struct crypto_tfm *tfm = req->base.tfm; 975 int err = n2_compute_chunks(req); 976 struct n2_crypto_chunk *c, *tmp; 977 unsigned long flags, hv_ret; 978 struct spu_queue *qp; 979 980 if (err) 981 return err; 982 983 qp = cpu_to_cwq[get_cpu()]; 984 err = -ENODEV; 985 if (!qp) 986 goto out; 987 988 spin_lock_irqsave(&qp->lock, flags); 989 990 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 991 err = __n2_crypt_chunk(tfm, c, qp, encrypt); 992 if (err) 993 break; 994 list_del(&c->entry); 995 if (unlikely(c != &rctx->chunk)) 996 kfree(c); 997 } 998 if (!err) { 999 hv_ret = wait_for_tail(qp); 1000 if (hv_ret != HV_EOK) 1001 err = -EINVAL; 1002 } 1003 1004 spin_unlock_irqrestore(&qp->lock, flags); 1005 1006 out: 1007 put_cpu(); 1008 1009 n2_chunk_complete(req, NULL); 1010 return err; 1011 } 1012 1013 static int n2_encrypt_ecb(struct ablkcipher_request *req) 1014 { 1015 return n2_do_ecb(req, true); 1016 } 1017 1018 static int n2_decrypt_ecb(struct ablkcipher_request *req) 1019 { 1020 return n2_do_ecb(req, false); 1021 } 1022 1023 static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt) 1024 { 1025 struct n2_request_context *rctx = ablkcipher_request_ctx(req); 1026 struct crypto_tfm *tfm = req->base.tfm; 1027 unsigned long flags, hv_ret, iv_paddr; 1028 int err = n2_compute_chunks(req); 1029 struct n2_crypto_chunk *c, *tmp; 1030 struct spu_queue *qp; 1031 void *final_iv_addr; 1032 1033 final_iv_addr = NULL; 1034 1035 if (err) 1036 return err; 1037 1038 qp = cpu_to_cwq[get_cpu()]; 1039 err = -ENODEV; 1040 if (!qp) 1041 goto out; 1042 1043 spin_lock_irqsave(&qp->lock, flags); 1044 1045 if (encrypt) { 1046 iv_paddr = __pa(rctx->walk.iv); 1047 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, 1048 entry) { 1049 c->iv_paddr = iv_paddr; 1050 err = __n2_crypt_chunk(tfm, c, qp, true); 1051 if (err) 1052 break; 1053 iv_paddr = c->dest_final - rctx->walk.blocksize; 1054 list_del(&c->entry); 1055 if (unlikely(c != &rctx->chunk)) 1056 kfree(c); 1057 } 1058 final_iv_addr = __va(iv_paddr); 1059 } else { 1060 list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, 1061 entry) { 1062 if (c == &rctx->chunk) { 1063 iv_paddr = __pa(rctx->walk.iv); 1064 } else { 1065 iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + 1066 tmp->arr[tmp->arr_len-1].src_len - 1067 rctx->walk.blocksize); 1068 } 1069 if (!final_iv_addr) { 1070 unsigned long pa; 1071 1072 pa = (c->arr[c->arr_len-1].src_paddr + 1073 c->arr[c->arr_len-1].src_len - 1074 rctx->walk.blocksize); 1075 final_iv_addr = rctx->temp_iv; 1076 memcpy(rctx->temp_iv, __va(pa), 1077 rctx->walk.blocksize); 1078 } 1079 c->iv_paddr = iv_paddr; 1080 err = __n2_crypt_chunk(tfm, c, qp, false); 1081 if (err) 1082 break; 1083 list_del(&c->entry); 1084 if (unlikely(c != &rctx->chunk)) 1085 kfree(c); 1086 } 1087 } 1088 if (!err) { 1089 hv_ret = wait_for_tail(qp); 1090 if (hv_ret != HV_EOK) 1091 err = -EINVAL; 1092 } 1093 1094 spin_unlock_irqrestore(&qp->lock, flags); 1095 1096 out: 1097 put_cpu(); 1098 1099 n2_chunk_complete(req, err ? NULL : final_iv_addr); 1100 return err; 1101 } 1102 1103 static int n2_encrypt_chaining(struct ablkcipher_request *req) 1104 { 1105 return n2_do_chaining(req, true); 1106 } 1107 1108 static int n2_decrypt_chaining(struct ablkcipher_request *req) 1109 { 1110 return n2_do_chaining(req, false); 1111 } 1112 1113 struct n2_cipher_tmpl { 1114 const char *name; 1115 const char *drv_name; 1116 u8 block_size; 1117 u8 enc_type; 1118 struct ablkcipher_alg ablkcipher; 1119 }; 1120 1121 static const struct n2_cipher_tmpl cipher_tmpls[] = { 1122 /* ARC4: only ECB is supported (chaining bits ignored) */ 1123 { .name = "ecb(arc4)", 1124 .drv_name = "ecb-arc4", 1125 .block_size = 1, 1126 .enc_type = (ENC_TYPE_ALG_RC4_STREAM | 1127 ENC_TYPE_CHAINING_ECB), 1128 .ablkcipher = { 1129 .min_keysize = 1, 1130 .max_keysize = 256, 1131 .setkey = n2_arc4_setkey, 1132 .encrypt = n2_encrypt_ecb, 1133 .decrypt = n2_decrypt_ecb, 1134 }, 1135 }, 1136 1137 /* DES: ECB CBC and CFB are supported */ 1138 { .name = "ecb(des)", 1139 .drv_name = "ecb-des", 1140 .block_size = DES_BLOCK_SIZE, 1141 .enc_type = (ENC_TYPE_ALG_DES | 1142 ENC_TYPE_CHAINING_ECB), 1143 .ablkcipher = { 1144 .min_keysize = DES_KEY_SIZE, 1145 .max_keysize = DES_KEY_SIZE, 1146 .setkey = n2_des_setkey, 1147 .encrypt = n2_encrypt_ecb, 1148 .decrypt = n2_decrypt_ecb, 1149 }, 1150 }, 1151 { .name = "cbc(des)", 1152 .drv_name = "cbc-des", 1153 .block_size = DES_BLOCK_SIZE, 1154 .enc_type = (ENC_TYPE_ALG_DES | 1155 ENC_TYPE_CHAINING_CBC), 1156 .ablkcipher = { 1157 .ivsize = DES_BLOCK_SIZE, 1158 .min_keysize = DES_KEY_SIZE, 1159 .max_keysize = DES_KEY_SIZE, 1160 .setkey = n2_des_setkey, 1161 .encrypt = n2_encrypt_chaining, 1162 .decrypt = n2_decrypt_chaining, 1163 }, 1164 }, 1165 { .name = "cfb(des)", 1166 .drv_name = "cfb-des", 1167 .block_size = DES_BLOCK_SIZE, 1168 .enc_type = (ENC_TYPE_ALG_DES | 1169 ENC_TYPE_CHAINING_CFB), 1170 .ablkcipher = { 1171 .min_keysize = DES_KEY_SIZE, 1172 .max_keysize = DES_KEY_SIZE, 1173 .setkey = n2_des_setkey, 1174 .encrypt = n2_encrypt_chaining, 1175 .decrypt = n2_decrypt_chaining, 1176 }, 1177 }, 1178 1179 /* 3DES: ECB CBC and CFB are supported */ 1180 { .name = "ecb(des3_ede)", 1181 .drv_name = "ecb-3des", 1182 .block_size = DES_BLOCK_SIZE, 1183 .enc_type = (ENC_TYPE_ALG_3DES | 1184 ENC_TYPE_CHAINING_ECB), 1185 .ablkcipher = { 1186 .min_keysize = 3 * DES_KEY_SIZE, 1187 .max_keysize = 3 * DES_KEY_SIZE, 1188 .setkey = n2_3des_setkey, 1189 .encrypt = n2_encrypt_ecb, 1190 .decrypt = n2_decrypt_ecb, 1191 }, 1192 }, 1193 { .name = "cbc(des3_ede)", 1194 .drv_name = "cbc-3des", 1195 .block_size = DES_BLOCK_SIZE, 1196 .enc_type = (ENC_TYPE_ALG_3DES | 1197 ENC_TYPE_CHAINING_CBC), 1198 .ablkcipher = { 1199 .ivsize = DES_BLOCK_SIZE, 1200 .min_keysize = 3 * DES_KEY_SIZE, 1201 .max_keysize = 3 * DES_KEY_SIZE, 1202 .setkey = n2_3des_setkey, 1203 .encrypt = n2_encrypt_chaining, 1204 .decrypt = n2_decrypt_chaining, 1205 }, 1206 }, 1207 { .name = "cfb(des3_ede)", 1208 .drv_name = "cfb-3des", 1209 .block_size = DES_BLOCK_SIZE, 1210 .enc_type = (ENC_TYPE_ALG_3DES | 1211 ENC_TYPE_CHAINING_CFB), 1212 .ablkcipher = { 1213 .min_keysize = 3 * DES_KEY_SIZE, 1214 .max_keysize = 3 * DES_KEY_SIZE, 1215 .setkey = n2_3des_setkey, 1216 .encrypt = n2_encrypt_chaining, 1217 .decrypt = n2_decrypt_chaining, 1218 }, 1219 }, 1220 /* AES: ECB CBC and CTR are supported */ 1221 { .name = "ecb(aes)", 1222 .drv_name = "ecb-aes", 1223 .block_size = AES_BLOCK_SIZE, 1224 .enc_type = (ENC_TYPE_ALG_AES128 | 1225 ENC_TYPE_CHAINING_ECB), 1226 .ablkcipher = { 1227 .min_keysize = AES_MIN_KEY_SIZE, 1228 .max_keysize = AES_MAX_KEY_SIZE, 1229 .setkey = n2_aes_setkey, 1230 .encrypt = n2_encrypt_ecb, 1231 .decrypt = n2_decrypt_ecb, 1232 }, 1233 }, 1234 { .name = "cbc(aes)", 1235 .drv_name = "cbc-aes", 1236 .block_size = AES_BLOCK_SIZE, 1237 .enc_type = (ENC_TYPE_ALG_AES128 | 1238 ENC_TYPE_CHAINING_CBC), 1239 .ablkcipher = { 1240 .ivsize = AES_BLOCK_SIZE, 1241 .min_keysize = AES_MIN_KEY_SIZE, 1242 .max_keysize = AES_MAX_KEY_SIZE, 1243 .setkey = n2_aes_setkey, 1244 .encrypt = n2_encrypt_chaining, 1245 .decrypt = n2_decrypt_chaining, 1246 }, 1247 }, 1248 { .name = "ctr(aes)", 1249 .drv_name = "ctr-aes", 1250 .block_size = AES_BLOCK_SIZE, 1251 .enc_type = (ENC_TYPE_ALG_AES128 | 1252 ENC_TYPE_CHAINING_COUNTER), 1253 .ablkcipher = { 1254 .ivsize = AES_BLOCK_SIZE, 1255 .min_keysize = AES_MIN_KEY_SIZE, 1256 .max_keysize = AES_MAX_KEY_SIZE, 1257 .setkey = n2_aes_setkey, 1258 .encrypt = n2_encrypt_chaining, 1259 .decrypt = n2_encrypt_chaining, 1260 }, 1261 }, 1262 1263 }; 1264 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls) 1265 1266 static LIST_HEAD(cipher_algs); 1267 1268 struct n2_hash_tmpl { 1269 const char *name; 1270 const u8 *hash_zero; 1271 const u32 *hash_init; 1272 u8 hw_op_hashsz; 1273 u8 digest_size; 1274 u8 block_size; 1275 u8 auth_type; 1276 u8 hmac_type; 1277 }; 1278 1279 static const u32 md5_init[MD5_HASH_WORDS] = { 1280 cpu_to_le32(MD5_H0), 1281 cpu_to_le32(MD5_H1), 1282 cpu_to_le32(MD5_H2), 1283 cpu_to_le32(MD5_H3), 1284 }; 1285 static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { 1286 SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 1287 }; 1288 static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = { 1289 SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, 1290 SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, 1291 }; 1292 static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = { 1293 SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, 1294 SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, 1295 }; 1296 1297 static const struct n2_hash_tmpl hash_tmpls[] = { 1298 { .name = "md5", 1299 .hash_zero = md5_zero_message_hash, 1300 .hash_init = md5_init, 1301 .auth_type = AUTH_TYPE_MD5, 1302 .hmac_type = AUTH_TYPE_HMAC_MD5, 1303 .hw_op_hashsz = MD5_DIGEST_SIZE, 1304 .digest_size = MD5_DIGEST_SIZE, 1305 .block_size = MD5_HMAC_BLOCK_SIZE }, 1306 { .name = "sha1", 1307 .hash_zero = sha1_zero_message_hash, 1308 .hash_init = sha1_init, 1309 .auth_type = AUTH_TYPE_SHA1, 1310 .hmac_type = AUTH_TYPE_HMAC_SHA1, 1311 .hw_op_hashsz = SHA1_DIGEST_SIZE, 1312 .digest_size = SHA1_DIGEST_SIZE, 1313 .block_size = SHA1_BLOCK_SIZE }, 1314 { .name = "sha256", 1315 .hash_zero = sha256_zero_message_hash, 1316 .hash_init = sha256_init, 1317 .auth_type = AUTH_TYPE_SHA256, 1318 .hmac_type = AUTH_TYPE_HMAC_SHA256, 1319 .hw_op_hashsz = SHA256_DIGEST_SIZE, 1320 .digest_size = SHA256_DIGEST_SIZE, 1321 .block_size = SHA256_BLOCK_SIZE }, 1322 { .name = "sha224", 1323 .hash_zero = sha224_zero_message_hash, 1324 .hash_init = sha224_init, 1325 .auth_type = AUTH_TYPE_SHA256, 1326 .hmac_type = AUTH_TYPE_RESERVED, 1327 .hw_op_hashsz = SHA256_DIGEST_SIZE, 1328 .digest_size = SHA224_DIGEST_SIZE, 1329 .block_size = SHA224_BLOCK_SIZE }, 1330 }; 1331 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) 1332 1333 static LIST_HEAD(ahash_algs); 1334 static LIST_HEAD(hmac_algs); 1335 1336 static int algs_registered; 1337 1338 static void __n2_unregister_algs(void) 1339 { 1340 struct n2_cipher_alg *cipher, *cipher_tmp; 1341 struct n2_ahash_alg *alg, *alg_tmp; 1342 struct n2_hmac_alg *hmac, *hmac_tmp; 1343 1344 list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) { 1345 crypto_unregister_alg(&cipher->alg); 1346 list_del(&cipher->entry); 1347 kfree(cipher); 1348 } 1349 list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { 1350 crypto_unregister_ahash(&hmac->derived.alg); 1351 list_del(&hmac->derived.entry); 1352 kfree(hmac); 1353 } 1354 list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { 1355 crypto_unregister_ahash(&alg->alg); 1356 list_del(&alg->entry); 1357 kfree(alg); 1358 } 1359 } 1360 1361 static int n2_cipher_cra_init(struct crypto_tfm *tfm) 1362 { 1363 tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context); 1364 return 0; 1365 } 1366 1367 static int __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl) 1368 { 1369 struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1370 struct crypto_alg *alg; 1371 int err; 1372 1373 if (!p) 1374 return -ENOMEM; 1375 1376 alg = &p->alg; 1377 1378 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1379 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name); 1380 alg->cra_priority = N2_CRA_PRIORITY; 1381 alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1382 CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; 1383 alg->cra_blocksize = tmpl->block_size; 1384 p->enc_type = tmpl->enc_type; 1385 alg->cra_ctxsize = sizeof(struct n2_cipher_context); 1386 alg->cra_type = &crypto_ablkcipher_type; 1387 alg->cra_u.ablkcipher = tmpl->ablkcipher; 1388 alg->cra_init = n2_cipher_cra_init; 1389 alg->cra_module = THIS_MODULE; 1390 1391 list_add(&p->entry, &cipher_algs); 1392 err = crypto_register_alg(alg); 1393 if (err) { 1394 pr_err("%s alg registration failed\n", alg->cra_name); 1395 list_del(&p->entry); 1396 kfree(p); 1397 } else { 1398 pr_info("%s alg registered\n", alg->cra_name); 1399 } 1400 return err; 1401 } 1402 1403 static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) 1404 { 1405 struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1406 struct ahash_alg *ahash; 1407 struct crypto_alg *base; 1408 int err; 1409 1410 if (!p) 1411 return -ENOMEM; 1412 1413 p->child_alg = n2ahash->alg.halg.base.cra_name; 1414 memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); 1415 INIT_LIST_HEAD(&p->derived.entry); 1416 1417 ahash = &p->derived.alg; 1418 ahash->digest = n2_hmac_async_digest; 1419 ahash->setkey = n2_hmac_async_setkey; 1420 1421 base = &ahash->halg.base; 1422 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg); 1423 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg); 1424 1425 base->cra_ctxsize = sizeof(struct n2_hmac_ctx); 1426 base->cra_init = n2_hmac_cra_init; 1427 base->cra_exit = n2_hmac_cra_exit; 1428 1429 list_add(&p->derived.entry, &hmac_algs); 1430 err = crypto_register_ahash(ahash); 1431 if (err) { 1432 pr_err("%s alg registration failed\n", base->cra_name); 1433 list_del(&p->derived.entry); 1434 kfree(p); 1435 } else { 1436 pr_info("%s alg registered\n", base->cra_name); 1437 } 1438 return err; 1439 } 1440 1441 static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) 1442 { 1443 struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1444 struct hash_alg_common *halg; 1445 struct crypto_alg *base; 1446 struct ahash_alg *ahash; 1447 int err; 1448 1449 if (!p) 1450 return -ENOMEM; 1451 1452 p->hash_zero = tmpl->hash_zero; 1453 p->hash_init = tmpl->hash_init; 1454 p->auth_type = tmpl->auth_type; 1455 p->hmac_type = tmpl->hmac_type; 1456 p->hw_op_hashsz = tmpl->hw_op_hashsz; 1457 p->digest_size = tmpl->digest_size; 1458 1459 ahash = &p->alg; 1460 ahash->init = n2_hash_async_init; 1461 ahash->update = n2_hash_async_update; 1462 ahash->final = n2_hash_async_final; 1463 ahash->finup = n2_hash_async_finup; 1464 ahash->digest = n2_hash_async_digest; 1465 1466 halg = &ahash->halg; 1467 halg->digestsize = tmpl->digest_size; 1468 1469 base = &halg->base; 1470 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1471 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name); 1472 base->cra_priority = N2_CRA_PRIORITY; 1473 base->cra_flags = CRYPTO_ALG_TYPE_AHASH | 1474 CRYPTO_ALG_KERN_DRIVER_ONLY | 1475 CRYPTO_ALG_NEED_FALLBACK; 1476 base->cra_blocksize = tmpl->block_size; 1477 base->cra_ctxsize = sizeof(struct n2_hash_ctx); 1478 base->cra_module = THIS_MODULE; 1479 base->cra_init = n2_hash_cra_init; 1480 base->cra_exit = n2_hash_cra_exit; 1481 1482 list_add(&p->entry, &ahash_algs); 1483 err = crypto_register_ahash(ahash); 1484 if (err) { 1485 pr_err("%s alg registration failed\n", base->cra_name); 1486 list_del(&p->entry); 1487 kfree(p); 1488 } else { 1489 pr_info("%s alg registered\n", base->cra_name); 1490 } 1491 if (!err && p->hmac_type != AUTH_TYPE_RESERVED) 1492 err = __n2_register_one_hmac(p); 1493 return err; 1494 } 1495 1496 static int n2_register_algs(void) 1497 { 1498 int i, err = 0; 1499 1500 mutex_lock(&spu_lock); 1501 if (algs_registered++) 1502 goto out; 1503 1504 for (i = 0; i < NUM_HASH_TMPLS; i++) { 1505 err = __n2_register_one_ahash(&hash_tmpls[i]); 1506 if (err) { 1507 __n2_unregister_algs(); 1508 goto out; 1509 } 1510 } 1511 for (i = 0; i < NUM_CIPHER_TMPLS; i++) { 1512 err = __n2_register_one_cipher(&cipher_tmpls[i]); 1513 if (err) { 1514 __n2_unregister_algs(); 1515 goto out; 1516 } 1517 } 1518 1519 out: 1520 mutex_unlock(&spu_lock); 1521 return err; 1522 } 1523 1524 static void n2_unregister_algs(void) 1525 { 1526 mutex_lock(&spu_lock); 1527 if (!--algs_registered) 1528 __n2_unregister_algs(); 1529 mutex_unlock(&spu_lock); 1530 } 1531 1532 /* To map CWQ queues to interrupt sources, the hypervisor API provides 1533 * a devino. This isn't very useful to us because all of the 1534 * interrupts listed in the device_node have been translated to 1535 * Linux virtual IRQ cookie numbers. 1536 * 1537 * So we have to back-translate, going through the 'intr' and 'ino' 1538 * property tables of the n2cp MDESC node, matching it with the OF 1539 * 'interrupts' property entries, in order to to figure out which 1540 * devino goes to which already-translated IRQ. 1541 */ 1542 static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, 1543 unsigned long dev_ino) 1544 { 1545 const unsigned int *dev_intrs; 1546 unsigned int intr; 1547 int i; 1548 1549 for (i = 0; i < ip->num_intrs; i++) { 1550 if (ip->ino_table[i].ino == dev_ino) 1551 break; 1552 } 1553 if (i == ip->num_intrs) 1554 return -ENODEV; 1555 1556 intr = ip->ino_table[i].intr; 1557 1558 dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL); 1559 if (!dev_intrs) 1560 return -ENODEV; 1561 1562 for (i = 0; i < dev->archdata.num_irqs; i++) { 1563 if (dev_intrs[i] == intr) 1564 return i; 1565 } 1566 1567 return -ENODEV; 1568 } 1569 1570 static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip, 1571 const char *irq_name, struct spu_queue *p, 1572 irq_handler_t handler) 1573 { 1574 unsigned long herr; 1575 int index; 1576 1577 herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino); 1578 if (herr) 1579 return -EINVAL; 1580 1581 index = find_devino_index(dev, ip, p->devino); 1582 if (index < 0) 1583 return index; 1584 1585 p->irq = dev->archdata.irqs[index]; 1586 1587 sprintf(p->irq_name, "%s-%d", irq_name, index); 1588 1589 return request_irq(p->irq, handler, 0, p->irq_name, p); 1590 } 1591 1592 static struct kmem_cache *queue_cache[2]; 1593 1594 static void *new_queue(unsigned long q_type) 1595 { 1596 return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL); 1597 } 1598 1599 static void free_queue(void *p, unsigned long q_type) 1600 { 1601 kmem_cache_free(queue_cache[q_type - 1], p); 1602 } 1603 1604 static int queue_cache_init(void) 1605 { 1606 if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 1607 queue_cache[HV_NCS_QTYPE_MAU - 1] = 1608 kmem_cache_create("mau_queue", 1609 (MAU_NUM_ENTRIES * 1610 MAU_ENTRY_SIZE), 1611 MAU_ENTRY_SIZE, 0, NULL); 1612 if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 1613 return -ENOMEM; 1614 1615 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) 1616 queue_cache[HV_NCS_QTYPE_CWQ - 1] = 1617 kmem_cache_create("cwq_queue", 1618 (CWQ_NUM_ENTRIES * 1619 CWQ_ENTRY_SIZE), 1620 CWQ_ENTRY_SIZE, 0, NULL); 1621 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { 1622 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 1623 return -ENOMEM; 1624 } 1625 return 0; 1626 } 1627 1628 static void queue_cache_destroy(void) 1629 { 1630 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 1631 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]); 1632 } 1633 1634 static int spu_queue_register(struct spu_queue *p, unsigned long q_type) 1635 { 1636 cpumask_var_t old_allowed; 1637 unsigned long hv_ret; 1638 1639 if (cpumask_empty(&p->sharing)) 1640 return -EINVAL; 1641 1642 if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL)) 1643 return -ENOMEM; 1644 1645 cpumask_copy(old_allowed, ¤t->cpus_allowed); 1646 1647 set_cpus_allowed_ptr(current, &p->sharing); 1648 1649 hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q), 1650 CWQ_NUM_ENTRIES, &p->qhandle); 1651 if (!hv_ret) 1652 sun4v_ncs_sethead_marker(p->qhandle, 0); 1653 1654 set_cpus_allowed_ptr(current, old_allowed); 1655 1656 free_cpumask_var(old_allowed); 1657 1658 return (hv_ret ? -EINVAL : 0); 1659 } 1660 1661 static int spu_queue_setup(struct spu_queue *p) 1662 { 1663 int err; 1664 1665 p->q = new_queue(p->q_type); 1666 if (!p->q) 1667 return -ENOMEM; 1668 1669 err = spu_queue_register(p, p->q_type); 1670 if (err) { 1671 free_queue(p->q, p->q_type); 1672 p->q = NULL; 1673 } 1674 1675 return err; 1676 } 1677 1678 static void spu_queue_destroy(struct spu_queue *p) 1679 { 1680 unsigned long hv_ret; 1681 1682 if (!p->q) 1683 return; 1684 1685 hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle); 1686 1687 if (!hv_ret) 1688 free_queue(p->q, p->q_type); 1689 } 1690 1691 static void spu_list_destroy(struct list_head *list) 1692 { 1693 struct spu_queue *p, *n; 1694 1695 list_for_each_entry_safe(p, n, list, list) { 1696 int i; 1697 1698 for (i = 0; i < NR_CPUS; i++) { 1699 if (cpu_to_cwq[i] == p) 1700 cpu_to_cwq[i] = NULL; 1701 } 1702 1703 if (p->irq) { 1704 free_irq(p->irq, p); 1705 p->irq = 0; 1706 } 1707 spu_queue_destroy(p); 1708 list_del(&p->list); 1709 kfree(p); 1710 } 1711 } 1712 1713 /* Walk the backward arcs of a CWQ 'exec-unit' node, 1714 * gathering cpu membership information. 1715 */ 1716 static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, 1717 struct platform_device *dev, 1718 u64 node, struct spu_queue *p, 1719 struct spu_queue **table) 1720 { 1721 u64 arc; 1722 1723 mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { 1724 u64 tgt = mdesc_arc_target(mdesc, arc); 1725 const char *name = mdesc_node_name(mdesc, tgt); 1726 const u64 *id; 1727 1728 if (strcmp(name, "cpu")) 1729 continue; 1730 id = mdesc_get_property(mdesc, tgt, "id", NULL); 1731 if (table[*id] != NULL) { 1732 dev_err(&dev->dev, "%s: SPU cpu slot already set.\n", 1733 dev->dev.of_node->full_name); 1734 return -EINVAL; 1735 } 1736 cpumask_set_cpu(*id, &p->sharing); 1737 table[*id] = p; 1738 } 1739 return 0; 1740 } 1741 1742 /* Process an 'exec-unit' MDESC node of type 'cwq'. */ 1743 static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, 1744 struct platform_device *dev, struct mdesc_handle *mdesc, 1745 u64 node, const char *iname, unsigned long q_type, 1746 irq_handler_t handler, struct spu_queue **table) 1747 { 1748 struct spu_queue *p; 1749 int err; 1750 1751 p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); 1752 if (!p) { 1753 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n", 1754 dev->dev.of_node->full_name); 1755 return -ENOMEM; 1756 } 1757 1758 cpumask_clear(&p->sharing); 1759 spin_lock_init(&p->lock); 1760 p->q_type = q_type; 1761 INIT_LIST_HEAD(&p->jobs); 1762 list_add(&p->list, list); 1763 1764 err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); 1765 if (err) 1766 return err; 1767 1768 err = spu_queue_setup(p); 1769 if (err) 1770 return err; 1771 1772 return spu_map_ino(dev, ip, iname, p, handler); 1773 } 1774 1775 static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev, 1776 struct spu_mdesc_info *ip, struct list_head *list, 1777 const char *exec_name, unsigned long q_type, 1778 irq_handler_t handler, struct spu_queue **table) 1779 { 1780 int err = 0; 1781 u64 node; 1782 1783 mdesc_for_each_node_by_name(mdesc, node, "exec-unit") { 1784 const char *type; 1785 1786 type = mdesc_get_property(mdesc, node, "type", NULL); 1787 if (!type || strcmp(type, exec_name)) 1788 continue; 1789 1790 err = handle_exec_unit(ip, list, dev, mdesc, node, 1791 exec_name, q_type, handler, table); 1792 if (err) { 1793 spu_list_destroy(list); 1794 break; 1795 } 1796 } 1797 1798 return err; 1799 } 1800 1801 static int get_irq_props(struct mdesc_handle *mdesc, u64 node, 1802 struct spu_mdesc_info *ip) 1803 { 1804 const u64 *ino; 1805 int ino_len; 1806 int i; 1807 1808 ino = mdesc_get_property(mdesc, node, "ino", &ino_len); 1809 if (!ino) { 1810 printk("NO 'ino'\n"); 1811 return -ENODEV; 1812 } 1813 1814 ip->num_intrs = ino_len / sizeof(u64); 1815 ip->ino_table = kzalloc((sizeof(struct ino_blob) * 1816 ip->num_intrs), 1817 GFP_KERNEL); 1818 if (!ip->ino_table) 1819 return -ENOMEM; 1820 1821 for (i = 0; i < ip->num_intrs; i++) { 1822 struct ino_blob *b = &ip->ino_table[i]; 1823 b->intr = i + 1; 1824 b->ino = ino[i]; 1825 } 1826 1827 return 0; 1828 } 1829 1830 static int grab_mdesc_irq_props(struct mdesc_handle *mdesc, 1831 struct platform_device *dev, 1832 struct spu_mdesc_info *ip, 1833 const char *node_name) 1834 { 1835 const unsigned int *reg; 1836 u64 node; 1837 1838 reg = of_get_property(dev->dev.of_node, "reg", NULL); 1839 if (!reg) 1840 return -ENODEV; 1841 1842 mdesc_for_each_node_by_name(mdesc, node, "virtual-device") { 1843 const char *name; 1844 const u64 *chdl; 1845 1846 name = mdesc_get_property(mdesc, node, "name", NULL); 1847 if (!name || strcmp(name, node_name)) 1848 continue; 1849 chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL); 1850 if (!chdl || (*chdl != *reg)) 1851 continue; 1852 ip->cfg_handle = *chdl; 1853 return get_irq_props(mdesc, node, ip); 1854 } 1855 1856 return -ENODEV; 1857 } 1858 1859 static unsigned long n2_spu_hvapi_major; 1860 static unsigned long n2_spu_hvapi_minor; 1861 1862 static int n2_spu_hvapi_register(void) 1863 { 1864 int err; 1865 1866 n2_spu_hvapi_major = 2; 1867 n2_spu_hvapi_minor = 0; 1868 1869 err = sun4v_hvapi_register(HV_GRP_NCS, 1870 n2_spu_hvapi_major, 1871 &n2_spu_hvapi_minor); 1872 1873 if (!err) 1874 pr_info("Registered NCS HVAPI version %lu.%lu\n", 1875 n2_spu_hvapi_major, 1876 n2_spu_hvapi_minor); 1877 1878 return err; 1879 } 1880 1881 static void n2_spu_hvapi_unregister(void) 1882 { 1883 sun4v_hvapi_unregister(HV_GRP_NCS); 1884 } 1885 1886 static int global_ref; 1887 1888 static int grab_global_resources(void) 1889 { 1890 int err = 0; 1891 1892 mutex_lock(&spu_lock); 1893 1894 if (global_ref++) 1895 goto out; 1896 1897 err = n2_spu_hvapi_register(); 1898 if (err) 1899 goto out; 1900 1901 err = queue_cache_init(); 1902 if (err) 1903 goto out_hvapi_release; 1904 1905 err = -ENOMEM; 1906 cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS, 1907 GFP_KERNEL); 1908 if (!cpu_to_cwq) 1909 goto out_queue_cache_destroy; 1910 1911 cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS, 1912 GFP_KERNEL); 1913 if (!cpu_to_mau) 1914 goto out_free_cwq_table; 1915 1916 err = 0; 1917 1918 out: 1919 if (err) 1920 global_ref--; 1921 mutex_unlock(&spu_lock); 1922 return err; 1923 1924 out_free_cwq_table: 1925 kfree(cpu_to_cwq); 1926 cpu_to_cwq = NULL; 1927 1928 out_queue_cache_destroy: 1929 queue_cache_destroy(); 1930 1931 out_hvapi_release: 1932 n2_spu_hvapi_unregister(); 1933 goto out; 1934 } 1935 1936 static void release_global_resources(void) 1937 { 1938 mutex_lock(&spu_lock); 1939 if (!--global_ref) { 1940 kfree(cpu_to_cwq); 1941 cpu_to_cwq = NULL; 1942 1943 kfree(cpu_to_mau); 1944 cpu_to_mau = NULL; 1945 1946 queue_cache_destroy(); 1947 n2_spu_hvapi_unregister(); 1948 } 1949 mutex_unlock(&spu_lock); 1950 } 1951 1952 static struct n2_crypto *alloc_n2cp(void) 1953 { 1954 struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL); 1955 1956 if (np) 1957 INIT_LIST_HEAD(&np->cwq_list); 1958 1959 return np; 1960 } 1961 1962 static void free_n2cp(struct n2_crypto *np) 1963 { 1964 if (np->cwq_info.ino_table) { 1965 kfree(np->cwq_info.ino_table); 1966 np->cwq_info.ino_table = NULL; 1967 } 1968 1969 kfree(np); 1970 } 1971 1972 static void n2_spu_driver_version(void) 1973 { 1974 static int n2_spu_version_printed; 1975 1976 if (n2_spu_version_printed++ == 0) 1977 pr_info("%s", version); 1978 } 1979 1980 static int n2_crypto_probe(struct platform_device *dev) 1981 { 1982 struct mdesc_handle *mdesc; 1983 const char *full_name; 1984 struct n2_crypto *np; 1985 int err; 1986 1987 n2_spu_driver_version(); 1988 1989 full_name = dev->dev.of_node->full_name; 1990 pr_info("Found N2CP at %s\n", full_name); 1991 1992 np = alloc_n2cp(); 1993 if (!np) { 1994 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n", 1995 full_name); 1996 return -ENOMEM; 1997 } 1998 1999 err = grab_global_resources(); 2000 if (err) { 2001 dev_err(&dev->dev, "%s: Unable to grab " 2002 "global resources.\n", full_name); 2003 goto out_free_n2cp; 2004 } 2005 2006 mdesc = mdesc_grab(); 2007 2008 if (!mdesc) { 2009 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 2010 full_name); 2011 err = -ENODEV; 2012 goto out_free_global; 2013 } 2014 err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); 2015 if (err) { 2016 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 2017 full_name); 2018 mdesc_release(mdesc); 2019 goto out_free_global; 2020 } 2021 2022 err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list, 2023 "cwq", HV_NCS_QTYPE_CWQ, cwq_intr, 2024 cpu_to_cwq); 2025 mdesc_release(mdesc); 2026 2027 if (err) { 2028 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n", 2029 full_name); 2030 goto out_free_global; 2031 } 2032 2033 err = n2_register_algs(); 2034 if (err) { 2035 dev_err(&dev->dev, "%s: Unable to register algorithms.\n", 2036 full_name); 2037 goto out_free_spu_list; 2038 } 2039 2040 dev_set_drvdata(&dev->dev, np); 2041 2042 return 0; 2043 2044 out_free_spu_list: 2045 spu_list_destroy(&np->cwq_list); 2046 2047 out_free_global: 2048 release_global_resources(); 2049 2050 out_free_n2cp: 2051 free_n2cp(np); 2052 2053 return err; 2054 } 2055 2056 static int n2_crypto_remove(struct platform_device *dev) 2057 { 2058 struct n2_crypto *np = dev_get_drvdata(&dev->dev); 2059 2060 n2_unregister_algs(); 2061 2062 spu_list_destroy(&np->cwq_list); 2063 2064 release_global_resources(); 2065 2066 free_n2cp(np); 2067 2068 return 0; 2069 } 2070 2071 static struct n2_mau *alloc_ncp(void) 2072 { 2073 struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL); 2074 2075 if (mp) 2076 INIT_LIST_HEAD(&mp->mau_list); 2077 2078 return mp; 2079 } 2080 2081 static void free_ncp(struct n2_mau *mp) 2082 { 2083 if (mp->mau_info.ino_table) { 2084 kfree(mp->mau_info.ino_table); 2085 mp->mau_info.ino_table = NULL; 2086 } 2087 2088 kfree(mp); 2089 } 2090 2091 static int n2_mau_probe(struct platform_device *dev) 2092 { 2093 struct mdesc_handle *mdesc; 2094 const char *full_name; 2095 struct n2_mau *mp; 2096 int err; 2097 2098 n2_spu_driver_version(); 2099 2100 full_name = dev->dev.of_node->full_name; 2101 pr_info("Found NCP at %s\n", full_name); 2102 2103 mp = alloc_ncp(); 2104 if (!mp) { 2105 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n", 2106 full_name); 2107 return -ENOMEM; 2108 } 2109 2110 err = grab_global_resources(); 2111 if (err) { 2112 dev_err(&dev->dev, "%s: Unable to grab " 2113 "global resources.\n", full_name); 2114 goto out_free_ncp; 2115 } 2116 2117 mdesc = mdesc_grab(); 2118 2119 if (!mdesc) { 2120 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 2121 full_name); 2122 err = -ENODEV; 2123 goto out_free_global; 2124 } 2125 2126 err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); 2127 if (err) { 2128 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 2129 full_name); 2130 mdesc_release(mdesc); 2131 goto out_free_global; 2132 } 2133 2134 err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list, 2135 "mau", HV_NCS_QTYPE_MAU, mau_intr, 2136 cpu_to_mau); 2137 mdesc_release(mdesc); 2138 2139 if (err) { 2140 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n", 2141 full_name); 2142 goto out_free_global; 2143 } 2144 2145 dev_set_drvdata(&dev->dev, mp); 2146 2147 return 0; 2148 2149 out_free_global: 2150 release_global_resources(); 2151 2152 out_free_ncp: 2153 free_ncp(mp); 2154 2155 return err; 2156 } 2157 2158 static int n2_mau_remove(struct platform_device *dev) 2159 { 2160 struct n2_mau *mp = dev_get_drvdata(&dev->dev); 2161 2162 spu_list_destroy(&mp->mau_list); 2163 2164 release_global_resources(); 2165 2166 free_ncp(mp); 2167 2168 return 0; 2169 } 2170 2171 static struct of_device_id n2_crypto_match[] = { 2172 { 2173 .name = "n2cp", 2174 .compatible = "SUNW,n2-cwq", 2175 }, 2176 { 2177 .name = "n2cp", 2178 .compatible = "SUNW,vf-cwq", 2179 }, 2180 { 2181 .name = "n2cp", 2182 .compatible = "SUNW,kt-cwq", 2183 }, 2184 {}, 2185 }; 2186 2187 MODULE_DEVICE_TABLE(of, n2_crypto_match); 2188 2189 static struct platform_driver n2_crypto_driver = { 2190 .driver = { 2191 .name = "n2cp", 2192 .of_match_table = n2_crypto_match, 2193 }, 2194 .probe = n2_crypto_probe, 2195 .remove = n2_crypto_remove, 2196 }; 2197 2198 static struct of_device_id n2_mau_match[] = { 2199 { 2200 .name = "ncp", 2201 .compatible = "SUNW,n2-mau", 2202 }, 2203 { 2204 .name = "ncp", 2205 .compatible = "SUNW,vf-mau", 2206 }, 2207 { 2208 .name = "ncp", 2209 .compatible = "SUNW,kt-mau", 2210 }, 2211 {}, 2212 }; 2213 2214 MODULE_DEVICE_TABLE(of, n2_mau_match); 2215 2216 static struct platform_driver n2_mau_driver = { 2217 .driver = { 2218 .name = "ncp", 2219 .of_match_table = n2_mau_match, 2220 }, 2221 .probe = n2_mau_probe, 2222 .remove = n2_mau_remove, 2223 }; 2224 2225 static struct platform_driver * const drivers[] = { 2226 &n2_crypto_driver, 2227 &n2_mau_driver, 2228 }; 2229 2230 static int __init n2_init(void) 2231 { 2232 return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); 2233 } 2234 2235 static void __exit n2_exit(void) 2236 { 2237 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); 2238 } 2239 2240 module_init(n2_init); 2241 module_exit(n2_exit); 2242