xref: /openbmc/linux/drivers/crypto/n2_core.c (revision 4800cd83)
1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
2  *
3  * Copyright (C) 2010 David S. Miller <davem@davemloft.net>
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/of.h>
11 #include <linux/of_device.h>
12 #include <linux/cpumask.h>
13 #include <linux/slab.h>
14 #include <linux/interrupt.h>
15 #include <linux/crypto.h>
16 #include <crypto/md5.h>
17 #include <crypto/sha.h>
18 #include <crypto/aes.h>
19 #include <crypto/des.h>
20 #include <linux/mutex.h>
21 #include <linux/delay.h>
22 #include <linux/sched.h>
23 
24 #include <crypto/internal/hash.h>
25 #include <crypto/scatterwalk.h>
26 #include <crypto/algapi.h>
27 
28 #include <asm/hypervisor.h>
29 #include <asm/mdesc.h>
30 
31 #include "n2_core.h"
32 
33 #define DRV_MODULE_NAME		"n2_crypto"
34 #define DRV_MODULE_VERSION	"0.1"
35 #define DRV_MODULE_RELDATE	"April 29, 2010"
36 
37 static char version[] __devinitdata =
38 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
39 
40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
41 MODULE_DESCRIPTION("Niagara2 Crypto driver");
42 MODULE_LICENSE("GPL");
43 MODULE_VERSION(DRV_MODULE_VERSION);
44 
45 #define N2_CRA_PRIORITY		300
46 
47 static DEFINE_MUTEX(spu_lock);
48 
49 struct spu_queue {
50 	cpumask_t		sharing;
51 	unsigned long		qhandle;
52 
53 	spinlock_t		lock;
54 	u8			q_type;
55 	void			*q;
56 	unsigned long		head;
57 	unsigned long		tail;
58 	struct list_head	jobs;
59 
60 	unsigned long		devino;
61 
62 	char			irq_name[32];
63 	unsigned int		irq;
64 
65 	struct list_head	list;
66 };
67 
68 static struct spu_queue **cpu_to_cwq;
69 static struct spu_queue **cpu_to_mau;
70 
71 static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
72 {
73 	if (q->q_type == HV_NCS_QTYPE_MAU) {
74 		off += MAU_ENTRY_SIZE;
75 		if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
76 			off = 0;
77 	} else {
78 		off += CWQ_ENTRY_SIZE;
79 		if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
80 			off = 0;
81 	}
82 	return off;
83 }
84 
85 struct n2_request_common {
86 	struct list_head	entry;
87 	unsigned int		offset;
88 };
89 #define OFFSET_NOT_RUNNING	(~(unsigned int)0)
90 
91 /* An async job request records the final tail value it used in
92  * n2_request_common->offset, test to see if that offset is in
93  * the range old_head, new_head, inclusive.
94  */
95 static inline bool job_finished(struct spu_queue *q, unsigned int offset,
96 				unsigned long old_head, unsigned long new_head)
97 {
98 	if (old_head <= new_head) {
99 		if (offset > old_head && offset <= new_head)
100 			return true;
101 	} else {
102 		if (offset > old_head || offset <= new_head)
103 			return true;
104 	}
105 	return false;
106 }
107 
108 /* When the HEAD marker is unequal to the actual HEAD, we get
109  * a virtual device INO interrupt.  We should process the
110  * completed CWQ entries and adjust the HEAD marker to clear
111  * the IRQ.
112  */
113 static irqreturn_t cwq_intr(int irq, void *dev_id)
114 {
115 	unsigned long off, new_head, hv_ret;
116 	struct spu_queue *q = dev_id;
117 
118 	pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
119 	       smp_processor_id(), q->qhandle);
120 
121 	spin_lock(&q->lock);
122 
123 	hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
124 
125 	pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
126 	       smp_processor_id(), new_head, hv_ret);
127 
128 	for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
129 		/* XXX ... XXX */
130 	}
131 
132 	hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
133 	if (hv_ret == HV_EOK)
134 		q->head = new_head;
135 
136 	spin_unlock(&q->lock);
137 
138 	return IRQ_HANDLED;
139 }
140 
141 static irqreturn_t mau_intr(int irq, void *dev_id)
142 {
143 	struct spu_queue *q = dev_id;
144 	unsigned long head, hv_ret;
145 
146 	spin_lock(&q->lock);
147 
148 	pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
149 	       smp_processor_id(), q->qhandle);
150 
151 	hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
152 
153 	pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
154 	       smp_processor_id(), head, hv_ret);
155 
156 	sun4v_ncs_sethead_marker(q->qhandle, head);
157 
158 	spin_unlock(&q->lock);
159 
160 	return IRQ_HANDLED;
161 }
162 
163 static void *spu_queue_next(struct spu_queue *q, void *cur)
164 {
165 	return q->q + spu_next_offset(q, cur - q->q);
166 }
167 
168 static int spu_queue_num_free(struct spu_queue *q)
169 {
170 	unsigned long head = q->head;
171 	unsigned long tail = q->tail;
172 	unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
173 	unsigned long diff;
174 
175 	if (head > tail)
176 		diff = head - tail;
177 	else
178 		diff = (end - tail) + head;
179 
180 	return (diff / CWQ_ENTRY_SIZE) - 1;
181 }
182 
183 static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
184 {
185 	int avail = spu_queue_num_free(q);
186 
187 	if (avail >= num_entries)
188 		return q->q + q->tail;
189 
190 	return NULL;
191 }
192 
193 static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
194 {
195 	unsigned long hv_ret, new_tail;
196 
197 	new_tail = spu_next_offset(q, last - q->q);
198 
199 	hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
200 	if (hv_ret == HV_EOK)
201 		q->tail = new_tail;
202 	return hv_ret;
203 }
204 
205 static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
206 			     int enc_type, int auth_type,
207 			     unsigned int hash_len,
208 			     bool sfas, bool sob, bool eob, bool encrypt,
209 			     int opcode)
210 {
211 	u64 word = (len - 1) & CONTROL_LEN;
212 
213 	word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
214 	word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
215 	word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
216 	if (sfas)
217 		word |= CONTROL_STORE_FINAL_AUTH_STATE;
218 	if (sob)
219 		word |= CONTROL_START_OF_BLOCK;
220 	if (eob)
221 		word |= CONTROL_END_OF_BLOCK;
222 	if (encrypt)
223 		word |= CONTROL_ENCRYPT;
224 	if (hmac_key_len)
225 		word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
226 	if (hash_len)
227 		word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
228 
229 	return word;
230 }
231 
232 #if 0
233 static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
234 {
235 	if (this_len >= 64 ||
236 	    qp->head != qp->tail)
237 		return true;
238 	return false;
239 }
240 #endif
241 
242 struct n2_ahash_alg {
243 	struct list_head	entry;
244 	const char		*hash_zero;
245 	const u32		*hash_init;
246 	u8			hw_op_hashsz;
247 	u8			digest_size;
248 	u8			auth_type;
249 	u8			hmac_type;
250 	struct ahash_alg	alg;
251 };
252 
253 static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
254 {
255 	struct crypto_alg *alg = tfm->__crt_alg;
256 	struct ahash_alg *ahash_alg;
257 
258 	ahash_alg = container_of(alg, struct ahash_alg, halg.base);
259 
260 	return container_of(ahash_alg, struct n2_ahash_alg, alg);
261 }
262 
263 struct n2_hmac_alg {
264 	const char		*child_alg;
265 	struct n2_ahash_alg	derived;
266 };
267 
268 static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
269 {
270 	struct crypto_alg *alg = tfm->__crt_alg;
271 	struct ahash_alg *ahash_alg;
272 
273 	ahash_alg = container_of(alg, struct ahash_alg, halg.base);
274 
275 	return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
276 }
277 
278 struct n2_hash_ctx {
279 	struct crypto_ahash		*fallback_tfm;
280 };
281 
282 #define N2_HASH_KEY_MAX			32 /* HW limit for all HMAC requests */
283 
284 struct n2_hmac_ctx {
285 	struct n2_hash_ctx		base;
286 
287 	struct crypto_shash		*child_shash;
288 
289 	int				hash_key_len;
290 	unsigned char			hash_key[N2_HASH_KEY_MAX];
291 };
292 
293 struct n2_hash_req_ctx {
294 	union {
295 		struct md5_state	md5;
296 		struct sha1_state	sha1;
297 		struct sha256_state	sha256;
298 	} u;
299 
300 	struct ahash_request		fallback_req;
301 };
302 
303 static int n2_hash_async_init(struct ahash_request *req)
304 {
305 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
306 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
307 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
308 
309 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
310 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
311 
312 	return crypto_ahash_init(&rctx->fallback_req);
313 }
314 
315 static int n2_hash_async_update(struct ahash_request *req)
316 {
317 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
318 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
319 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
320 
321 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
322 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
323 	rctx->fallback_req.nbytes = req->nbytes;
324 	rctx->fallback_req.src = req->src;
325 
326 	return crypto_ahash_update(&rctx->fallback_req);
327 }
328 
329 static int n2_hash_async_final(struct ahash_request *req)
330 {
331 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
332 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
333 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
334 
335 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
336 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
337 	rctx->fallback_req.result = req->result;
338 
339 	return crypto_ahash_final(&rctx->fallback_req);
340 }
341 
342 static int n2_hash_async_finup(struct ahash_request *req)
343 {
344 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
345 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
346 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
347 
348 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
349 	rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
350 	rctx->fallback_req.nbytes = req->nbytes;
351 	rctx->fallback_req.src = req->src;
352 	rctx->fallback_req.result = req->result;
353 
354 	return crypto_ahash_finup(&rctx->fallback_req);
355 }
356 
357 static int n2_hash_cra_init(struct crypto_tfm *tfm)
358 {
359 	const char *fallback_driver_name = tfm->__crt_alg->cra_name;
360 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
361 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
362 	struct crypto_ahash *fallback_tfm;
363 	int err;
364 
365 	fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
366 					  CRYPTO_ALG_NEED_FALLBACK);
367 	if (IS_ERR(fallback_tfm)) {
368 		pr_warning("Fallback driver '%s' could not be loaded!\n",
369 			   fallback_driver_name);
370 		err = PTR_ERR(fallback_tfm);
371 		goto out;
372 	}
373 
374 	crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
375 					 crypto_ahash_reqsize(fallback_tfm)));
376 
377 	ctx->fallback_tfm = fallback_tfm;
378 	return 0;
379 
380 out:
381 	return err;
382 }
383 
384 static void n2_hash_cra_exit(struct crypto_tfm *tfm)
385 {
386 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
387 	struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
388 
389 	crypto_free_ahash(ctx->fallback_tfm);
390 }
391 
392 static int n2_hmac_cra_init(struct crypto_tfm *tfm)
393 {
394 	const char *fallback_driver_name = tfm->__crt_alg->cra_name;
395 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
396 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
397 	struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
398 	struct crypto_ahash *fallback_tfm;
399 	struct crypto_shash *child_shash;
400 	int err;
401 
402 	fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
403 					  CRYPTO_ALG_NEED_FALLBACK);
404 	if (IS_ERR(fallback_tfm)) {
405 		pr_warning("Fallback driver '%s' could not be loaded!\n",
406 			   fallback_driver_name);
407 		err = PTR_ERR(fallback_tfm);
408 		goto out;
409 	}
410 
411 	child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
412 	if (IS_ERR(child_shash)) {
413 		pr_warning("Child shash '%s' could not be loaded!\n",
414 			   n2alg->child_alg);
415 		err = PTR_ERR(child_shash);
416 		goto out_free_fallback;
417 	}
418 
419 	crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
420 					 crypto_ahash_reqsize(fallback_tfm)));
421 
422 	ctx->child_shash = child_shash;
423 	ctx->base.fallback_tfm = fallback_tfm;
424 	return 0;
425 
426 out_free_fallback:
427 	crypto_free_ahash(fallback_tfm);
428 
429 out:
430 	return err;
431 }
432 
433 static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
434 {
435 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
436 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
437 
438 	crypto_free_ahash(ctx->base.fallback_tfm);
439 	crypto_free_shash(ctx->child_shash);
440 }
441 
442 static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
443 				unsigned int keylen)
444 {
445 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
446 	struct crypto_shash *child_shash = ctx->child_shash;
447 	struct crypto_ahash *fallback_tfm;
448 	struct {
449 		struct shash_desc shash;
450 		char ctx[crypto_shash_descsize(child_shash)];
451 	} desc;
452 	int err, bs, ds;
453 
454 	fallback_tfm = ctx->base.fallback_tfm;
455 	err = crypto_ahash_setkey(fallback_tfm, key, keylen);
456 	if (err)
457 		return err;
458 
459 	desc.shash.tfm = child_shash;
460 	desc.shash.flags = crypto_ahash_get_flags(tfm) &
461 		CRYPTO_TFM_REQ_MAY_SLEEP;
462 
463 	bs = crypto_shash_blocksize(child_shash);
464 	ds = crypto_shash_digestsize(child_shash);
465 	BUG_ON(ds > N2_HASH_KEY_MAX);
466 	if (keylen > bs) {
467 		err = crypto_shash_digest(&desc.shash, key, keylen,
468 					  ctx->hash_key);
469 		if (err)
470 			return err;
471 		keylen = ds;
472 	} else if (keylen <= N2_HASH_KEY_MAX)
473 		memcpy(ctx->hash_key, key, keylen);
474 
475 	ctx->hash_key_len = keylen;
476 
477 	return err;
478 }
479 
480 static unsigned long wait_for_tail(struct spu_queue *qp)
481 {
482 	unsigned long head, hv_ret;
483 
484 	do {
485 		hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
486 		if (hv_ret != HV_EOK) {
487 			pr_err("Hypervisor error on gethead\n");
488 			break;
489 		}
490 		if (head == qp->tail) {
491 			qp->head = head;
492 			break;
493 		}
494 	} while (1);
495 	return hv_ret;
496 }
497 
498 static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
499 					      struct cwq_initial_entry *ent)
500 {
501 	unsigned long hv_ret = spu_queue_submit(qp, ent);
502 
503 	if (hv_ret == HV_EOK)
504 		hv_ret = wait_for_tail(qp);
505 
506 	return hv_ret;
507 }
508 
509 static int n2_do_async_digest(struct ahash_request *req,
510 			      unsigned int auth_type, unsigned int digest_size,
511 			      unsigned int result_size, void *hash_loc,
512 			      unsigned long auth_key, unsigned int auth_key_len)
513 {
514 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
515 	struct cwq_initial_entry *ent;
516 	struct crypto_hash_walk walk;
517 	struct spu_queue *qp;
518 	unsigned long flags;
519 	int err = -ENODEV;
520 	int nbytes, cpu;
521 
522 	/* The total effective length of the operation may not
523 	 * exceed 2^16.
524 	 */
525 	if (unlikely(req->nbytes > (1 << 16))) {
526 		struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
527 		struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
528 
529 		ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
530 		rctx->fallback_req.base.flags =
531 			req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
532 		rctx->fallback_req.nbytes = req->nbytes;
533 		rctx->fallback_req.src = req->src;
534 		rctx->fallback_req.result = req->result;
535 
536 		return crypto_ahash_digest(&rctx->fallback_req);
537 	}
538 
539 	nbytes = crypto_hash_walk_first(req, &walk);
540 
541 	cpu = get_cpu();
542 	qp = cpu_to_cwq[cpu];
543 	if (!qp)
544 		goto out;
545 
546 	spin_lock_irqsave(&qp->lock, flags);
547 
548 	/* XXX can do better, improve this later by doing a by-hand scatterlist
549 	 * XXX walk, etc.
550 	 */
551 	ent = qp->q + qp->tail;
552 
553 	ent->control = control_word_base(nbytes, auth_key_len, 0,
554 					 auth_type, digest_size,
555 					 false, true, false, false,
556 					 OPCODE_INPLACE_BIT |
557 					 OPCODE_AUTH_MAC);
558 	ent->src_addr = __pa(walk.data);
559 	ent->auth_key_addr = auth_key;
560 	ent->auth_iv_addr = __pa(hash_loc);
561 	ent->final_auth_state_addr = 0UL;
562 	ent->enc_key_addr = 0UL;
563 	ent->enc_iv_addr = 0UL;
564 	ent->dest_addr = __pa(hash_loc);
565 
566 	nbytes = crypto_hash_walk_done(&walk, 0);
567 	while (nbytes > 0) {
568 		ent = spu_queue_next(qp, ent);
569 
570 		ent->control = (nbytes - 1);
571 		ent->src_addr = __pa(walk.data);
572 		ent->auth_key_addr = 0UL;
573 		ent->auth_iv_addr = 0UL;
574 		ent->final_auth_state_addr = 0UL;
575 		ent->enc_key_addr = 0UL;
576 		ent->enc_iv_addr = 0UL;
577 		ent->dest_addr = 0UL;
578 
579 		nbytes = crypto_hash_walk_done(&walk, 0);
580 	}
581 	ent->control |= CONTROL_END_OF_BLOCK;
582 
583 	if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
584 		err = -EINVAL;
585 	else
586 		err = 0;
587 
588 	spin_unlock_irqrestore(&qp->lock, flags);
589 
590 	if (!err)
591 		memcpy(req->result, hash_loc, result_size);
592 out:
593 	put_cpu();
594 
595 	return err;
596 }
597 
598 static int n2_hash_async_digest(struct ahash_request *req)
599 {
600 	struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
601 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
602 	int ds;
603 
604 	ds = n2alg->digest_size;
605 	if (unlikely(req->nbytes == 0)) {
606 		memcpy(req->result, n2alg->hash_zero, ds);
607 		return 0;
608 	}
609 	memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
610 
611 	return n2_do_async_digest(req, n2alg->auth_type,
612 				  n2alg->hw_op_hashsz, ds,
613 				  &rctx->u, 0UL, 0);
614 }
615 
616 static int n2_hmac_async_digest(struct ahash_request *req)
617 {
618 	struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
619 	struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
620 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
621 	struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
622 	int ds;
623 
624 	ds = n2alg->derived.digest_size;
625 	if (unlikely(req->nbytes == 0) ||
626 	    unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
627 		struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
628 		struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
629 
630 		ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
631 		rctx->fallback_req.base.flags =
632 			req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
633 		rctx->fallback_req.nbytes = req->nbytes;
634 		rctx->fallback_req.src = req->src;
635 		rctx->fallback_req.result = req->result;
636 
637 		return crypto_ahash_digest(&rctx->fallback_req);
638 	}
639 	memcpy(&rctx->u, n2alg->derived.hash_init,
640 	       n2alg->derived.hw_op_hashsz);
641 
642 	return n2_do_async_digest(req, n2alg->derived.hmac_type,
643 				  n2alg->derived.hw_op_hashsz, ds,
644 				  &rctx->u,
645 				  __pa(&ctx->hash_key),
646 				  ctx->hash_key_len);
647 }
648 
649 struct n2_cipher_context {
650 	int			key_len;
651 	int			enc_type;
652 	union {
653 		u8		aes[AES_MAX_KEY_SIZE];
654 		u8		des[DES_KEY_SIZE];
655 		u8		des3[3 * DES_KEY_SIZE];
656 		u8		arc4[258]; /* S-box, X, Y */
657 	} key;
658 };
659 
660 #define N2_CHUNK_ARR_LEN	16
661 
662 struct n2_crypto_chunk {
663 	struct list_head	entry;
664 	unsigned long		iv_paddr : 44;
665 	unsigned long		arr_len : 20;
666 	unsigned long		dest_paddr;
667 	unsigned long		dest_final;
668 	struct {
669 		unsigned long	src_paddr : 44;
670 		unsigned long	src_len : 20;
671 	} arr[N2_CHUNK_ARR_LEN];
672 };
673 
674 struct n2_request_context {
675 	struct ablkcipher_walk	walk;
676 	struct list_head	chunk_list;
677 	struct n2_crypto_chunk	chunk;
678 	u8			temp_iv[16];
679 };
680 
681 /* The SPU allows some level of flexibility for partial cipher blocks
682  * being specified in a descriptor.
683  *
684  * It merely requires that every descriptor's length field is at least
685  * as large as the cipher block size.  This means that a cipher block
686  * can span at most 2 descriptors.  However, this does not allow a
687  * partial block to span into the final descriptor as that would
688  * violate the rule (since every descriptor's length must be at lest
689  * the block size).  So, for example, assuming an 8 byte block size:
690  *
691  *	0xe --> 0xa --> 0x8
692  *
693  * is a valid length sequence, whereas:
694  *
695  *	0xe --> 0xb --> 0x7
696  *
697  * is not a valid sequence.
698  */
699 
700 struct n2_cipher_alg {
701 	struct list_head	entry;
702 	u8			enc_type;
703 	struct crypto_alg	alg;
704 };
705 
706 static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
707 {
708 	struct crypto_alg *alg = tfm->__crt_alg;
709 
710 	return container_of(alg, struct n2_cipher_alg, alg);
711 }
712 
713 struct n2_cipher_request_context {
714 	struct ablkcipher_walk	walk;
715 };
716 
717 static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
718 			 unsigned int keylen)
719 {
720 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
721 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
722 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
723 
724 	ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
725 
726 	switch (keylen) {
727 	case AES_KEYSIZE_128:
728 		ctx->enc_type |= ENC_TYPE_ALG_AES128;
729 		break;
730 	case AES_KEYSIZE_192:
731 		ctx->enc_type |= ENC_TYPE_ALG_AES192;
732 		break;
733 	case AES_KEYSIZE_256:
734 		ctx->enc_type |= ENC_TYPE_ALG_AES256;
735 		break;
736 	default:
737 		crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
738 		return -EINVAL;
739 	}
740 
741 	ctx->key_len = keylen;
742 	memcpy(ctx->key.aes, key, keylen);
743 	return 0;
744 }
745 
746 static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
747 			 unsigned int keylen)
748 {
749 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
750 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
751 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
752 	u32 tmp[DES_EXPKEY_WORDS];
753 	int err;
754 
755 	ctx->enc_type = n2alg->enc_type;
756 
757 	if (keylen != DES_KEY_SIZE) {
758 		crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
759 		return -EINVAL;
760 	}
761 
762 	err = des_ekey(tmp, key);
763 	if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
764 		tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
765 		return -EINVAL;
766 	}
767 
768 	ctx->key_len = keylen;
769 	memcpy(ctx->key.des, key, keylen);
770 	return 0;
771 }
772 
773 static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
774 			  unsigned int keylen)
775 {
776 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
777 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
778 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
779 
780 	ctx->enc_type = n2alg->enc_type;
781 
782 	if (keylen != (3 * DES_KEY_SIZE)) {
783 		crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
784 		return -EINVAL;
785 	}
786 	ctx->key_len = keylen;
787 	memcpy(ctx->key.des3, key, keylen);
788 	return 0;
789 }
790 
791 static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
792 			  unsigned int keylen)
793 {
794 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
795 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
796 	struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
797 	u8 *s = ctx->key.arc4;
798 	u8 *x = s + 256;
799 	u8 *y = x + 1;
800 	int i, j, k;
801 
802 	ctx->enc_type = n2alg->enc_type;
803 
804 	j = k = 0;
805 	*x = 0;
806 	*y = 0;
807 	for (i = 0; i < 256; i++)
808 		s[i] = i;
809 	for (i = 0; i < 256; i++) {
810 		u8 a = s[i];
811 		j = (j + key[k] + a) & 0xff;
812 		s[i] = s[j];
813 		s[j] = a;
814 		if (++k >= keylen)
815 			k = 0;
816 	}
817 
818 	return 0;
819 }
820 
821 static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
822 {
823 	int this_len = nbytes;
824 
825 	this_len -= (nbytes & (block_size - 1));
826 	return this_len > (1 << 16) ? (1 << 16) : this_len;
827 }
828 
829 static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
830 			    struct spu_queue *qp, bool encrypt)
831 {
832 	struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
833 	struct cwq_initial_entry *ent;
834 	bool in_place;
835 	int i;
836 
837 	ent = spu_queue_alloc(qp, cp->arr_len);
838 	if (!ent) {
839 		pr_info("queue_alloc() of %d fails\n",
840 			cp->arr_len);
841 		return -EBUSY;
842 	}
843 
844 	in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
845 
846 	ent->control = control_word_base(cp->arr[0].src_len,
847 					 0, ctx->enc_type, 0, 0,
848 					 false, true, false, encrypt,
849 					 OPCODE_ENCRYPT |
850 					 (in_place ? OPCODE_INPLACE_BIT : 0));
851 	ent->src_addr = cp->arr[0].src_paddr;
852 	ent->auth_key_addr = 0UL;
853 	ent->auth_iv_addr = 0UL;
854 	ent->final_auth_state_addr = 0UL;
855 	ent->enc_key_addr = __pa(&ctx->key);
856 	ent->enc_iv_addr = cp->iv_paddr;
857 	ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
858 
859 	for (i = 1; i < cp->arr_len; i++) {
860 		ent = spu_queue_next(qp, ent);
861 
862 		ent->control = cp->arr[i].src_len - 1;
863 		ent->src_addr = cp->arr[i].src_paddr;
864 		ent->auth_key_addr = 0UL;
865 		ent->auth_iv_addr = 0UL;
866 		ent->final_auth_state_addr = 0UL;
867 		ent->enc_key_addr = 0UL;
868 		ent->enc_iv_addr = 0UL;
869 		ent->dest_addr = 0UL;
870 	}
871 	ent->control |= CONTROL_END_OF_BLOCK;
872 
873 	return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
874 }
875 
876 static int n2_compute_chunks(struct ablkcipher_request *req)
877 {
878 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
879 	struct ablkcipher_walk *walk = &rctx->walk;
880 	struct n2_crypto_chunk *chunk;
881 	unsigned long dest_prev;
882 	unsigned int tot_len;
883 	bool prev_in_place;
884 	int err, nbytes;
885 
886 	ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
887 	err = ablkcipher_walk_phys(req, walk);
888 	if (err)
889 		return err;
890 
891 	INIT_LIST_HEAD(&rctx->chunk_list);
892 
893 	chunk = &rctx->chunk;
894 	INIT_LIST_HEAD(&chunk->entry);
895 
896 	chunk->iv_paddr = 0UL;
897 	chunk->arr_len = 0;
898 	chunk->dest_paddr = 0UL;
899 
900 	prev_in_place = false;
901 	dest_prev = ~0UL;
902 	tot_len = 0;
903 
904 	while ((nbytes = walk->nbytes) != 0) {
905 		unsigned long dest_paddr, src_paddr;
906 		bool in_place;
907 		int this_len;
908 
909 		src_paddr = (page_to_phys(walk->src.page) +
910 			     walk->src.offset);
911 		dest_paddr = (page_to_phys(walk->dst.page) +
912 			      walk->dst.offset);
913 		in_place = (src_paddr == dest_paddr);
914 		this_len = cipher_descriptor_len(nbytes, walk->blocksize);
915 
916 		if (chunk->arr_len != 0) {
917 			if (in_place != prev_in_place ||
918 			    (!prev_in_place &&
919 			     dest_paddr != dest_prev) ||
920 			    chunk->arr_len == N2_CHUNK_ARR_LEN ||
921 			    tot_len + this_len > (1 << 16)) {
922 				chunk->dest_final = dest_prev;
923 				list_add_tail(&chunk->entry,
924 					      &rctx->chunk_list);
925 				chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
926 				if (!chunk) {
927 					err = -ENOMEM;
928 					break;
929 				}
930 				INIT_LIST_HEAD(&chunk->entry);
931 			}
932 		}
933 		if (chunk->arr_len == 0) {
934 			chunk->dest_paddr = dest_paddr;
935 			tot_len = 0;
936 		}
937 		chunk->arr[chunk->arr_len].src_paddr = src_paddr;
938 		chunk->arr[chunk->arr_len].src_len = this_len;
939 		chunk->arr_len++;
940 
941 		dest_prev = dest_paddr + this_len;
942 		prev_in_place = in_place;
943 		tot_len += this_len;
944 
945 		err = ablkcipher_walk_done(req, walk, nbytes - this_len);
946 		if (err)
947 			break;
948 	}
949 	if (!err && chunk->arr_len != 0) {
950 		chunk->dest_final = dest_prev;
951 		list_add_tail(&chunk->entry, &rctx->chunk_list);
952 	}
953 
954 	return err;
955 }
956 
957 static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
958 {
959 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
960 	struct n2_crypto_chunk *c, *tmp;
961 
962 	if (final_iv)
963 		memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
964 
965 	ablkcipher_walk_complete(&rctx->walk);
966 	list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
967 		list_del(&c->entry);
968 		if (unlikely(c != &rctx->chunk))
969 			kfree(c);
970 	}
971 
972 }
973 
974 static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
975 {
976 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
977 	struct crypto_tfm *tfm = req->base.tfm;
978 	int err = n2_compute_chunks(req);
979 	struct n2_crypto_chunk *c, *tmp;
980 	unsigned long flags, hv_ret;
981 	struct spu_queue *qp;
982 
983 	if (err)
984 		return err;
985 
986 	qp = cpu_to_cwq[get_cpu()];
987 	err = -ENODEV;
988 	if (!qp)
989 		goto out;
990 
991 	spin_lock_irqsave(&qp->lock, flags);
992 
993 	list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
994 		err = __n2_crypt_chunk(tfm, c, qp, encrypt);
995 		if (err)
996 			break;
997 		list_del(&c->entry);
998 		if (unlikely(c != &rctx->chunk))
999 			kfree(c);
1000 	}
1001 	if (!err) {
1002 		hv_ret = wait_for_tail(qp);
1003 		if (hv_ret != HV_EOK)
1004 			err = -EINVAL;
1005 	}
1006 
1007 	spin_unlock_irqrestore(&qp->lock, flags);
1008 
1009 	put_cpu();
1010 
1011 out:
1012 	n2_chunk_complete(req, NULL);
1013 	return err;
1014 }
1015 
1016 static int n2_encrypt_ecb(struct ablkcipher_request *req)
1017 {
1018 	return n2_do_ecb(req, true);
1019 }
1020 
1021 static int n2_decrypt_ecb(struct ablkcipher_request *req)
1022 {
1023 	return n2_do_ecb(req, false);
1024 }
1025 
1026 static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
1027 {
1028 	struct n2_request_context *rctx = ablkcipher_request_ctx(req);
1029 	struct crypto_tfm *tfm = req->base.tfm;
1030 	unsigned long flags, hv_ret, iv_paddr;
1031 	int err = n2_compute_chunks(req);
1032 	struct n2_crypto_chunk *c, *tmp;
1033 	struct spu_queue *qp;
1034 	void *final_iv_addr;
1035 
1036 	final_iv_addr = NULL;
1037 
1038 	if (err)
1039 		return err;
1040 
1041 	qp = cpu_to_cwq[get_cpu()];
1042 	err = -ENODEV;
1043 	if (!qp)
1044 		goto out;
1045 
1046 	spin_lock_irqsave(&qp->lock, flags);
1047 
1048 	if (encrypt) {
1049 		iv_paddr = __pa(rctx->walk.iv);
1050 		list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
1051 					 entry) {
1052 			c->iv_paddr = iv_paddr;
1053 			err = __n2_crypt_chunk(tfm, c, qp, true);
1054 			if (err)
1055 				break;
1056 			iv_paddr = c->dest_final - rctx->walk.blocksize;
1057 			list_del(&c->entry);
1058 			if (unlikely(c != &rctx->chunk))
1059 				kfree(c);
1060 		}
1061 		final_iv_addr = __va(iv_paddr);
1062 	} else {
1063 		list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1064 						 entry) {
1065 			if (c == &rctx->chunk) {
1066 				iv_paddr = __pa(rctx->walk.iv);
1067 			} else {
1068 				iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1069 					    tmp->arr[tmp->arr_len-1].src_len -
1070 					    rctx->walk.blocksize);
1071 			}
1072 			if (!final_iv_addr) {
1073 				unsigned long pa;
1074 
1075 				pa = (c->arr[c->arr_len-1].src_paddr +
1076 				      c->arr[c->arr_len-1].src_len -
1077 				      rctx->walk.blocksize);
1078 				final_iv_addr = rctx->temp_iv;
1079 				memcpy(rctx->temp_iv, __va(pa),
1080 				       rctx->walk.blocksize);
1081 			}
1082 			c->iv_paddr = iv_paddr;
1083 			err = __n2_crypt_chunk(tfm, c, qp, false);
1084 			if (err)
1085 				break;
1086 			list_del(&c->entry);
1087 			if (unlikely(c != &rctx->chunk))
1088 				kfree(c);
1089 		}
1090 	}
1091 	if (!err) {
1092 		hv_ret = wait_for_tail(qp);
1093 		if (hv_ret != HV_EOK)
1094 			err = -EINVAL;
1095 	}
1096 
1097 	spin_unlock_irqrestore(&qp->lock, flags);
1098 
1099 	put_cpu();
1100 
1101 out:
1102 	n2_chunk_complete(req, err ? NULL : final_iv_addr);
1103 	return err;
1104 }
1105 
1106 static int n2_encrypt_chaining(struct ablkcipher_request *req)
1107 {
1108 	return n2_do_chaining(req, true);
1109 }
1110 
1111 static int n2_decrypt_chaining(struct ablkcipher_request *req)
1112 {
1113 	return n2_do_chaining(req, false);
1114 }
1115 
1116 struct n2_cipher_tmpl {
1117 	const char		*name;
1118 	const char		*drv_name;
1119 	u8			block_size;
1120 	u8			enc_type;
1121 	struct ablkcipher_alg	ablkcipher;
1122 };
1123 
1124 static const struct n2_cipher_tmpl cipher_tmpls[] = {
1125 	/* ARC4: only ECB is supported (chaining bits ignored) */
1126 	{	.name		= "ecb(arc4)",
1127 		.drv_name	= "ecb-arc4",
1128 		.block_size	= 1,
1129 		.enc_type	= (ENC_TYPE_ALG_RC4_STREAM |
1130 				   ENC_TYPE_CHAINING_ECB),
1131 		.ablkcipher	= {
1132 			.min_keysize	= 1,
1133 			.max_keysize	= 256,
1134 			.setkey		= n2_arc4_setkey,
1135 			.encrypt	= n2_encrypt_ecb,
1136 			.decrypt	= n2_decrypt_ecb,
1137 		},
1138 	},
1139 
1140 	/* DES: ECB CBC and CFB are supported */
1141 	{	.name		= "ecb(des)",
1142 		.drv_name	= "ecb-des",
1143 		.block_size	= DES_BLOCK_SIZE,
1144 		.enc_type	= (ENC_TYPE_ALG_DES |
1145 				   ENC_TYPE_CHAINING_ECB),
1146 		.ablkcipher	= {
1147 			.min_keysize	= DES_KEY_SIZE,
1148 			.max_keysize	= DES_KEY_SIZE,
1149 			.setkey		= n2_des_setkey,
1150 			.encrypt	= n2_encrypt_ecb,
1151 			.decrypt	= n2_decrypt_ecb,
1152 		},
1153 	},
1154 	{	.name		= "cbc(des)",
1155 		.drv_name	= "cbc-des",
1156 		.block_size	= DES_BLOCK_SIZE,
1157 		.enc_type	= (ENC_TYPE_ALG_DES |
1158 				   ENC_TYPE_CHAINING_CBC),
1159 		.ablkcipher	= {
1160 			.ivsize		= DES_BLOCK_SIZE,
1161 			.min_keysize	= DES_KEY_SIZE,
1162 			.max_keysize	= DES_KEY_SIZE,
1163 			.setkey		= n2_des_setkey,
1164 			.encrypt	= n2_encrypt_chaining,
1165 			.decrypt	= n2_decrypt_chaining,
1166 		},
1167 	},
1168 	{	.name		= "cfb(des)",
1169 		.drv_name	= "cfb-des",
1170 		.block_size	= DES_BLOCK_SIZE,
1171 		.enc_type	= (ENC_TYPE_ALG_DES |
1172 				   ENC_TYPE_CHAINING_CFB),
1173 		.ablkcipher	= {
1174 			.min_keysize	= DES_KEY_SIZE,
1175 			.max_keysize	= DES_KEY_SIZE,
1176 			.setkey		= n2_des_setkey,
1177 			.encrypt	= n2_encrypt_chaining,
1178 			.decrypt	= n2_decrypt_chaining,
1179 		},
1180 	},
1181 
1182 	/* 3DES: ECB CBC and CFB are supported */
1183 	{	.name		= "ecb(des3_ede)",
1184 		.drv_name	= "ecb-3des",
1185 		.block_size	= DES_BLOCK_SIZE,
1186 		.enc_type	= (ENC_TYPE_ALG_3DES |
1187 				   ENC_TYPE_CHAINING_ECB),
1188 		.ablkcipher	= {
1189 			.min_keysize	= 3 * DES_KEY_SIZE,
1190 			.max_keysize	= 3 * DES_KEY_SIZE,
1191 			.setkey		= n2_3des_setkey,
1192 			.encrypt	= n2_encrypt_ecb,
1193 			.decrypt	= n2_decrypt_ecb,
1194 		},
1195 	},
1196 	{	.name		= "cbc(des3_ede)",
1197 		.drv_name	= "cbc-3des",
1198 		.block_size	= DES_BLOCK_SIZE,
1199 		.enc_type	= (ENC_TYPE_ALG_3DES |
1200 				   ENC_TYPE_CHAINING_CBC),
1201 		.ablkcipher	= {
1202 			.ivsize		= DES_BLOCK_SIZE,
1203 			.min_keysize	= 3 * DES_KEY_SIZE,
1204 			.max_keysize	= 3 * DES_KEY_SIZE,
1205 			.setkey		= n2_3des_setkey,
1206 			.encrypt	= n2_encrypt_chaining,
1207 			.decrypt	= n2_decrypt_chaining,
1208 		},
1209 	},
1210 	{	.name		= "cfb(des3_ede)",
1211 		.drv_name	= "cfb-3des",
1212 		.block_size	= DES_BLOCK_SIZE,
1213 		.enc_type	= (ENC_TYPE_ALG_3DES |
1214 				   ENC_TYPE_CHAINING_CFB),
1215 		.ablkcipher	= {
1216 			.min_keysize	= 3 * DES_KEY_SIZE,
1217 			.max_keysize	= 3 * DES_KEY_SIZE,
1218 			.setkey		= n2_3des_setkey,
1219 			.encrypt	= n2_encrypt_chaining,
1220 			.decrypt	= n2_decrypt_chaining,
1221 		},
1222 	},
1223 	/* AES: ECB CBC and CTR are supported */
1224 	{	.name		= "ecb(aes)",
1225 		.drv_name	= "ecb-aes",
1226 		.block_size	= AES_BLOCK_SIZE,
1227 		.enc_type	= (ENC_TYPE_ALG_AES128 |
1228 				   ENC_TYPE_CHAINING_ECB),
1229 		.ablkcipher	= {
1230 			.min_keysize	= AES_MIN_KEY_SIZE,
1231 			.max_keysize	= AES_MAX_KEY_SIZE,
1232 			.setkey		= n2_aes_setkey,
1233 			.encrypt	= n2_encrypt_ecb,
1234 			.decrypt	= n2_decrypt_ecb,
1235 		},
1236 	},
1237 	{	.name		= "cbc(aes)",
1238 		.drv_name	= "cbc-aes",
1239 		.block_size	= AES_BLOCK_SIZE,
1240 		.enc_type	= (ENC_TYPE_ALG_AES128 |
1241 				   ENC_TYPE_CHAINING_CBC),
1242 		.ablkcipher	= {
1243 			.ivsize		= AES_BLOCK_SIZE,
1244 			.min_keysize	= AES_MIN_KEY_SIZE,
1245 			.max_keysize	= AES_MAX_KEY_SIZE,
1246 			.setkey		= n2_aes_setkey,
1247 			.encrypt	= n2_encrypt_chaining,
1248 			.decrypt	= n2_decrypt_chaining,
1249 		},
1250 	},
1251 	{	.name		= "ctr(aes)",
1252 		.drv_name	= "ctr-aes",
1253 		.block_size	= AES_BLOCK_SIZE,
1254 		.enc_type	= (ENC_TYPE_ALG_AES128 |
1255 				   ENC_TYPE_CHAINING_COUNTER),
1256 		.ablkcipher	= {
1257 			.ivsize		= AES_BLOCK_SIZE,
1258 			.min_keysize	= AES_MIN_KEY_SIZE,
1259 			.max_keysize	= AES_MAX_KEY_SIZE,
1260 			.setkey		= n2_aes_setkey,
1261 			.encrypt	= n2_encrypt_chaining,
1262 			.decrypt	= n2_encrypt_chaining,
1263 		},
1264 	},
1265 
1266 };
1267 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1268 
1269 static LIST_HEAD(cipher_algs);
1270 
1271 struct n2_hash_tmpl {
1272 	const char	*name;
1273 	const char	*hash_zero;
1274 	const u32	*hash_init;
1275 	u8		hw_op_hashsz;
1276 	u8		digest_size;
1277 	u8		block_size;
1278 	u8		auth_type;
1279 	u8		hmac_type;
1280 };
1281 
1282 static const char md5_zero[MD5_DIGEST_SIZE] = {
1283 	0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
1284 	0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
1285 };
1286 static const u32 md5_init[MD5_HASH_WORDS] = {
1287 	cpu_to_le32(0x67452301),
1288 	cpu_to_le32(0xefcdab89),
1289 	cpu_to_le32(0x98badcfe),
1290 	cpu_to_le32(0x10325476),
1291 };
1292 static const char sha1_zero[SHA1_DIGEST_SIZE] = {
1293 	0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
1294 	0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
1295 	0x07, 0x09
1296 };
1297 static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
1298 	SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
1299 };
1300 static const char sha256_zero[SHA256_DIGEST_SIZE] = {
1301 	0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
1302 	0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
1303 	0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
1304 	0x1b, 0x78, 0x52, 0xb8, 0x55
1305 };
1306 static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
1307 	SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
1308 	SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
1309 };
1310 static const char sha224_zero[SHA224_DIGEST_SIZE] = {
1311 	0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
1312 	0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
1313 	0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
1314 	0x2f
1315 };
1316 static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
1317 	SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
1318 	SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
1319 };
1320 
1321 static const struct n2_hash_tmpl hash_tmpls[] = {
1322 	{ .name		= "md5",
1323 	  .hash_zero	= md5_zero,
1324 	  .hash_init	= md5_init,
1325 	  .auth_type	= AUTH_TYPE_MD5,
1326 	  .hmac_type	= AUTH_TYPE_HMAC_MD5,
1327 	  .hw_op_hashsz	= MD5_DIGEST_SIZE,
1328 	  .digest_size	= MD5_DIGEST_SIZE,
1329 	  .block_size	= MD5_HMAC_BLOCK_SIZE },
1330 	{ .name		= "sha1",
1331 	  .hash_zero	= sha1_zero,
1332 	  .hash_init	= sha1_init,
1333 	  .auth_type	= AUTH_TYPE_SHA1,
1334 	  .hmac_type	= AUTH_TYPE_HMAC_SHA1,
1335 	  .hw_op_hashsz	= SHA1_DIGEST_SIZE,
1336 	  .digest_size	= SHA1_DIGEST_SIZE,
1337 	  .block_size	= SHA1_BLOCK_SIZE },
1338 	{ .name		= "sha256",
1339 	  .hash_zero	= sha256_zero,
1340 	  .hash_init	= sha256_init,
1341 	  .auth_type	= AUTH_TYPE_SHA256,
1342 	  .hmac_type	= AUTH_TYPE_HMAC_SHA256,
1343 	  .hw_op_hashsz	= SHA256_DIGEST_SIZE,
1344 	  .digest_size	= SHA256_DIGEST_SIZE,
1345 	  .block_size	= SHA256_BLOCK_SIZE },
1346 	{ .name		= "sha224",
1347 	  .hash_zero	= sha224_zero,
1348 	  .hash_init	= sha224_init,
1349 	  .auth_type	= AUTH_TYPE_SHA256,
1350 	  .hmac_type	= AUTH_TYPE_RESERVED,
1351 	  .hw_op_hashsz	= SHA256_DIGEST_SIZE,
1352 	  .digest_size	= SHA224_DIGEST_SIZE,
1353 	  .block_size	= SHA224_BLOCK_SIZE },
1354 };
1355 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1356 
1357 static LIST_HEAD(ahash_algs);
1358 static LIST_HEAD(hmac_algs);
1359 
1360 static int algs_registered;
1361 
1362 static void __n2_unregister_algs(void)
1363 {
1364 	struct n2_cipher_alg *cipher, *cipher_tmp;
1365 	struct n2_ahash_alg *alg, *alg_tmp;
1366 	struct n2_hmac_alg *hmac, *hmac_tmp;
1367 
1368 	list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1369 		crypto_unregister_alg(&cipher->alg);
1370 		list_del(&cipher->entry);
1371 		kfree(cipher);
1372 	}
1373 	list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
1374 		crypto_unregister_ahash(&hmac->derived.alg);
1375 		list_del(&hmac->derived.entry);
1376 		kfree(hmac);
1377 	}
1378 	list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1379 		crypto_unregister_ahash(&alg->alg);
1380 		list_del(&alg->entry);
1381 		kfree(alg);
1382 	}
1383 }
1384 
1385 static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1386 {
1387 	tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1388 	return 0;
1389 }
1390 
1391 static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
1392 {
1393 	struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1394 	struct crypto_alg *alg;
1395 	int err;
1396 
1397 	if (!p)
1398 		return -ENOMEM;
1399 
1400 	alg = &p->alg;
1401 
1402 	snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1403 	snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1404 	alg->cra_priority = N2_CRA_PRIORITY;
1405 	alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1406 	alg->cra_blocksize = tmpl->block_size;
1407 	p->enc_type = tmpl->enc_type;
1408 	alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1409 	alg->cra_type = &crypto_ablkcipher_type;
1410 	alg->cra_u.ablkcipher = tmpl->ablkcipher;
1411 	alg->cra_init = n2_cipher_cra_init;
1412 	alg->cra_module = THIS_MODULE;
1413 
1414 	list_add(&p->entry, &cipher_algs);
1415 	err = crypto_register_alg(alg);
1416 	if (err) {
1417 		pr_err("%s alg registration failed\n", alg->cra_name);
1418 		list_del(&p->entry);
1419 		kfree(p);
1420 	} else {
1421 		pr_info("%s alg registered\n", alg->cra_name);
1422 	}
1423 	return err;
1424 }
1425 
1426 static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
1427 {
1428 	struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1429 	struct ahash_alg *ahash;
1430 	struct crypto_alg *base;
1431 	int err;
1432 
1433 	if (!p)
1434 		return -ENOMEM;
1435 
1436 	p->child_alg = n2ahash->alg.halg.base.cra_name;
1437 	memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
1438 	INIT_LIST_HEAD(&p->derived.entry);
1439 
1440 	ahash = &p->derived.alg;
1441 	ahash->digest = n2_hmac_async_digest;
1442 	ahash->setkey = n2_hmac_async_setkey;
1443 
1444 	base = &ahash->halg.base;
1445 	snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
1446 	snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
1447 
1448 	base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
1449 	base->cra_init = n2_hmac_cra_init;
1450 	base->cra_exit = n2_hmac_cra_exit;
1451 
1452 	list_add(&p->derived.entry, &hmac_algs);
1453 	err = crypto_register_ahash(ahash);
1454 	if (err) {
1455 		pr_err("%s alg registration failed\n", base->cra_name);
1456 		list_del(&p->derived.entry);
1457 		kfree(p);
1458 	} else {
1459 		pr_info("%s alg registered\n", base->cra_name);
1460 	}
1461 	return err;
1462 }
1463 
1464 static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
1465 {
1466 	struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1467 	struct hash_alg_common *halg;
1468 	struct crypto_alg *base;
1469 	struct ahash_alg *ahash;
1470 	int err;
1471 
1472 	if (!p)
1473 		return -ENOMEM;
1474 
1475 	p->hash_zero = tmpl->hash_zero;
1476 	p->hash_init = tmpl->hash_init;
1477 	p->auth_type = tmpl->auth_type;
1478 	p->hmac_type = tmpl->hmac_type;
1479 	p->hw_op_hashsz = tmpl->hw_op_hashsz;
1480 	p->digest_size = tmpl->digest_size;
1481 
1482 	ahash = &p->alg;
1483 	ahash->init = n2_hash_async_init;
1484 	ahash->update = n2_hash_async_update;
1485 	ahash->final = n2_hash_async_final;
1486 	ahash->finup = n2_hash_async_finup;
1487 	ahash->digest = n2_hash_async_digest;
1488 
1489 	halg = &ahash->halg;
1490 	halg->digestsize = tmpl->digest_size;
1491 
1492 	base = &halg->base;
1493 	snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1494 	snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1495 	base->cra_priority = N2_CRA_PRIORITY;
1496 	base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK;
1497 	base->cra_blocksize = tmpl->block_size;
1498 	base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1499 	base->cra_module = THIS_MODULE;
1500 	base->cra_init = n2_hash_cra_init;
1501 	base->cra_exit = n2_hash_cra_exit;
1502 
1503 	list_add(&p->entry, &ahash_algs);
1504 	err = crypto_register_ahash(ahash);
1505 	if (err) {
1506 		pr_err("%s alg registration failed\n", base->cra_name);
1507 		list_del(&p->entry);
1508 		kfree(p);
1509 	} else {
1510 		pr_info("%s alg registered\n", base->cra_name);
1511 	}
1512 	if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
1513 		err = __n2_register_one_hmac(p);
1514 	return err;
1515 }
1516 
1517 static int __devinit n2_register_algs(void)
1518 {
1519 	int i, err = 0;
1520 
1521 	mutex_lock(&spu_lock);
1522 	if (algs_registered++)
1523 		goto out;
1524 
1525 	for (i = 0; i < NUM_HASH_TMPLS; i++) {
1526 		err = __n2_register_one_ahash(&hash_tmpls[i]);
1527 		if (err) {
1528 			__n2_unregister_algs();
1529 			goto out;
1530 		}
1531 	}
1532 	for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1533 		err = __n2_register_one_cipher(&cipher_tmpls[i]);
1534 		if (err) {
1535 			__n2_unregister_algs();
1536 			goto out;
1537 		}
1538 	}
1539 
1540 out:
1541 	mutex_unlock(&spu_lock);
1542 	return err;
1543 }
1544 
1545 static void __devexit n2_unregister_algs(void)
1546 {
1547 	mutex_lock(&spu_lock);
1548 	if (!--algs_registered)
1549 		__n2_unregister_algs();
1550 	mutex_unlock(&spu_lock);
1551 }
1552 
1553 /* To map CWQ queues to interrupt sources, the hypervisor API provides
1554  * a devino.  This isn't very useful to us because all of the
1555  * interrupts listed in the device_node have been translated to
1556  * Linux virtual IRQ cookie numbers.
1557  *
1558  * So we have to back-translate, going through the 'intr' and 'ino'
1559  * property tables of the n2cp MDESC node, matching it with the OF
1560  * 'interrupts' property entries, in order to to figure out which
1561  * devino goes to which already-translated IRQ.
1562  */
1563 static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
1564 			     unsigned long dev_ino)
1565 {
1566 	const unsigned int *dev_intrs;
1567 	unsigned int intr;
1568 	int i;
1569 
1570 	for (i = 0; i < ip->num_intrs; i++) {
1571 		if (ip->ino_table[i].ino == dev_ino)
1572 			break;
1573 	}
1574 	if (i == ip->num_intrs)
1575 		return -ENODEV;
1576 
1577 	intr = ip->ino_table[i].intr;
1578 
1579 	dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
1580 	if (!dev_intrs)
1581 		return -ENODEV;
1582 
1583 	for (i = 0; i < dev->archdata.num_irqs; i++) {
1584 		if (dev_intrs[i] == intr)
1585 			return i;
1586 	}
1587 
1588 	return -ENODEV;
1589 }
1590 
1591 static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
1592 		       const char *irq_name, struct spu_queue *p,
1593 		       irq_handler_t handler)
1594 {
1595 	unsigned long herr;
1596 	int index;
1597 
1598 	herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1599 	if (herr)
1600 		return -EINVAL;
1601 
1602 	index = find_devino_index(dev, ip, p->devino);
1603 	if (index < 0)
1604 		return index;
1605 
1606 	p->irq = dev->archdata.irqs[index];
1607 
1608 	sprintf(p->irq_name, "%s-%d", irq_name, index);
1609 
1610 	return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,
1611 			   p->irq_name, p);
1612 }
1613 
1614 static struct kmem_cache *queue_cache[2];
1615 
1616 static void *new_queue(unsigned long q_type)
1617 {
1618 	return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1619 }
1620 
1621 static void free_queue(void *p, unsigned long q_type)
1622 {
1623 	return kmem_cache_free(queue_cache[q_type - 1], p);
1624 }
1625 
1626 static int queue_cache_init(void)
1627 {
1628 	if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1629 		queue_cache[HV_NCS_QTYPE_MAU - 1] =
1630 			kmem_cache_create("mau_queue",
1631 					  (MAU_NUM_ENTRIES *
1632 					   MAU_ENTRY_SIZE),
1633 					  MAU_ENTRY_SIZE, 0, NULL);
1634 	if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1635 		return -ENOMEM;
1636 
1637 	if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1638 		queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1639 			kmem_cache_create("cwq_queue",
1640 					  (CWQ_NUM_ENTRIES *
1641 					   CWQ_ENTRY_SIZE),
1642 					  CWQ_ENTRY_SIZE, 0, NULL);
1643 	if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1644 		kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1645 		return -ENOMEM;
1646 	}
1647 	return 0;
1648 }
1649 
1650 static void queue_cache_destroy(void)
1651 {
1652 	kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1653 	kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1654 }
1655 
1656 static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1657 {
1658 	cpumask_var_t old_allowed;
1659 	unsigned long hv_ret;
1660 
1661 	if (cpumask_empty(&p->sharing))
1662 		return -EINVAL;
1663 
1664 	if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1665 		return -ENOMEM;
1666 
1667 	cpumask_copy(old_allowed, &current->cpus_allowed);
1668 
1669 	set_cpus_allowed_ptr(current, &p->sharing);
1670 
1671 	hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1672 				 CWQ_NUM_ENTRIES, &p->qhandle);
1673 	if (!hv_ret)
1674 		sun4v_ncs_sethead_marker(p->qhandle, 0);
1675 
1676 	set_cpus_allowed_ptr(current, old_allowed);
1677 
1678 	free_cpumask_var(old_allowed);
1679 
1680 	return (hv_ret ? -EINVAL : 0);
1681 }
1682 
1683 static int spu_queue_setup(struct spu_queue *p)
1684 {
1685 	int err;
1686 
1687 	p->q = new_queue(p->q_type);
1688 	if (!p->q)
1689 		return -ENOMEM;
1690 
1691 	err = spu_queue_register(p, p->q_type);
1692 	if (err) {
1693 		free_queue(p->q, p->q_type);
1694 		p->q = NULL;
1695 	}
1696 
1697 	return err;
1698 }
1699 
1700 static void spu_queue_destroy(struct spu_queue *p)
1701 {
1702 	unsigned long hv_ret;
1703 
1704 	if (!p->q)
1705 		return;
1706 
1707 	hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1708 
1709 	if (!hv_ret)
1710 		free_queue(p->q, p->q_type);
1711 }
1712 
1713 static void spu_list_destroy(struct list_head *list)
1714 {
1715 	struct spu_queue *p, *n;
1716 
1717 	list_for_each_entry_safe(p, n, list, list) {
1718 		int i;
1719 
1720 		for (i = 0; i < NR_CPUS; i++) {
1721 			if (cpu_to_cwq[i] == p)
1722 				cpu_to_cwq[i] = NULL;
1723 		}
1724 
1725 		if (p->irq) {
1726 			free_irq(p->irq, p);
1727 			p->irq = 0;
1728 		}
1729 		spu_queue_destroy(p);
1730 		list_del(&p->list);
1731 		kfree(p);
1732 	}
1733 }
1734 
1735 /* Walk the backward arcs of a CWQ 'exec-unit' node,
1736  * gathering cpu membership information.
1737  */
1738 static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1739 			       struct platform_device *dev,
1740 			       u64 node, struct spu_queue *p,
1741 			       struct spu_queue **table)
1742 {
1743 	u64 arc;
1744 
1745 	mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1746 		u64 tgt = mdesc_arc_target(mdesc, arc);
1747 		const char *name = mdesc_node_name(mdesc, tgt);
1748 		const u64 *id;
1749 
1750 		if (strcmp(name, "cpu"))
1751 			continue;
1752 		id = mdesc_get_property(mdesc, tgt, "id", NULL);
1753 		if (table[*id] != NULL) {
1754 			dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
1755 				dev->dev.of_node->full_name);
1756 			return -EINVAL;
1757 		}
1758 		cpu_set(*id, p->sharing);
1759 		table[*id] = p;
1760 	}
1761 	return 0;
1762 }
1763 
1764 /* Process an 'exec-unit' MDESC node of type 'cwq'.  */
1765 static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1766 			    struct platform_device *dev, struct mdesc_handle *mdesc,
1767 			    u64 node, const char *iname, unsigned long q_type,
1768 			    irq_handler_t handler, struct spu_queue **table)
1769 {
1770 	struct spu_queue *p;
1771 	int err;
1772 
1773 	p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1774 	if (!p) {
1775 		dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
1776 			dev->dev.of_node->full_name);
1777 		return -ENOMEM;
1778 	}
1779 
1780 	cpus_clear(p->sharing);
1781 	spin_lock_init(&p->lock);
1782 	p->q_type = q_type;
1783 	INIT_LIST_HEAD(&p->jobs);
1784 	list_add(&p->list, list);
1785 
1786 	err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1787 	if (err)
1788 		return err;
1789 
1790 	err = spu_queue_setup(p);
1791 	if (err)
1792 		return err;
1793 
1794 	return spu_map_ino(dev, ip, iname, p, handler);
1795 }
1796 
1797 static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
1798 			  struct spu_mdesc_info *ip, struct list_head *list,
1799 			  const char *exec_name, unsigned long q_type,
1800 			  irq_handler_t handler, struct spu_queue **table)
1801 {
1802 	int err = 0;
1803 	u64 node;
1804 
1805 	mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1806 		const char *type;
1807 
1808 		type = mdesc_get_property(mdesc, node, "type", NULL);
1809 		if (!type || strcmp(type, exec_name))
1810 			continue;
1811 
1812 		err = handle_exec_unit(ip, list, dev, mdesc, node,
1813 				       exec_name, q_type, handler, table);
1814 		if (err) {
1815 			spu_list_destroy(list);
1816 			break;
1817 		}
1818 	}
1819 
1820 	return err;
1821 }
1822 
1823 static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,
1824 				   struct spu_mdesc_info *ip)
1825 {
1826 	const u64 *intr, *ino;
1827 	int intr_len, ino_len;
1828 	int i;
1829 
1830 	intr = mdesc_get_property(mdesc, node, "intr", &intr_len);
1831 	if (!intr)
1832 		return -ENODEV;
1833 
1834 	ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1835 	if (!ino)
1836 		return -ENODEV;
1837 
1838 	if (intr_len != ino_len)
1839 		return -EINVAL;
1840 
1841 	ip->num_intrs = intr_len / sizeof(u64);
1842 	ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1843 				 ip->num_intrs),
1844 				GFP_KERNEL);
1845 	if (!ip->ino_table)
1846 		return -ENOMEM;
1847 
1848 	for (i = 0; i < ip->num_intrs; i++) {
1849 		struct ino_blob *b = &ip->ino_table[i];
1850 		b->intr = intr[i];
1851 		b->ino = ino[i];
1852 	}
1853 
1854 	return 0;
1855 }
1856 
1857 static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1858 					  struct platform_device *dev,
1859 					  struct spu_mdesc_info *ip,
1860 					  const char *node_name)
1861 {
1862 	const unsigned int *reg;
1863 	u64 node;
1864 
1865 	reg = of_get_property(dev->dev.of_node, "reg", NULL);
1866 	if (!reg)
1867 		return -ENODEV;
1868 
1869 	mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1870 		const char *name;
1871 		const u64 *chdl;
1872 
1873 		name = mdesc_get_property(mdesc, node, "name", NULL);
1874 		if (!name || strcmp(name, node_name))
1875 			continue;
1876 		chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1877 		if (!chdl || (*chdl != *reg))
1878 			continue;
1879 		ip->cfg_handle = *chdl;
1880 		return get_irq_props(mdesc, node, ip);
1881 	}
1882 
1883 	return -ENODEV;
1884 }
1885 
1886 static unsigned long n2_spu_hvapi_major;
1887 static unsigned long n2_spu_hvapi_minor;
1888 
1889 static int __devinit n2_spu_hvapi_register(void)
1890 {
1891 	int err;
1892 
1893 	n2_spu_hvapi_major = 2;
1894 	n2_spu_hvapi_minor = 0;
1895 
1896 	err = sun4v_hvapi_register(HV_GRP_NCS,
1897 				   n2_spu_hvapi_major,
1898 				   &n2_spu_hvapi_minor);
1899 
1900 	if (!err)
1901 		pr_info("Registered NCS HVAPI version %lu.%lu\n",
1902 			n2_spu_hvapi_major,
1903 			n2_spu_hvapi_minor);
1904 
1905 	return err;
1906 }
1907 
1908 static void n2_spu_hvapi_unregister(void)
1909 {
1910 	sun4v_hvapi_unregister(HV_GRP_NCS);
1911 }
1912 
1913 static int global_ref;
1914 
1915 static int __devinit grab_global_resources(void)
1916 {
1917 	int err = 0;
1918 
1919 	mutex_lock(&spu_lock);
1920 
1921 	if (global_ref++)
1922 		goto out;
1923 
1924 	err = n2_spu_hvapi_register();
1925 	if (err)
1926 		goto out;
1927 
1928 	err = queue_cache_init();
1929 	if (err)
1930 		goto out_hvapi_release;
1931 
1932 	err = -ENOMEM;
1933 	cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1934 			     GFP_KERNEL);
1935 	if (!cpu_to_cwq)
1936 		goto out_queue_cache_destroy;
1937 
1938 	cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1939 			     GFP_KERNEL);
1940 	if (!cpu_to_mau)
1941 		goto out_free_cwq_table;
1942 
1943 	err = 0;
1944 
1945 out:
1946 	if (err)
1947 		global_ref--;
1948 	mutex_unlock(&spu_lock);
1949 	return err;
1950 
1951 out_free_cwq_table:
1952 	kfree(cpu_to_cwq);
1953 	cpu_to_cwq = NULL;
1954 
1955 out_queue_cache_destroy:
1956 	queue_cache_destroy();
1957 
1958 out_hvapi_release:
1959 	n2_spu_hvapi_unregister();
1960 	goto out;
1961 }
1962 
1963 static void release_global_resources(void)
1964 {
1965 	mutex_lock(&spu_lock);
1966 	if (!--global_ref) {
1967 		kfree(cpu_to_cwq);
1968 		cpu_to_cwq = NULL;
1969 
1970 		kfree(cpu_to_mau);
1971 		cpu_to_mau = NULL;
1972 
1973 		queue_cache_destroy();
1974 		n2_spu_hvapi_unregister();
1975 	}
1976 	mutex_unlock(&spu_lock);
1977 }
1978 
1979 static struct n2_crypto * __devinit alloc_n2cp(void)
1980 {
1981 	struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1982 
1983 	if (np)
1984 		INIT_LIST_HEAD(&np->cwq_list);
1985 
1986 	return np;
1987 }
1988 
1989 static void free_n2cp(struct n2_crypto *np)
1990 {
1991 	if (np->cwq_info.ino_table) {
1992 		kfree(np->cwq_info.ino_table);
1993 		np->cwq_info.ino_table = NULL;
1994 	}
1995 
1996 	kfree(np);
1997 }
1998 
1999 static void __devinit n2_spu_driver_version(void)
2000 {
2001 	static int n2_spu_version_printed;
2002 
2003 	if (n2_spu_version_printed++ == 0)
2004 		pr_info("%s", version);
2005 }
2006 
2007 static int __devinit n2_crypto_probe(struct platform_device *dev,
2008 				     const struct of_device_id *match)
2009 {
2010 	struct mdesc_handle *mdesc;
2011 	const char *full_name;
2012 	struct n2_crypto *np;
2013 	int err;
2014 
2015 	n2_spu_driver_version();
2016 
2017 	full_name = dev->dev.of_node->full_name;
2018 	pr_info("Found N2CP at %s\n", full_name);
2019 
2020 	np = alloc_n2cp();
2021 	if (!np) {
2022 		dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
2023 			full_name);
2024 		return -ENOMEM;
2025 	}
2026 
2027 	err = grab_global_resources();
2028 	if (err) {
2029 		dev_err(&dev->dev, "%s: Unable to grab "
2030 			"global resources.\n", full_name);
2031 		goto out_free_n2cp;
2032 	}
2033 
2034 	mdesc = mdesc_grab();
2035 
2036 	if (!mdesc) {
2037 		dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2038 			full_name);
2039 		err = -ENODEV;
2040 		goto out_free_global;
2041 	}
2042 	err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
2043 	if (err) {
2044 		dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2045 			full_name);
2046 		mdesc_release(mdesc);
2047 		goto out_free_global;
2048 	}
2049 
2050 	err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
2051 			     "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
2052 			     cpu_to_cwq);
2053 	mdesc_release(mdesc);
2054 
2055 	if (err) {
2056 		dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
2057 			full_name);
2058 		goto out_free_global;
2059 	}
2060 
2061 	err = n2_register_algs();
2062 	if (err) {
2063 		dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
2064 			full_name);
2065 		goto out_free_spu_list;
2066 	}
2067 
2068 	dev_set_drvdata(&dev->dev, np);
2069 
2070 	return 0;
2071 
2072 out_free_spu_list:
2073 	spu_list_destroy(&np->cwq_list);
2074 
2075 out_free_global:
2076 	release_global_resources();
2077 
2078 out_free_n2cp:
2079 	free_n2cp(np);
2080 
2081 	return err;
2082 }
2083 
2084 static int __devexit n2_crypto_remove(struct platform_device *dev)
2085 {
2086 	struct n2_crypto *np = dev_get_drvdata(&dev->dev);
2087 
2088 	n2_unregister_algs();
2089 
2090 	spu_list_destroy(&np->cwq_list);
2091 
2092 	release_global_resources();
2093 
2094 	free_n2cp(np);
2095 
2096 	return 0;
2097 }
2098 
2099 static struct n2_mau * __devinit alloc_ncp(void)
2100 {
2101 	struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
2102 
2103 	if (mp)
2104 		INIT_LIST_HEAD(&mp->mau_list);
2105 
2106 	return mp;
2107 }
2108 
2109 static void free_ncp(struct n2_mau *mp)
2110 {
2111 	if (mp->mau_info.ino_table) {
2112 		kfree(mp->mau_info.ino_table);
2113 		mp->mau_info.ino_table = NULL;
2114 	}
2115 
2116 	kfree(mp);
2117 }
2118 
2119 static int __devinit n2_mau_probe(struct platform_device *dev,
2120 				     const struct of_device_id *match)
2121 {
2122 	struct mdesc_handle *mdesc;
2123 	const char *full_name;
2124 	struct n2_mau *mp;
2125 	int err;
2126 
2127 	n2_spu_driver_version();
2128 
2129 	full_name = dev->dev.of_node->full_name;
2130 	pr_info("Found NCP at %s\n", full_name);
2131 
2132 	mp = alloc_ncp();
2133 	if (!mp) {
2134 		dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
2135 			full_name);
2136 		return -ENOMEM;
2137 	}
2138 
2139 	err = grab_global_resources();
2140 	if (err) {
2141 		dev_err(&dev->dev, "%s: Unable to grab "
2142 			"global resources.\n", full_name);
2143 		goto out_free_ncp;
2144 	}
2145 
2146 	mdesc = mdesc_grab();
2147 
2148 	if (!mdesc) {
2149 		dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2150 			full_name);
2151 		err = -ENODEV;
2152 		goto out_free_global;
2153 	}
2154 
2155 	err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
2156 	if (err) {
2157 		dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2158 			full_name);
2159 		mdesc_release(mdesc);
2160 		goto out_free_global;
2161 	}
2162 
2163 	err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
2164 			     "mau", HV_NCS_QTYPE_MAU, mau_intr,
2165 			     cpu_to_mau);
2166 	mdesc_release(mdesc);
2167 
2168 	if (err) {
2169 		dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
2170 			full_name);
2171 		goto out_free_global;
2172 	}
2173 
2174 	dev_set_drvdata(&dev->dev, mp);
2175 
2176 	return 0;
2177 
2178 out_free_global:
2179 	release_global_resources();
2180 
2181 out_free_ncp:
2182 	free_ncp(mp);
2183 
2184 	return err;
2185 }
2186 
2187 static int __devexit n2_mau_remove(struct platform_device *dev)
2188 {
2189 	struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2190 
2191 	spu_list_destroy(&mp->mau_list);
2192 
2193 	release_global_resources();
2194 
2195 	free_ncp(mp);
2196 
2197 	return 0;
2198 }
2199 
2200 static struct of_device_id n2_crypto_match[] = {
2201 	{
2202 		.name = "n2cp",
2203 		.compatible = "SUNW,n2-cwq",
2204 	},
2205 	{
2206 		.name = "n2cp",
2207 		.compatible = "SUNW,vf-cwq",
2208 	},
2209 	{},
2210 };
2211 
2212 MODULE_DEVICE_TABLE(of, n2_crypto_match);
2213 
2214 static struct of_platform_driver n2_crypto_driver = {
2215 	.driver = {
2216 		.name		=	"n2cp",
2217 		.owner		=	THIS_MODULE,
2218 		.of_match_table	=	n2_crypto_match,
2219 	},
2220 	.probe		=	n2_crypto_probe,
2221 	.remove		=	__devexit_p(n2_crypto_remove),
2222 };
2223 
2224 static struct of_device_id n2_mau_match[] = {
2225 	{
2226 		.name = "ncp",
2227 		.compatible = "SUNW,n2-mau",
2228 	},
2229 	{
2230 		.name = "ncp",
2231 		.compatible = "SUNW,vf-mau",
2232 	},
2233 	{},
2234 };
2235 
2236 MODULE_DEVICE_TABLE(of, n2_mau_match);
2237 
2238 static struct of_platform_driver n2_mau_driver = {
2239 	.driver = {
2240 		.name		=	"ncp",
2241 		.owner		=	THIS_MODULE,
2242 		.of_match_table	=	n2_mau_match,
2243 	},
2244 	.probe		=	n2_mau_probe,
2245 	.remove		=	__devexit_p(n2_mau_remove),
2246 };
2247 
2248 static int __init n2_init(void)
2249 {
2250 	int err = of_register_platform_driver(&n2_crypto_driver);
2251 
2252 	if (!err) {
2253 		err = of_register_platform_driver(&n2_mau_driver);
2254 		if (err)
2255 			of_unregister_platform_driver(&n2_crypto_driver);
2256 	}
2257 	return err;
2258 }
2259 
2260 static void __exit n2_exit(void)
2261 {
2262 	of_unregister_platform_driver(&n2_mau_driver);
2263 	of_unregister_platform_driver(&n2_crypto_driver);
2264 }
2265 
2266 module_init(n2_init);
2267 module_exit(n2_exit);
2268