10a625fd2SDavid S. Miller /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. 20a625fd2SDavid S. Miller * 30a625fd2SDavid S. Miller * Copyright (C) 2010 David S. Miller <davem@davemloft.net> 40a625fd2SDavid S. Miller */ 50a625fd2SDavid S. Miller 60a625fd2SDavid S. Miller #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 70a625fd2SDavid S. Miller 80a625fd2SDavid S. Miller #include <linux/kernel.h> 90a625fd2SDavid S. Miller #include <linux/module.h> 100a625fd2SDavid S. Miller #include <linux/of.h> 110a625fd2SDavid S. Miller #include <linux/of_device.h> 120a625fd2SDavid S. Miller #include <linux/cpumask.h> 130a625fd2SDavid S. Miller #include <linux/slab.h> 140a625fd2SDavid S. Miller #include <linux/interrupt.h> 150a625fd2SDavid S. Miller #include <linux/crypto.h> 160a625fd2SDavid S. Miller #include <crypto/md5.h> 170a625fd2SDavid S. Miller #include <crypto/sha.h> 180a625fd2SDavid S. Miller #include <crypto/aes.h> 190a625fd2SDavid S. Miller #include <crypto/des.h> 200a625fd2SDavid S. Miller #include <linux/mutex.h> 210a625fd2SDavid S. Miller #include <linux/delay.h> 220a625fd2SDavid S. Miller #include <linux/sched.h> 230a625fd2SDavid S. Miller 240a625fd2SDavid S. Miller #include <crypto/internal/hash.h> 250a625fd2SDavid S. Miller #include <crypto/scatterwalk.h> 260a625fd2SDavid S. Miller #include <crypto/algapi.h> 270a625fd2SDavid S. Miller 280a625fd2SDavid S. Miller #include <asm/hypervisor.h> 290a625fd2SDavid S. Miller #include <asm/mdesc.h> 300a625fd2SDavid S. Miller 310a625fd2SDavid S. Miller #include "n2_core.h" 320a625fd2SDavid S. Miller 330a625fd2SDavid S. Miller #define DRV_MODULE_NAME "n2_crypto" 340a625fd2SDavid S. Miller #define DRV_MODULE_VERSION "0.1" 350a625fd2SDavid S. Miller #define DRV_MODULE_RELDATE "April 29, 2010" 360a625fd2SDavid S. Miller 370a625fd2SDavid S. Miller static char version[] __devinitdata = 380a625fd2SDavid S. Miller DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 390a625fd2SDavid S. Miller 400a625fd2SDavid S. Miller MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 410a625fd2SDavid S. Miller MODULE_DESCRIPTION("Niagara2 Crypto driver"); 420a625fd2SDavid S. Miller MODULE_LICENSE("GPL"); 430a625fd2SDavid S. Miller MODULE_VERSION(DRV_MODULE_VERSION); 440a625fd2SDavid S. Miller 450a625fd2SDavid S. Miller #define N2_CRA_PRIORITY 300 460a625fd2SDavid S. Miller 470a625fd2SDavid S. Miller static DEFINE_MUTEX(spu_lock); 480a625fd2SDavid S. Miller 490a625fd2SDavid S. Miller struct spu_queue { 500a625fd2SDavid S. Miller cpumask_t sharing; 510a625fd2SDavid S. Miller unsigned long qhandle; 520a625fd2SDavid S. Miller 530a625fd2SDavid S. Miller spinlock_t lock; 540a625fd2SDavid S. Miller u8 q_type; 550a625fd2SDavid S. Miller void *q; 560a625fd2SDavid S. Miller unsigned long head; 570a625fd2SDavid S. Miller unsigned long tail; 580a625fd2SDavid S. Miller struct list_head jobs; 590a625fd2SDavid S. Miller 600a625fd2SDavid S. Miller unsigned long devino; 610a625fd2SDavid S. Miller 620a625fd2SDavid S. Miller char irq_name[32]; 630a625fd2SDavid S. Miller unsigned int irq; 640a625fd2SDavid S. Miller 650a625fd2SDavid S. Miller struct list_head list; 660a625fd2SDavid S. Miller }; 670a625fd2SDavid S. Miller 680a625fd2SDavid S. Miller static struct spu_queue **cpu_to_cwq; 690a625fd2SDavid S. Miller static struct spu_queue **cpu_to_mau; 700a625fd2SDavid S. Miller 710a625fd2SDavid S. Miller static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) 720a625fd2SDavid S. Miller { 730a625fd2SDavid S. Miller if (q->q_type == HV_NCS_QTYPE_MAU) { 740a625fd2SDavid S. Miller off += MAU_ENTRY_SIZE; 750a625fd2SDavid S. Miller if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) 760a625fd2SDavid S. Miller off = 0; 770a625fd2SDavid S. Miller } else { 780a625fd2SDavid S. Miller off += CWQ_ENTRY_SIZE; 790a625fd2SDavid S. Miller if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) 800a625fd2SDavid S. Miller off = 0; 810a625fd2SDavid S. Miller } 820a625fd2SDavid S. Miller return off; 830a625fd2SDavid S. Miller } 840a625fd2SDavid S. Miller 850a625fd2SDavid S. Miller struct n2_request_common { 860a625fd2SDavid S. Miller struct list_head entry; 870a625fd2SDavid S. Miller unsigned int offset; 880a625fd2SDavid S. Miller }; 890a625fd2SDavid S. Miller #define OFFSET_NOT_RUNNING (~(unsigned int)0) 900a625fd2SDavid S. Miller 910a625fd2SDavid S. Miller /* An async job request records the final tail value it used in 920a625fd2SDavid S. Miller * n2_request_common->offset, test to see if that offset is in 930a625fd2SDavid S. Miller * the range old_head, new_head, inclusive. 940a625fd2SDavid S. Miller */ 950a625fd2SDavid S. Miller static inline bool job_finished(struct spu_queue *q, unsigned int offset, 960a625fd2SDavid S. Miller unsigned long old_head, unsigned long new_head) 970a625fd2SDavid S. Miller { 980a625fd2SDavid S. Miller if (old_head <= new_head) { 990a625fd2SDavid S. Miller if (offset > old_head && offset <= new_head) 1000a625fd2SDavid S. Miller return true; 1010a625fd2SDavid S. Miller } else { 1020a625fd2SDavid S. Miller if (offset > old_head || offset <= new_head) 1030a625fd2SDavid S. Miller return true; 1040a625fd2SDavid S. Miller } 1050a625fd2SDavid S. Miller return false; 1060a625fd2SDavid S. Miller } 1070a625fd2SDavid S. Miller 1080a625fd2SDavid S. Miller /* When the HEAD marker is unequal to the actual HEAD, we get 1090a625fd2SDavid S. Miller * a virtual device INO interrupt. We should process the 1100a625fd2SDavid S. Miller * completed CWQ entries and adjust the HEAD marker to clear 1110a625fd2SDavid S. Miller * the IRQ. 1120a625fd2SDavid S. Miller */ 1130a625fd2SDavid S. Miller static irqreturn_t cwq_intr(int irq, void *dev_id) 1140a625fd2SDavid S. Miller { 1150a625fd2SDavid S. Miller unsigned long off, new_head, hv_ret; 1160a625fd2SDavid S. Miller struct spu_queue *q = dev_id; 1170a625fd2SDavid S. Miller 1180a625fd2SDavid S. Miller pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n", 1190a625fd2SDavid S. Miller smp_processor_id(), q->qhandle); 1200a625fd2SDavid S. Miller 1210a625fd2SDavid S. Miller spin_lock(&q->lock); 1220a625fd2SDavid S. Miller 1230a625fd2SDavid S. Miller hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head); 1240a625fd2SDavid S. Miller 1250a625fd2SDavid S. Miller pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n", 1260a625fd2SDavid S. Miller smp_processor_id(), new_head, hv_ret); 1270a625fd2SDavid S. Miller 1280a625fd2SDavid S. Miller for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { 1290a625fd2SDavid S. Miller /* XXX ... XXX */ 1300a625fd2SDavid S. Miller } 1310a625fd2SDavid S. Miller 1320a625fd2SDavid S. Miller hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head); 1330a625fd2SDavid S. Miller if (hv_ret == HV_EOK) 1340a625fd2SDavid S. Miller q->head = new_head; 1350a625fd2SDavid S. Miller 1360a625fd2SDavid S. Miller spin_unlock(&q->lock); 1370a625fd2SDavid S. Miller 1380a625fd2SDavid S. Miller return IRQ_HANDLED; 1390a625fd2SDavid S. Miller } 1400a625fd2SDavid S. Miller 1410a625fd2SDavid S. Miller static irqreturn_t mau_intr(int irq, void *dev_id) 1420a625fd2SDavid S. Miller { 1430a625fd2SDavid S. Miller struct spu_queue *q = dev_id; 1440a625fd2SDavid S. Miller unsigned long head, hv_ret; 1450a625fd2SDavid S. Miller 1460a625fd2SDavid S. Miller spin_lock(&q->lock); 1470a625fd2SDavid S. Miller 1480a625fd2SDavid S. Miller pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n", 1490a625fd2SDavid S. Miller smp_processor_id(), q->qhandle); 1500a625fd2SDavid S. Miller 1510a625fd2SDavid S. Miller hv_ret = sun4v_ncs_gethead(q->qhandle, &head); 1520a625fd2SDavid S. Miller 1530a625fd2SDavid S. Miller pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n", 1540a625fd2SDavid S. Miller smp_processor_id(), head, hv_ret); 1550a625fd2SDavid S. Miller 1560a625fd2SDavid S. Miller sun4v_ncs_sethead_marker(q->qhandle, head); 1570a625fd2SDavid S. Miller 1580a625fd2SDavid S. Miller spin_unlock(&q->lock); 1590a625fd2SDavid S. Miller 1600a625fd2SDavid S. Miller return IRQ_HANDLED; 1610a625fd2SDavid S. Miller } 1620a625fd2SDavid S. Miller 1630a625fd2SDavid S. Miller static void *spu_queue_next(struct spu_queue *q, void *cur) 1640a625fd2SDavid S. Miller { 1650a625fd2SDavid S. Miller return q->q + spu_next_offset(q, cur - q->q); 1660a625fd2SDavid S. Miller } 1670a625fd2SDavid S. Miller 1680a625fd2SDavid S. Miller static int spu_queue_num_free(struct spu_queue *q) 1690a625fd2SDavid S. Miller { 1700a625fd2SDavid S. Miller unsigned long head = q->head; 1710a625fd2SDavid S. Miller unsigned long tail = q->tail; 1720a625fd2SDavid S. Miller unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); 1730a625fd2SDavid S. Miller unsigned long diff; 1740a625fd2SDavid S. Miller 1750a625fd2SDavid S. Miller if (head > tail) 1760a625fd2SDavid S. Miller diff = head - tail; 1770a625fd2SDavid S. Miller else 1780a625fd2SDavid S. Miller diff = (end - tail) + head; 1790a625fd2SDavid S. Miller 1800a625fd2SDavid S. Miller return (diff / CWQ_ENTRY_SIZE) - 1; 1810a625fd2SDavid S. Miller } 1820a625fd2SDavid S. Miller 1830a625fd2SDavid S. Miller static void *spu_queue_alloc(struct spu_queue *q, int num_entries) 1840a625fd2SDavid S. Miller { 1850a625fd2SDavid S. Miller int avail = spu_queue_num_free(q); 1860a625fd2SDavid S. Miller 1870a625fd2SDavid S. Miller if (avail >= num_entries) 1880a625fd2SDavid S. Miller return q->q + q->tail; 1890a625fd2SDavid S. Miller 1900a625fd2SDavid S. Miller return NULL; 1910a625fd2SDavid S. Miller } 1920a625fd2SDavid S. Miller 1930a625fd2SDavid S. Miller static unsigned long spu_queue_submit(struct spu_queue *q, void *last) 1940a625fd2SDavid S. Miller { 1950a625fd2SDavid S. Miller unsigned long hv_ret, new_tail; 1960a625fd2SDavid S. Miller 1970a625fd2SDavid S. Miller new_tail = spu_next_offset(q, last - q->q); 1980a625fd2SDavid S. Miller 1990a625fd2SDavid S. Miller hv_ret = sun4v_ncs_settail(q->qhandle, new_tail); 2000a625fd2SDavid S. Miller if (hv_ret == HV_EOK) 2010a625fd2SDavid S. Miller q->tail = new_tail; 2020a625fd2SDavid S. Miller return hv_ret; 2030a625fd2SDavid S. Miller } 2040a625fd2SDavid S. Miller 2050a625fd2SDavid S. Miller static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, 2060a625fd2SDavid S. Miller int enc_type, int auth_type, 2070a625fd2SDavid S. Miller unsigned int hash_len, 2080a625fd2SDavid S. Miller bool sfas, bool sob, bool eob, bool encrypt, 2090a625fd2SDavid S. Miller int opcode) 2100a625fd2SDavid S. Miller { 2110a625fd2SDavid S. Miller u64 word = (len - 1) & CONTROL_LEN; 2120a625fd2SDavid S. Miller 2130a625fd2SDavid S. Miller word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); 2140a625fd2SDavid S. Miller word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); 2150a625fd2SDavid S. Miller word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); 2160a625fd2SDavid S. Miller if (sfas) 2170a625fd2SDavid S. Miller word |= CONTROL_STORE_FINAL_AUTH_STATE; 2180a625fd2SDavid S. Miller if (sob) 2190a625fd2SDavid S. Miller word |= CONTROL_START_OF_BLOCK; 2200a625fd2SDavid S. Miller if (eob) 2210a625fd2SDavid S. Miller word |= CONTROL_END_OF_BLOCK; 2220a625fd2SDavid S. Miller if (encrypt) 2230a625fd2SDavid S. Miller word |= CONTROL_ENCRYPT; 2240a625fd2SDavid S. Miller if (hmac_key_len) 2250a625fd2SDavid S. Miller word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; 2260a625fd2SDavid S. Miller if (hash_len) 2270a625fd2SDavid S. Miller word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; 2280a625fd2SDavid S. Miller 2290a625fd2SDavid S. Miller return word; 2300a625fd2SDavid S. Miller } 2310a625fd2SDavid S. Miller 2320a625fd2SDavid S. Miller #if 0 2330a625fd2SDavid S. Miller static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) 2340a625fd2SDavid S. Miller { 2350a625fd2SDavid S. Miller if (this_len >= 64 || 2360a625fd2SDavid S. Miller qp->head != qp->tail) 2370a625fd2SDavid S. Miller return true; 2380a625fd2SDavid S. Miller return false; 2390a625fd2SDavid S. Miller } 2400a625fd2SDavid S. Miller #endif 2410a625fd2SDavid S. Miller 2420a625fd2SDavid S. Miller struct n2_base_ctx { 2430a625fd2SDavid S. Miller struct list_head list; 2440a625fd2SDavid S. Miller }; 2450a625fd2SDavid S. Miller 2460a625fd2SDavid S. Miller static void n2_base_ctx_init(struct n2_base_ctx *ctx) 2470a625fd2SDavid S. Miller { 2480a625fd2SDavid S. Miller INIT_LIST_HEAD(&ctx->list); 2490a625fd2SDavid S. Miller } 2500a625fd2SDavid S. Miller 2510a625fd2SDavid S. Miller struct n2_hash_ctx { 2520a625fd2SDavid S. Miller struct n2_base_ctx base; 2530a625fd2SDavid S. Miller 254c9aa55e5SDavid S. Miller struct crypto_ahash *fallback_tfm; 255c9aa55e5SDavid S. Miller }; 2560a625fd2SDavid S. Miller 257c9aa55e5SDavid S. Miller struct n2_hash_req_ctx { 2580a625fd2SDavid S. Miller union { 2590a625fd2SDavid S. Miller struct md5_state md5; 2600a625fd2SDavid S. Miller struct sha1_state sha1; 2610a625fd2SDavid S. Miller struct sha256_state sha256; 2620a625fd2SDavid S. Miller } u; 2630a625fd2SDavid S. Miller 2640a625fd2SDavid S. Miller unsigned char hash_key[64]; 2650a625fd2SDavid S. Miller unsigned char keyed_zero_hash[32]; 266c9aa55e5SDavid S. Miller 267c9aa55e5SDavid S. Miller struct ahash_request fallback_req; 2680a625fd2SDavid S. Miller }; 2690a625fd2SDavid S. Miller 2700a625fd2SDavid S. Miller static int n2_hash_async_init(struct ahash_request *req) 2710a625fd2SDavid S. Miller { 272c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 2730a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2740a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 2750a625fd2SDavid S. Miller 276c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 277c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 2780a625fd2SDavid S. Miller 279c9aa55e5SDavid S. Miller return crypto_ahash_init(&rctx->fallback_req); 2800a625fd2SDavid S. Miller } 2810a625fd2SDavid S. Miller 2820a625fd2SDavid S. Miller static int n2_hash_async_update(struct ahash_request *req) 2830a625fd2SDavid S. Miller { 284c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 2850a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2860a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 2870a625fd2SDavid S. Miller 288c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 289c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 290c9aa55e5SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes; 291c9aa55e5SDavid S. Miller rctx->fallback_req.src = req->src; 2920a625fd2SDavid S. Miller 293c9aa55e5SDavid S. Miller return crypto_ahash_update(&rctx->fallback_req); 2940a625fd2SDavid S. Miller } 2950a625fd2SDavid S. Miller 2960a625fd2SDavid S. Miller static int n2_hash_async_final(struct ahash_request *req) 2970a625fd2SDavid S. Miller { 298c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 2990a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 3000a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 3010a625fd2SDavid S. Miller 302c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 303c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 304c9aa55e5SDavid S. Miller rctx->fallback_req.result = req->result; 3050a625fd2SDavid S. Miller 306c9aa55e5SDavid S. Miller return crypto_ahash_final(&rctx->fallback_req); 3070a625fd2SDavid S. Miller } 3080a625fd2SDavid S. Miller 3090a625fd2SDavid S. Miller static int n2_hash_async_finup(struct ahash_request *req) 3100a625fd2SDavid S. Miller { 311c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 3120a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 3130a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 3140a625fd2SDavid S. Miller 315c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 316c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 317c9aa55e5SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes; 318c9aa55e5SDavid S. Miller rctx->fallback_req.src = req->src; 319c9aa55e5SDavid S. Miller rctx->fallback_req.result = req->result; 3200a625fd2SDavid S. Miller 321c9aa55e5SDavid S. Miller return crypto_ahash_finup(&rctx->fallback_req); 3220a625fd2SDavid S. Miller } 3230a625fd2SDavid S. Miller 3240a625fd2SDavid S. Miller static int n2_hash_cra_init(struct crypto_tfm *tfm) 3250a625fd2SDavid S. Miller { 3260a625fd2SDavid S. Miller const char *fallback_driver_name = tfm->__crt_alg->cra_name; 3270a625fd2SDavid S. Miller struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 3280a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3290a625fd2SDavid S. Miller struct crypto_ahash *fallback_tfm; 3300a625fd2SDavid S. Miller int err; 3310a625fd2SDavid S. Miller 3320a625fd2SDavid S. Miller fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 3330a625fd2SDavid S. Miller CRYPTO_ALG_NEED_FALLBACK); 3340a625fd2SDavid S. Miller if (IS_ERR(fallback_tfm)) { 3350a625fd2SDavid S. Miller pr_warning("Fallback driver '%s' could not be loaded!\n", 3360a625fd2SDavid S. Miller fallback_driver_name); 3370a625fd2SDavid S. Miller err = PTR_ERR(fallback_tfm); 3380a625fd2SDavid S. Miller goto out; 3390a625fd2SDavid S. Miller } 3400a625fd2SDavid S. Miller 341c9aa55e5SDavid S. Miller crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 342c9aa55e5SDavid S. Miller crypto_ahash_reqsize(fallback_tfm))); 343c9aa55e5SDavid S. Miller 344c9aa55e5SDavid S. Miller ctx->fallback_tfm = fallback_tfm; 3450a625fd2SDavid S. Miller return 0; 3460a625fd2SDavid S. Miller 3470a625fd2SDavid S. Miller out: 3480a625fd2SDavid S. Miller return err; 3490a625fd2SDavid S. Miller } 3500a625fd2SDavid S. Miller 3510a625fd2SDavid S. Miller static void n2_hash_cra_exit(struct crypto_tfm *tfm) 3520a625fd2SDavid S. Miller { 3530a625fd2SDavid S. Miller struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 3540a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3550a625fd2SDavid S. Miller 356c9aa55e5SDavid S. Miller crypto_free_ahash(ctx->fallback_tfm); 3570a625fd2SDavid S. Miller } 3580a625fd2SDavid S. Miller 3590a625fd2SDavid S. Miller static unsigned long wait_for_tail(struct spu_queue *qp) 3600a625fd2SDavid S. Miller { 3610a625fd2SDavid S. Miller unsigned long head, hv_ret; 3620a625fd2SDavid S. Miller 3630a625fd2SDavid S. Miller do { 3640a625fd2SDavid S. Miller hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); 3650a625fd2SDavid S. Miller if (hv_ret != HV_EOK) { 3660a625fd2SDavid S. Miller pr_err("Hypervisor error on gethead\n"); 3670a625fd2SDavid S. Miller break; 3680a625fd2SDavid S. Miller } 3690a625fd2SDavid S. Miller if (head == qp->tail) { 3700a625fd2SDavid S. Miller qp->head = head; 3710a625fd2SDavid S. Miller break; 3720a625fd2SDavid S. Miller } 3730a625fd2SDavid S. Miller } while (1); 3740a625fd2SDavid S. Miller return hv_ret; 3750a625fd2SDavid S. Miller } 3760a625fd2SDavid S. Miller 3770a625fd2SDavid S. Miller static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, 3780a625fd2SDavid S. Miller struct cwq_initial_entry *ent) 3790a625fd2SDavid S. Miller { 3800a625fd2SDavid S. Miller unsigned long hv_ret = spu_queue_submit(qp, ent); 3810a625fd2SDavid S. Miller 3820a625fd2SDavid S. Miller if (hv_ret == HV_EOK) 3830a625fd2SDavid S. Miller hv_ret = wait_for_tail(qp); 3840a625fd2SDavid S. Miller 3850a625fd2SDavid S. Miller return hv_ret; 3860a625fd2SDavid S. Miller } 3870a625fd2SDavid S. Miller 3880a625fd2SDavid S. Miller static int n2_hash_async_digest(struct ahash_request *req, 3890a625fd2SDavid S. Miller unsigned int auth_type, unsigned int digest_size, 3900a625fd2SDavid S. Miller unsigned int result_size, void *hash_loc) 3910a625fd2SDavid S. Miller { 3920a625fd2SDavid S. Miller struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 3930a625fd2SDavid S. Miller struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 3940a625fd2SDavid S. Miller struct cwq_initial_entry *ent; 3950a625fd2SDavid S. Miller struct crypto_hash_walk walk; 3960a625fd2SDavid S. Miller struct spu_queue *qp; 3970a625fd2SDavid S. Miller unsigned long flags; 3980a625fd2SDavid S. Miller int err = -ENODEV; 3990a625fd2SDavid S. Miller int nbytes, cpu; 4000a625fd2SDavid S. Miller 4010a625fd2SDavid S. Miller /* The total effective length of the operation may not 4020a625fd2SDavid S. Miller * exceed 2^16. 4030a625fd2SDavid S. Miller */ 4040a625fd2SDavid S. Miller if (unlikely(req->nbytes > (1 << 16))) { 405c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 4060a625fd2SDavid S. Miller 407c9aa55e5SDavid S. Miller ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 408c9aa55e5SDavid S. Miller rctx->fallback_req.base.flags = 409c9aa55e5SDavid S. Miller req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 410c9aa55e5SDavid S. Miller rctx->fallback_req.nbytes = req->nbytes; 411c9aa55e5SDavid S. Miller rctx->fallback_req.src = req->src; 412c9aa55e5SDavid S. Miller rctx->fallback_req.result = req->result; 413c9aa55e5SDavid S. Miller 414c9aa55e5SDavid S. Miller return crypto_ahash_digest(&rctx->fallback_req); 4150a625fd2SDavid S. Miller } 4160a625fd2SDavid S. Miller 4170a625fd2SDavid S. Miller n2_base_ctx_init(&ctx->base); 4180a625fd2SDavid S. Miller 4190a625fd2SDavid S. Miller nbytes = crypto_hash_walk_first(req, &walk); 4200a625fd2SDavid S. Miller 4210a625fd2SDavid S. Miller cpu = get_cpu(); 4220a625fd2SDavid S. Miller qp = cpu_to_cwq[cpu]; 4230a625fd2SDavid S. Miller if (!qp) 4240a625fd2SDavid S. Miller goto out; 4250a625fd2SDavid S. Miller 4260a625fd2SDavid S. Miller spin_lock_irqsave(&qp->lock, flags); 4270a625fd2SDavid S. Miller 4280a625fd2SDavid S. Miller /* XXX can do better, improve this later by doing a by-hand scatterlist 4290a625fd2SDavid S. Miller * XXX walk, etc. 4300a625fd2SDavid S. Miller */ 4310a625fd2SDavid S. Miller ent = qp->q + qp->tail; 4320a625fd2SDavid S. Miller 4330a625fd2SDavid S. Miller ent->control = control_word_base(nbytes, 0, 0, 4340a625fd2SDavid S. Miller auth_type, digest_size, 4350a625fd2SDavid S. Miller false, true, false, false, 4360a625fd2SDavid S. Miller OPCODE_INPLACE_BIT | 4370a625fd2SDavid S. Miller OPCODE_AUTH_MAC); 4380a625fd2SDavid S. Miller ent->src_addr = __pa(walk.data); 4390a625fd2SDavid S. Miller ent->auth_key_addr = 0UL; 4400a625fd2SDavid S. Miller ent->auth_iv_addr = __pa(hash_loc); 4410a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL; 4420a625fd2SDavid S. Miller ent->enc_key_addr = 0UL; 4430a625fd2SDavid S. Miller ent->enc_iv_addr = 0UL; 4440a625fd2SDavid S. Miller ent->dest_addr = __pa(hash_loc); 4450a625fd2SDavid S. Miller 4460a625fd2SDavid S. Miller nbytes = crypto_hash_walk_done(&walk, 0); 4470a625fd2SDavid S. Miller while (nbytes > 0) { 4480a625fd2SDavid S. Miller ent = spu_queue_next(qp, ent); 4490a625fd2SDavid S. Miller 4500a625fd2SDavid S. Miller ent->control = (nbytes - 1); 4510a625fd2SDavid S. Miller ent->src_addr = __pa(walk.data); 4520a625fd2SDavid S. Miller ent->auth_key_addr = 0UL; 4530a625fd2SDavid S. Miller ent->auth_iv_addr = 0UL; 4540a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL; 4550a625fd2SDavid S. Miller ent->enc_key_addr = 0UL; 4560a625fd2SDavid S. Miller ent->enc_iv_addr = 0UL; 4570a625fd2SDavid S. Miller ent->dest_addr = 0UL; 4580a625fd2SDavid S. Miller 4590a625fd2SDavid S. Miller nbytes = crypto_hash_walk_done(&walk, 0); 4600a625fd2SDavid S. Miller } 4610a625fd2SDavid S. Miller ent->control |= CONTROL_END_OF_BLOCK; 4620a625fd2SDavid S. Miller 4630a625fd2SDavid S. Miller if (submit_and_wait_for_tail(qp, ent) != HV_EOK) 4640a625fd2SDavid S. Miller err = -EINVAL; 4650a625fd2SDavid S. Miller else 4660a625fd2SDavid S. Miller err = 0; 4670a625fd2SDavid S. Miller 4680a625fd2SDavid S. Miller spin_unlock_irqrestore(&qp->lock, flags); 4690a625fd2SDavid S. Miller 4700a625fd2SDavid S. Miller if (!err) 4710a625fd2SDavid S. Miller memcpy(req->result, hash_loc, result_size); 4720a625fd2SDavid S. Miller out: 4730a625fd2SDavid S. Miller put_cpu(); 4740a625fd2SDavid S. Miller 4750a625fd2SDavid S. Miller return err; 4760a625fd2SDavid S. Miller } 4770a625fd2SDavid S. Miller 4780a625fd2SDavid S. Miller static int n2_md5_async_digest(struct ahash_request *req) 4790a625fd2SDavid S. Miller { 480c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 481c9aa55e5SDavid S. Miller struct md5_state *m = &rctx->u.md5; 4820a625fd2SDavid S. Miller 4830a625fd2SDavid S. Miller if (unlikely(req->nbytes == 0)) { 4840a625fd2SDavid S. Miller static const char md5_zero[MD5_DIGEST_SIZE] = { 4850a625fd2SDavid S. Miller 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 4860a625fd2SDavid S. Miller 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, 4870a625fd2SDavid S. Miller }; 4880a625fd2SDavid S. Miller 4890a625fd2SDavid S. Miller memcpy(req->result, md5_zero, MD5_DIGEST_SIZE); 4900a625fd2SDavid S. Miller return 0; 4910a625fd2SDavid S. Miller } 4920a625fd2SDavid S. Miller m->hash[0] = cpu_to_le32(0x67452301); 4930a625fd2SDavid S. Miller m->hash[1] = cpu_to_le32(0xefcdab89); 4940a625fd2SDavid S. Miller m->hash[2] = cpu_to_le32(0x98badcfe); 4950a625fd2SDavid S. Miller m->hash[3] = cpu_to_le32(0x10325476); 4960a625fd2SDavid S. Miller 4970a625fd2SDavid S. Miller return n2_hash_async_digest(req, AUTH_TYPE_MD5, 4980a625fd2SDavid S. Miller MD5_DIGEST_SIZE, MD5_DIGEST_SIZE, 4990a625fd2SDavid S. Miller m->hash); 5000a625fd2SDavid S. Miller } 5010a625fd2SDavid S. Miller 5020a625fd2SDavid S. Miller static int n2_sha1_async_digest(struct ahash_request *req) 5030a625fd2SDavid S. Miller { 504c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 505c9aa55e5SDavid S. Miller struct sha1_state *s = &rctx->u.sha1; 5060a625fd2SDavid S. Miller 5070a625fd2SDavid S. Miller if (unlikely(req->nbytes == 0)) { 5080a625fd2SDavid S. Miller static const char sha1_zero[SHA1_DIGEST_SIZE] = { 5090a625fd2SDavid S. Miller 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 5100a625fd2SDavid S. Miller 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 5110a625fd2SDavid S. Miller 0x07, 0x09 5120a625fd2SDavid S. Miller }; 5130a625fd2SDavid S. Miller 5140a625fd2SDavid S. Miller memcpy(req->result, sha1_zero, SHA1_DIGEST_SIZE); 5150a625fd2SDavid S. Miller return 0; 5160a625fd2SDavid S. Miller } 5170a625fd2SDavid S. Miller s->state[0] = SHA1_H0; 5180a625fd2SDavid S. Miller s->state[1] = SHA1_H1; 5190a625fd2SDavid S. Miller s->state[2] = SHA1_H2; 5200a625fd2SDavid S. Miller s->state[3] = SHA1_H3; 5210a625fd2SDavid S. Miller s->state[4] = SHA1_H4; 5220a625fd2SDavid S. Miller 5230a625fd2SDavid S. Miller return n2_hash_async_digest(req, AUTH_TYPE_SHA1, 5240a625fd2SDavid S. Miller SHA1_DIGEST_SIZE, SHA1_DIGEST_SIZE, 5250a625fd2SDavid S. Miller s->state); 5260a625fd2SDavid S. Miller } 5270a625fd2SDavid S. Miller 5280a625fd2SDavid S. Miller static int n2_sha256_async_digest(struct ahash_request *req) 5290a625fd2SDavid S. Miller { 530c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 531c9aa55e5SDavid S. Miller struct sha256_state *s = &rctx->u.sha256; 5320a625fd2SDavid S. Miller 5330a625fd2SDavid S. Miller if (req->nbytes == 0) { 5340a625fd2SDavid S. Miller static const char sha256_zero[SHA256_DIGEST_SIZE] = { 5350a625fd2SDavid S. Miller 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 5360a625fd2SDavid S. Miller 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 5370a625fd2SDavid S. Miller 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 5380a625fd2SDavid S. Miller 0x1b, 0x78, 0x52, 0xb8, 0x55 5390a625fd2SDavid S. Miller }; 5400a625fd2SDavid S. Miller 5410a625fd2SDavid S. Miller memcpy(req->result, sha256_zero, SHA256_DIGEST_SIZE); 5420a625fd2SDavid S. Miller return 0; 5430a625fd2SDavid S. Miller } 5440a625fd2SDavid S. Miller s->state[0] = SHA256_H0; 5450a625fd2SDavid S. Miller s->state[1] = SHA256_H1; 5460a625fd2SDavid S. Miller s->state[2] = SHA256_H2; 5470a625fd2SDavid S. Miller s->state[3] = SHA256_H3; 5480a625fd2SDavid S. Miller s->state[4] = SHA256_H4; 5490a625fd2SDavid S. Miller s->state[5] = SHA256_H5; 5500a625fd2SDavid S. Miller s->state[6] = SHA256_H6; 5510a625fd2SDavid S. Miller s->state[7] = SHA256_H7; 5520a625fd2SDavid S. Miller 5530a625fd2SDavid S. Miller return n2_hash_async_digest(req, AUTH_TYPE_SHA256, 5540a625fd2SDavid S. Miller SHA256_DIGEST_SIZE, SHA256_DIGEST_SIZE, 5550a625fd2SDavid S. Miller s->state); 5560a625fd2SDavid S. Miller } 5570a625fd2SDavid S. Miller 5580a625fd2SDavid S. Miller static int n2_sha224_async_digest(struct ahash_request *req) 5590a625fd2SDavid S. Miller { 560c9aa55e5SDavid S. Miller struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 561c9aa55e5SDavid S. Miller struct sha256_state *s = &rctx->u.sha256; 5620a625fd2SDavid S. Miller 5630a625fd2SDavid S. Miller if (req->nbytes == 0) { 5640a625fd2SDavid S. Miller static const char sha224_zero[SHA224_DIGEST_SIZE] = { 5650a625fd2SDavid S. Miller 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, 5660a625fd2SDavid S. Miller 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, 5670a625fd2SDavid S. Miller 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, 5680a625fd2SDavid S. Miller 0x2f 5690a625fd2SDavid S. Miller }; 5700a625fd2SDavid S. Miller 5710a625fd2SDavid S. Miller memcpy(req->result, sha224_zero, SHA224_DIGEST_SIZE); 5720a625fd2SDavid S. Miller return 0; 5730a625fd2SDavid S. Miller } 5740a625fd2SDavid S. Miller s->state[0] = SHA224_H0; 5750a625fd2SDavid S. Miller s->state[1] = SHA224_H1; 5760a625fd2SDavid S. Miller s->state[2] = SHA224_H2; 5770a625fd2SDavid S. Miller s->state[3] = SHA224_H3; 5780a625fd2SDavid S. Miller s->state[4] = SHA224_H4; 5790a625fd2SDavid S. Miller s->state[5] = SHA224_H5; 5800a625fd2SDavid S. Miller s->state[6] = SHA224_H6; 5810a625fd2SDavid S. Miller s->state[7] = SHA224_H7; 5820a625fd2SDavid S. Miller 5830a625fd2SDavid S. Miller return n2_hash_async_digest(req, AUTH_TYPE_SHA256, 5840a625fd2SDavid S. Miller SHA256_DIGEST_SIZE, SHA224_DIGEST_SIZE, 5850a625fd2SDavid S. Miller s->state); 5860a625fd2SDavid S. Miller } 5870a625fd2SDavid S. Miller 5880a625fd2SDavid S. Miller struct n2_cipher_context { 5890a625fd2SDavid S. Miller int key_len; 5900a625fd2SDavid S. Miller int enc_type; 5910a625fd2SDavid S. Miller union { 5920a625fd2SDavid S. Miller u8 aes[AES_MAX_KEY_SIZE]; 5930a625fd2SDavid S. Miller u8 des[DES_KEY_SIZE]; 5940a625fd2SDavid S. Miller u8 des3[3 * DES_KEY_SIZE]; 5950a625fd2SDavid S. Miller u8 arc4[258]; /* S-box, X, Y */ 5960a625fd2SDavid S. Miller } key; 5970a625fd2SDavid S. Miller }; 5980a625fd2SDavid S. Miller 5990a625fd2SDavid S. Miller #define N2_CHUNK_ARR_LEN 16 6000a625fd2SDavid S. Miller 6010a625fd2SDavid S. Miller struct n2_crypto_chunk { 6020a625fd2SDavid S. Miller struct list_head entry; 6030a625fd2SDavid S. Miller unsigned long iv_paddr : 44; 6040a625fd2SDavid S. Miller unsigned long arr_len : 20; 6050a625fd2SDavid S. Miller unsigned long dest_paddr; 6060a625fd2SDavid S. Miller unsigned long dest_final; 6070a625fd2SDavid S. Miller struct { 6080a625fd2SDavid S. Miller unsigned long src_paddr : 44; 6090a625fd2SDavid S. Miller unsigned long src_len : 20; 6100a625fd2SDavid S. Miller } arr[N2_CHUNK_ARR_LEN]; 6110a625fd2SDavid S. Miller }; 6120a625fd2SDavid S. Miller 6130a625fd2SDavid S. Miller struct n2_request_context { 6140a625fd2SDavid S. Miller struct ablkcipher_walk walk; 6150a625fd2SDavid S. Miller struct list_head chunk_list; 6160a625fd2SDavid S. Miller struct n2_crypto_chunk chunk; 6170a625fd2SDavid S. Miller u8 temp_iv[16]; 6180a625fd2SDavid S. Miller }; 6190a625fd2SDavid S. Miller 6200a625fd2SDavid S. Miller /* The SPU allows some level of flexibility for partial cipher blocks 6210a625fd2SDavid S. Miller * being specified in a descriptor. 6220a625fd2SDavid S. Miller * 6230a625fd2SDavid S. Miller * It merely requires that every descriptor's length field is at least 6240a625fd2SDavid S. Miller * as large as the cipher block size. This means that a cipher block 6250a625fd2SDavid S. Miller * can span at most 2 descriptors. However, this does not allow a 6260a625fd2SDavid S. Miller * partial block to span into the final descriptor as that would 6270a625fd2SDavid S. Miller * violate the rule (since every descriptor's length must be at lest 6280a625fd2SDavid S. Miller * the block size). So, for example, assuming an 8 byte block size: 6290a625fd2SDavid S. Miller * 6300a625fd2SDavid S. Miller * 0xe --> 0xa --> 0x8 6310a625fd2SDavid S. Miller * 6320a625fd2SDavid S. Miller * is a valid length sequence, whereas: 6330a625fd2SDavid S. Miller * 6340a625fd2SDavid S. Miller * 0xe --> 0xb --> 0x7 6350a625fd2SDavid S. Miller * 6360a625fd2SDavid S. Miller * is not a valid sequence. 6370a625fd2SDavid S. Miller */ 6380a625fd2SDavid S. Miller 6390a625fd2SDavid S. Miller struct n2_cipher_alg { 6400a625fd2SDavid S. Miller struct list_head entry; 6410a625fd2SDavid S. Miller u8 enc_type; 6420a625fd2SDavid S. Miller struct crypto_alg alg; 6430a625fd2SDavid S. Miller }; 6440a625fd2SDavid S. Miller 6450a625fd2SDavid S. Miller static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm) 6460a625fd2SDavid S. Miller { 6470a625fd2SDavid S. Miller struct crypto_alg *alg = tfm->__crt_alg; 6480a625fd2SDavid S. Miller 6490a625fd2SDavid S. Miller return container_of(alg, struct n2_cipher_alg, alg); 6500a625fd2SDavid S. Miller } 6510a625fd2SDavid S. Miller 6520a625fd2SDavid S. Miller struct n2_cipher_request_context { 6530a625fd2SDavid S. Miller struct ablkcipher_walk walk; 6540a625fd2SDavid S. Miller }; 6550a625fd2SDavid S. Miller 6560a625fd2SDavid S. Miller static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 6570a625fd2SDavid S. Miller unsigned int keylen) 6580a625fd2SDavid S. Miller { 6590a625fd2SDavid S. Miller struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 6600a625fd2SDavid S. Miller struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 6610a625fd2SDavid S. Miller struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 6620a625fd2SDavid S. Miller 6630a625fd2SDavid S. Miller ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); 6640a625fd2SDavid S. Miller 6650a625fd2SDavid S. Miller switch (keylen) { 6660a625fd2SDavid S. Miller case AES_KEYSIZE_128: 6670a625fd2SDavid S. Miller ctx->enc_type |= ENC_TYPE_ALG_AES128; 6680a625fd2SDavid S. Miller break; 6690a625fd2SDavid S. Miller case AES_KEYSIZE_192: 6700a625fd2SDavid S. Miller ctx->enc_type |= ENC_TYPE_ALG_AES192; 6710a625fd2SDavid S. Miller break; 6720a625fd2SDavid S. Miller case AES_KEYSIZE_256: 6730a625fd2SDavid S. Miller ctx->enc_type |= ENC_TYPE_ALG_AES256; 6740a625fd2SDavid S. Miller break; 6750a625fd2SDavid S. Miller default: 6760a625fd2SDavid S. Miller crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 6770a625fd2SDavid S. Miller return -EINVAL; 6780a625fd2SDavid S. Miller } 6790a625fd2SDavid S. Miller 6800a625fd2SDavid S. Miller ctx->key_len = keylen; 6810a625fd2SDavid S. Miller memcpy(ctx->key.aes, key, keylen); 6820a625fd2SDavid S. Miller return 0; 6830a625fd2SDavid S. Miller } 6840a625fd2SDavid S. Miller 6850a625fd2SDavid S. Miller static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 6860a625fd2SDavid S. Miller unsigned int keylen) 6870a625fd2SDavid S. Miller { 6880a625fd2SDavid S. Miller struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 6890a625fd2SDavid S. Miller struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 6900a625fd2SDavid S. Miller struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 6910a625fd2SDavid S. Miller u32 tmp[DES_EXPKEY_WORDS]; 6920a625fd2SDavid S. Miller int err; 6930a625fd2SDavid S. Miller 6940a625fd2SDavid S. Miller ctx->enc_type = n2alg->enc_type; 6950a625fd2SDavid S. Miller 6960a625fd2SDavid S. Miller if (keylen != DES_KEY_SIZE) { 6970a625fd2SDavid S. Miller crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 6980a625fd2SDavid S. Miller return -EINVAL; 6990a625fd2SDavid S. Miller } 7000a625fd2SDavid S. Miller 7010a625fd2SDavid S. Miller err = des_ekey(tmp, key); 7020a625fd2SDavid S. Miller if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 7030a625fd2SDavid S. Miller tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; 7040a625fd2SDavid S. Miller return -EINVAL; 7050a625fd2SDavid S. Miller } 7060a625fd2SDavid S. Miller 7070a625fd2SDavid S. Miller ctx->key_len = keylen; 7080a625fd2SDavid S. Miller memcpy(ctx->key.des, key, keylen); 7090a625fd2SDavid S. Miller return 0; 7100a625fd2SDavid S. Miller } 7110a625fd2SDavid S. Miller 7120a625fd2SDavid S. Miller static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 7130a625fd2SDavid S. Miller unsigned int keylen) 7140a625fd2SDavid S. Miller { 7150a625fd2SDavid S. Miller struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 7160a625fd2SDavid S. Miller struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 7170a625fd2SDavid S. Miller struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 7180a625fd2SDavid S. Miller 7190a625fd2SDavid S. Miller ctx->enc_type = n2alg->enc_type; 7200a625fd2SDavid S. Miller 7210a625fd2SDavid S. Miller if (keylen != (3 * DES_KEY_SIZE)) { 7220a625fd2SDavid S. Miller crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 7230a625fd2SDavid S. Miller return -EINVAL; 7240a625fd2SDavid S. Miller } 7250a625fd2SDavid S. Miller ctx->key_len = keylen; 7260a625fd2SDavid S. Miller memcpy(ctx->key.des3, key, keylen); 7270a625fd2SDavid S. Miller return 0; 7280a625fd2SDavid S. Miller } 7290a625fd2SDavid S. Miller 7300a625fd2SDavid S. Miller static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 7310a625fd2SDavid S. Miller unsigned int keylen) 7320a625fd2SDavid S. Miller { 7330a625fd2SDavid S. Miller struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 7340a625fd2SDavid S. Miller struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 7350a625fd2SDavid S. Miller struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); 7360a625fd2SDavid S. Miller u8 *s = ctx->key.arc4; 7370a625fd2SDavid S. Miller u8 *x = s + 256; 7380a625fd2SDavid S. Miller u8 *y = x + 1; 7390a625fd2SDavid S. Miller int i, j, k; 7400a625fd2SDavid S. Miller 7410a625fd2SDavid S. Miller ctx->enc_type = n2alg->enc_type; 7420a625fd2SDavid S. Miller 7430a625fd2SDavid S. Miller j = k = 0; 7440a625fd2SDavid S. Miller *x = 0; 7450a625fd2SDavid S. Miller *y = 0; 7460a625fd2SDavid S. Miller for (i = 0; i < 256; i++) 7470a625fd2SDavid S. Miller s[i] = i; 7480a625fd2SDavid S. Miller for (i = 0; i < 256; i++) { 7490a625fd2SDavid S. Miller u8 a = s[i]; 7500a625fd2SDavid S. Miller j = (j + key[k] + a) & 0xff; 7510a625fd2SDavid S. Miller s[i] = s[j]; 7520a625fd2SDavid S. Miller s[j] = a; 7530a625fd2SDavid S. Miller if (++k >= keylen) 7540a625fd2SDavid S. Miller k = 0; 7550a625fd2SDavid S. Miller } 7560a625fd2SDavid S. Miller 7570a625fd2SDavid S. Miller return 0; 7580a625fd2SDavid S. Miller } 7590a625fd2SDavid S. Miller 7600a625fd2SDavid S. Miller static inline int cipher_descriptor_len(int nbytes, unsigned int block_size) 7610a625fd2SDavid S. Miller { 7620a625fd2SDavid S. Miller int this_len = nbytes; 7630a625fd2SDavid S. Miller 7640a625fd2SDavid S. Miller this_len -= (nbytes & (block_size - 1)); 7650a625fd2SDavid S. Miller return this_len > (1 << 16) ? (1 << 16) : this_len; 7660a625fd2SDavid S. Miller } 7670a625fd2SDavid S. Miller 7680a625fd2SDavid S. Miller static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp, 7690a625fd2SDavid S. Miller struct spu_queue *qp, bool encrypt) 7700a625fd2SDavid S. Miller { 7710a625fd2SDavid S. Miller struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); 7720a625fd2SDavid S. Miller struct cwq_initial_entry *ent; 7730a625fd2SDavid S. Miller bool in_place; 7740a625fd2SDavid S. Miller int i; 7750a625fd2SDavid S. Miller 7760a625fd2SDavid S. Miller ent = spu_queue_alloc(qp, cp->arr_len); 7770a625fd2SDavid S. Miller if (!ent) { 7780a625fd2SDavid S. Miller pr_info("queue_alloc() of %d fails\n", 7790a625fd2SDavid S. Miller cp->arr_len); 7800a625fd2SDavid S. Miller return -EBUSY; 7810a625fd2SDavid S. Miller } 7820a625fd2SDavid S. Miller 7830a625fd2SDavid S. Miller in_place = (cp->dest_paddr == cp->arr[0].src_paddr); 7840a625fd2SDavid S. Miller 7850a625fd2SDavid S. Miller ent->control = control_word_base(cp->arr[0].src_len, 7860a625fd2SDavid S. Miller 0, ctx->enc_type, 0, 0, 7870a625fd2SDavid S. Miller false, true, false, encrypt, 7880a625fd2SDavid S. Miller OPCODE_ENCRYPT | 7890a625fd2SDavid S. Miller (in_place ? OPCODE_INPLACE_BIT : 0)); 7900a625fd2SDavid S. Miller ent->src_addr = cp->arr[0].src_paddr; 7910a625fd2SDavid S. Miller ent->auth_key_addr = 0UL; 7920a625fd2SDavid S. Miller ent->auth_iv_addr = 0UL; 7930a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL; 7940a625fd2SDavid S. Miller ent->enc_key_addr = __pa(&ctx->key); 7950a625fd2SDavid S. Miller ent->enc_iv_addr = cp->iv_paddr; 7960a625fd2SDavid S. Miller ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); 7970a625fd2SDavid S. Miller 7980a625fd2SDavid S. Miller for (i = 1; i < cp->arr_len; i++) { 7990a625fd2SDavid S. Miller ent = spu_queue_next(qp, ent); 8000a625fd2SDavid S. Miller 8010a625fd2SDavid S. Miller ent->control = cp->arr[i].src_len - 1; 8020a625fd2SDavid S. Miller ent->src_addr = cp->arr[i].src_paddr; 8030a625fd2SDavid S. Miller ent->auth_key_addr = 0UL; 8040a625fd2SDavid S. Miller ent->auth_iv_addr = 0UL; 8050a625fd2SDavid S. Miller ent->final_auth_state_addr = 0UL; 8060a625fd2SDavid S. Miller ent->enc_key_addr = 0UL; 8070a625fd2SDavid S. Miller ent->enc_iv_addr = 0UL; 8080a625fd2SDavid S. Miller ent->dest_addr = 0UL; 8090a625fd2SDavid S. Miller } 8100a625fd2SDavid S. Miller ent->control |= CONTROL_END_OF_BLOCK; 8110a625fd2SDavid S. Miller 8120a625fd2SDavid S. Miller return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; 8130a625fd2SDavid S. Miller } 8140a625fd2SDavid S. Miller 8150a625fd2SDavid S. Miller static int n2_compute_chunks(struct ablkcipher_request *req) 8160a625fd2SDavid S. Miller { 8170a625fd2SDavid S. Miller struct n2_request_context *rctx = ablkcipher_request_ctx(req); 8180a625fd2SDavid S. Miller struct ablkcipher_walk *walk = &rctx->walk; 8190a625fd2SDavid S. Miller struct n2_crypto_chunk *chunk; 8200a625fd2SDavid S. Miller unsigned long dest_prev; 8210a625fd2SDavid S. Miller unsigned int tot_len; 8220a625fd2SDavid S. Miller bool prev_in_place; 8230a625fd2SDavid S. Miller int err, nbytes; 8240a625fd2SDavid S. Miller 8250a625fd2SDavid S. Miller ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes); 8260a625fd2SDavid S. Miller err = ablkcipher_walk_phys(req, walk); 8270a625fd2SDavid S. Miller if (err) 8280a625fd2SDavid S. Miller return err; 8290a625fd2SDavid S. Miller 8300a625fd2SDavid S. Miller INIT_LIST_HEAD(&rctx->chunk_list); 8310a625fd2SDavid S. Miller 8320a625fd2SDavid S. Miller chunk = &rctx->chunk; 8330a625fd2SDavid S. Miller INIT_LIST_HEAD(&chunk->entry); 8340a625fd2SDavid S. Miller 8350a625fd2SDavid S. Miller chunk->iv_paddr = 0UL; 8360a625fd2SDavid S. Miller chunk->arr_len = 0; 8370a625fd2SDavid S. Miller chunk->dest_paddr = 0UL; 8380a625fd2SDavid S. Miller 8390a625fd2SDavid S. Miller prev_in_place = false; 8400a625fd2SDavid S. Miller dest_prev = ~0UL; 8410a625fd2SDavid S. Miller tot_len = 0; 8420a625fd2SDavid S. Miller 8430a625fd2SDavid S. Miller while ((nbytes = walk->nbytes) != 0) { 8440a625fd2SDavid S. Miller unsigned long dest_paddr, src_paddr; 8450a625fd2SDavid S. Miller bool in_place; 8460a625fd2SDavid S. Miller int this_len; 8470a625fd2SDavid S. Miller 8480a625fd2SDavid S. Miller src_paddr = (page_to_phys(walk->src.page) + 8490a625fd2SDavid S. Miller walk->src.offset); 8500a625fd2SDavid S. Miller dest_paddr = (page_to_phys(walk->dst.page) + 8510a625fd2SDavid S. Miller walk->dst.offset); 8520a625fd2SDavid S. Miller in_place = (src_paddr == dest_paddr); 8530a625fd2SDavid S. Miller this_len = cipher_descriptor_len(nbytes, walk->blocksize); 8540a625fd2SDavid S. Miller 8550a625fd2SDavid S. Miller if (chunk->arr_len != 0) { 8560a625fd2SDavid S. Miller if (in_place != prev_in_place || 8570a625fd2SDavid S. Miller (!prev_in_place && 8580a625fd2SDavid S. Miller dest_paddr != dest_prev) || 8590a625fd2SDavid S. Miller chunk->arr_len == N2_CHUNK_ARR_LEN || 8600a625fd2SDavid S. Miller tot_len + this_len > (1 << 16)) { 8610a625fd2SDavid S. Miller chunk->dest_final = dest_prev; 8620a625fd2SDavid S. Miller list_add_tail(&chunk->entry, 8630a625fd2SDavid S. Miller &rctx->chunk_list); 8640a625fd2SDavid S. Miller chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC); 8650a625fd2SDavid S. Miller if (!chunk) { 8660a625fd2SDavid S. Miller err = -ENOMEM; 8670a625fd2SDavid S. Miller break; 8680a625fd2SDavid S. Miller } 8690a625fd2SDavid S. Miller INIT_LIST_HEAD(&chunk->entry); 8700a625fd2SDavid S. Miller } 8710a625fd2SDavid S. Miller } 8720a625fd2SDavid S. Miller if (chunk->arr_len == 0) { 8730a625fd2SDavid S. Miller chunk->dest_paddr = dest_paddr; 8740a625fd2SDavid S. Miller tot_len = 0; 8750a625fd2SDavid S. Miller } 8760a625fd2SDavid S. Miller chunk->arr[chunk->arr_len].src_paddr = src_paddr; 8770a625fd2SDavid S. Miller chunk->arr[chunk->arr_len].src_len = this_len; 8780a625fd2SDavid S. Miller chunk->arr_len++; 8790a625fd2SDavid S. Miller 8800a625fd2SDavid S. Miller dest_prev = dest_paddr + this_len; 8810a625fd2SDavid S. Miller prev_in_place = in_place; 8820a625fd2SDavid S. Miller tot_len += this_len; 8830a625fd2SDavid S. Miller 8840a625fd2SDavid S. Miller err = ablkcipher_walk_done(req, walk, nbytes - this_len); 8850a625fd2SDavid S. Miller if (err) 8860a625fd2SDavid S. Miller break; 8870a625fd2SDavid S. Miller } 8880a625fd2SDavid S. Miller if (!err && chunk->arr_len != 0) { 8890a625fd2SDavid S. Miller chunk->dest_final = dest_prev; 8900a625fd2SDavid S. Miller list_add_tail(&chunk->entry, &rctx->chunk_list); 8910a625fd2SDavid S. Miller } 8920a625fd2SDavid S. Miller 8930a625fd2SDavid S. Miller return err; 8940a625fd2SDavid S. Miller } 8950a625fd2SDavid S. Miller 8960a625fd2SDavid S. Miller static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv) 8970a625fd2SDavid S. Miller { 8980a625fd2SDavid S. Miller struct n2_request_context *rctx = ablkcipher_request_ctx(req); 8990a625fd2SDavid S. Miller struct n2_crypto_chunk *c, *tmp; 9000a625fd2SDavid S. Miller 9010a625fd2SDavid S. Miller if (final_iv) 9020a625fd2SDavid S. Miller memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); 9030a625fd2SDavid S. Miller 9040a625fd2SDavid S. Miller ablkcipher_walk_complete(&rctx->walk); 9050a625fd2SDavid S. Miller list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 9060a625fd2SDavid S. Miller list_del(&c->entry); 9070a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk)) 9080a625fd2SDavid S. Miller kfree(c); 9090a625fd2SDavid S. Miller } 9100a625fd2SDavid S. Miller 9110a625fd2SDavid S. Miller } 9120a625fd2SDavid S. Miller 9130a625fd2SDavid S. Miller static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt) 9140a625fd2SDavid S. Miller { 9150a625fd2SDavid S. Miller struct n2_request_context *rctx = ablkcipher_request_ctx(req); 9160a625fd2SDavid S. Miller struct crypto_tfm *tfm = req->base.tfm; 9170a625fd2SDavid S. Miller int err = n2_compute_chunks(req); 9180a625fd2SDavid S. Miller struct n2_crypto_chunk *c, *tmp; 9190a625fd2SDavid S. Miller unsigned long flags, hv_ret; 9200a625fd2SDavid S. Miller struct spu_queue *qp; 9210a625fd2SDavid S. Miller 9220a625fd2SDavid S. Miller if (err) 9230a625fd2SDavid S. Miller return err; 9240a625fd2SDavid S. Miller 9250a625fd2SDavid S. Miller qp = cpu_to_cwq[get_cpu()]; 9260a625fd2SDavid S. Miller err = -ENODEV; 9270a625fd2SDavid S. Miller if (!qp) 9280a625fd2SDavid S. Miller goto out; 9290a625fd2SDavid S. Miller 9300a625fd2SDavid S. Miller spin_lock_irqsave(&qp->lock, flags); 9310a625fd2SDavid S. Miller 9320a625fd2SDavid S. Miller list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 9330a625fd2SDavid S. Miller err = __n2_crypt_chunk(tfm, c, qp, encrypt); 9340a625fd2SDavid S. Miller if (err) 9350a625fd2SDavid S. Miller break; 9360a625fd2SDavid S. Miller list_del(&c->entry); 9370a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk)) 9380a625fd2SDavid S. Miller kfree(c); 9390a625fd2SDavid S. Miller } 9400a625fd2SDavid S. Miller if (!err) { 9410a625fd2SDavid S. Miller hv_ret = wait_for_tail(qp); 9420a625fd2SDavid S. Miller if (hv_ret != HV_EOK) 9430a625fd2SDavid S. Miller err = -EINVAL; 9440a625fd2SDavid S. Miller } 9450a625fd2SDavid S. Miller 9460a625fd2SDavid S. Miller spin_unlock_irqrestore(&qp->lock, flags); 9470a625fd2SDavid S. Miller 9480a625fd2SDavid S. Miller put_cpu(); 9490a625fd2SDavid S. Miller 9500a625fd2SDavid S. Miller out: 9510a625fd2SDavid S. Miller n2_chunk_complete(req, NULL); 9520a625fd2SDavid S. Miller return err; 9530a625fd2SDavid S. Miller } 9540a625fd2SDavid S. Miller 9550a625fd2SDavid S. Miller static int n2_encrypt_ecb(struct ablkcipher_request *req) 9560a625fd2SDavid S. Miller { 9570a625fd2SDavid S. Miller return n2_do_ecb(req, true); 9580a625fd2SDavid S. Miller } 9590a625fd2SDavid S. Miller 9600a625fd2SDavid S. Miller static int n2_decrypt_ecb(struct ablkcipher_request *req) 9610a625fd2SDavid S. Miller { 9620a625fd2SDavid S. Miller return n2_do_ecb(req, false); 9630a625fd2SDavid S. Miller } 9640a625fd2SDavid S. Miller 9650a625fd2SDavid S. Miller static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt) 9660a625fd2SDavid S. Miller { 9670a625fd2SDavid S. Miller struct n2_request_context *rctx = ablkcipher_request_ctx(req); 9680a625fd2SDavid S. Miller struct crypto_tfm *tfm = req->base.tfm; 9690a625fd2SDavid S. Miller unsigned long flags, hv_ret, iv_paddr; 9700a625fd2SDavid S. Miller int err = n2_compute_chunks(req); 9710a625fd2SDavid S. Miller struct n2_crypto_chunk *c, *tmp; 9720a625fd2SDavid S. Miller struct spu_queue *qp; 9730a625fd2SDavid S. Miller void *final_iv_addr; 9740a625fd2SDavid S. Miller 9750a625fd2SDavid S. Miller final_iv_addr = NULL; 9760a625fd2SDavid S. Miller 9770a625fd2SDavid S. Miller if (err) 9780a625fd2SDavid S. Miller return err; 9790a625fd2SDavid S. Miller 9800a625fd2SDavid S. Miller qp = cpu_to_cwq[get_cpu()]; 9810a625fd2SDavid S. Miller err = -ENODEV; 9820a625fd2SDavid S. Miller if (!qp) 9830a625fd2SDavid S. Miller goto out; 9840a625fd2SDavid S. Miller 9850a625fd2SDavid S. Miller spin_lock_irqsave(&qp->lock, flags); 9860a625fd2SDavid S. Miller 9870a625fd2SDavid S. Miller if (encrypt) { 9880a625fd2SDavid S. Miller iv_paddr = __pa(rctx->walk.iv); 9890a625fd2SDavid S. Miller list_for_each_entry_safe(c, tmp, &rctx->chunk_list, 9900a625fd2SDavid S. Miller entry) { 9910a625fd2SDavid S. Miller c->iv_paddr = iv_paddr; 9920a625fd2SDavid S. Miller err = __n2_crypt_chunk(tfm, c, qp, true); 9930a625fd2SDavid S. Miller if (err) 9940a625fd2SDavid S. Miller break; 9950a625fd2SDavid S. Miller iv_paddr = c->dest_final - rctx->walk.blocksize; 9960a625fd2SDavid S. Miller list_del(&c->entry); 9970a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk)) 9980a625fd2SDavid S. Miller kfree(c); 9990a625fd2SDavid S. Miller } 10000a625fd2SDavid S. Miller final_iv_addr = __va(iv_paddr); 10010a625fd2SDavid S. Miller } else { 10020a625fd2SDavid S. Miller list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, 10030a625fd2SDavid S. Miller entry) { 10040a625fd2SDavid S. Miller if (c == &rctx->chunk) { 10050a625fd2SDavid S. Miller iv_paddr = __pa(rctx->walk.iv); 10060a625fd2SDavid S. Miller } else { 10070a625fd2SDavid S. Miller iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + 10080a625fd2SDavid S. Miller tmp->arr[tmp->arr_len-1].src_len - 10090a625fd2SDavid S. Miller rctx->walk.blocksize); 10100a625fd2SDavid S. Miller } 10110a625fd2SDavid S. Miller if (!final_iv_addr) { 10120a625fd2SDavid S. Miller unsigned long pa; 10130a625fd2SDavid S. Miller 10140a625fd2SDavid S. Miller pa = (c->arr[c->arr_len-1].src_paddr + 10150a625fd2SDavid S. Miller c->arr[c->arr_len-1].src_len - 10160a625fd2SDavid S. Miller rctx->walk.blocksize); 10170a625fd2SDavid S. Miller final_iv_addr = rctx->temp_iv; 10180a625fd2SDavid S. Miller memcpy(rctx->temp_iv, __va(pa), 10190a625fd2SDavid S. Miller rctx->walk.blocksize); 10200a625fd2SDavid S. Miller } 10210a625fd2SDavid S. Miller c->iv_paddr = iv_paddr; 10220a625fd2SDavid S. Miller err = __n2_crypt_chunk(tfm, c, qp, false); 10230a625fd2SDavid S. Miller if (err) 10240a625fd2SDavid S. Miller break; 10250a625fd2SDavid S. Miller list_del(&c->entry); 10260a625fd2SDavid S. Miller if (unlikely(c != &rctx->chunk)) 10270a625fd2SDavid S. Miller kfree(c); 10280a625fd2SDavid S. Miller } 10290a625fd2SDavid S. Miller } 10300a625fd2SDavid S. Miller if (!err) { 10310a625fd2SDavid S. Miller hv_ret = wait_for_tail(qp); 10320a625fd2SDavid S. Miller if (hv_ret != HV_EOK) 10330a625fd2SDavid S. Miller err = -EINVAL; 10340a625fd2SDavid S. Miller } 10350a625fd2SDavid S. Miller 10360a625fd2SDavid S. Miller spin_unlock_irqrestore(&qp->lock, flags); 10370a625fd2SDavid S. Miller 10380a625fd2SDavid S. Miller put_cpu(); 10390a625fd2SDavid S. Miller 10400a625fd2SDavid S. Miller out: 10410a625fd2SDavid S. Miller n2_chunk_complete(req, err ? NULL : final_iv_addr); 10420a625fd2SDavid S. Miller return err; 10430a625fd2SDavid S. Miller } 10440a625fd2SDavid S. Miller 10450a625fd2SDavid S. Miller static int n2_encrypt_chaining(struct ablkcipher_request *req) 10460a625fd2SDavid S. Miller { 10470a625fd2SDavid S. Miller return n2_do_chaining(req, true); 10480a625fd2SDavid S. Miller } 10490a625fd2SDavid S. Miller 10500a625fd2SDavid S. Miller static int n2_decrypt_chaining(struct ablkcipher_request *req) 10510a625fd2SDavid S. Miller { 10520a625fd2SDavid S. Miller return n2_do_chaining(req, false); 10530a625fd2SDavid S. Miller } 10540a625fd2SDavid S. Miller 10550a625fd2SDavid S. Miller struct n2_cipher_tmpl { 10560a625fd2SDavid S. Miller const char *name; 10570a625fd2SDavid S. Miller const char *drv_name; 10580a625fd2SDavid S. Miller u8 block_size; 10590a625fd2SDavid S. Miller u8 enc_type; 10600a625fd2SDavid S. Miller struct ablkcipher_alg ablkcipher; 10610a625fd2SDavid S. Miller }; 10620a625fd2SDavid S. Miller 10630a625fd2SDavid S. Miller static const struct n2_cipher_tmpl cipher_tmpls[] = { 10640a625fd2SDavid S. Miller /* ARC4: only ECB is supported (chaining bits ignored) */ 10650a625fd2SDavid S. Miller { .name = "ecb(arc4)", 10660a625fd2SDavid S. Miller .drv_name = "ecb-arc4", 10670a625fd2SDavid S. Miller .block_size = 1, 10680a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_RC4_STREAM | 10690a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB), 10700a625fd2SDavid S. Miller .ablkcipher = { 10710a625fd2SDavid S. Miller .min_keysize = 1, 10720a625fd2SDavid S. Miller .max_keysize = 256, 10730a625fd2SDavid S. Miller .setkey = n2_arc4_setkey, 10740a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb, 10750a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb, 10760a625fd2SDavid S. Miller }, 10770a625fd2SDavid S. Miller }, 10780a625fd2SDavid S. Miller 10790a625fd2SDavid S. Miller /* DES: ECB CBC and CFB are supported */ 10800a625fd2SDavid S. Miller { .name = "ecb(des)", 10810a625fd2SDavid S. Miller .drv_name = "ecb-des", 10820a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE, 10830a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_DES | 10840a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB), 10850a625fd2SDavid S. Miller .ablkcipher = { 10860a625fd2SDavid S. Miller .min_keysize = DES_KEY_SIZE, 10870a625fd2SDavid S. Miller .max_keysize = DES_KEY_SIZE, 10880a625fd2SDavid S. Miller .setkey = n2_des_setkey, 10890a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb, 10900a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb, 10910a625fd2SDavid S. Miller }, 10920a625fd2SDavid S. Miller }, 10930a625fd2SDavid S. Miller { .name = "cbc(des)", 10940a625fd2SDavid S. Miller .drv_name = "cbc-des", 10950a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE, 10960a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_DES | 10970a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CBC), 10980a625fd2SDavid S. Miller .ablkcipher = { 10990a625fd2SDavid S. Miller .ivsize = DES_BLOCK_SIZE, 11000a625fd2SDavid S. Miller .min_keysize = DES_KEY_SIZE, 11010a625fd2SDavid S. Miller .max_keysize = DES_KEY_SIZE, 11020a625fd2SDavid S. Miller .setkey = n2_des_setkey, 11030a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining, 11040a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining, 11050a625fd2SDavid S. Miller }, 11060a625fd2SDavid S. Miller }, 11070a625fd2SDavid S. Miller { .name = "cfb(des)", 11080a625fd2SDavid S. Miller .drv_name = "cfb-des", 11090a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE, 11100a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_DES | 11110a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CFB), 11120a625fd2SDavid S. Miller .ablkcipher = { 11130a625fd2SDavid S. Miller .min_keysize = DES_KEY_SIZE, 11140a625fd2SDavid S. Miller .max_keysize = DES_KEY_SIZE, 11150a625fd2SDavid S. Miller .setkey = n2_des_setkey, 11160a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining, 11170a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining, 11180a625fd2SDavid S. Miller }, 11190a625fd2SDavid S. Miller }, 11200a625fd2SDavid S. Miller 11210a625fd2SDavid S. Miller /* 3DES: ECB CBC and CFB are supported */ 11220a625fd2SDavid S. Miller { .name = "ecb(des3_ede)", 11230a625fd2SDavid S. Miller .drv_name = "ecb-3des", 11240a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE, 11250a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_3DES | 11260a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB), 11270a625fd2SDavid S. Miller .ablkcipher = { 11280a625fd2SDavid S. Miller .min_keysize = 3 * DES_KEY_SIZE, 11290a625fd2SDavid S. Miller .max_keysize = 3 * DES_KEY_SIZE, 11300a625fd2SDavid S. Miller .setkey = n2_3des_setkey, 11310a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb, 11320a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb, 11330a625fd2SDavid S. Miller }, 11340a625fd2SDavid S. Miller }, 11350a625fd2SDavid S. Miller { .name = "cbc(des3_ede)", 11360a625fd2SDavid S. Miller .drv_name = "cbc-3des", 11370a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE, 11380a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_3DES | 11390a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CBC), 11400a625fd2SDavid S. Miller .ablkcipher = { 11410a625fd2SDavid S. Miller .ivsize = DES_BLOCK_SIZE, 11420a625fd2SDavid S. Miller .min_keysize = 3 * DES_KEY_SIZE, 11430a625fd2SDavid S. Miller .max_keysize = 3 * DES_KEY_SIZE, 11440a625fd2SDavid S. Miller .setkey = n2_3des_setkey, 11450a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining, 11460a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining, 11470a625fd2SDavid S. Miller }, 11480a625fd2SDavid S. Miller }, 11490a625fd2SDavid S. Miller { .name = "cfb(des3_ede)", 11500a625fd2SDavid S. Miller .drv_name = "cfb-3des", 11510a625fd2SDavid S. Miller .block_size = DES_BLOCK_SIZE, 11520a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_3DES | 11530a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CFB), 11540a625fd2SDavid S. Miller .ablkcipher = { 11550a625fd2SDavid S. Miller .min_keysize = 3 * DES_KEY_SIZE, 11560a625fd2SDavid S. Miller .max_keysize = 3 * DES_KEY_SIZE, 11570a625fd2SDavid S. Miller .setkey = n2_3des_setkey, 11580a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining, 11590a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining, 11600a625fd2SDavid S. Miller }, 11610a625fd2SDavid S. Miller }, 11620a625fd2SDavid S. Miller /* AES: ECB CBC and CTR are supported */ 11630a625fd2SDavid S. Miller { .name = "ecb(aes)", 11640a625fd2SDavid S. Miller .drv_name = "ecb-aes", 11650a625fd2SDavid S. Miller .block_size = AES_BLOCK_SIZE, 11660a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_AES128 | 11670a625fd2SDavid S. Miller ENC_TYPE_CHAINING_ECB), 11680a625fd2SDavid S. Miller .ablkcipher = { 11690a625fd2SDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 11700a625fd2SDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 11710a625fd2SDavid S. Miller .setkey = n2_aes_setkey, 11720a625fd2SDavid S. Miller .encrypt = n2_encrypt_ecb, 11730a625fd2SDavid S. Miller .decrypt = n2_decrypt_ecb, 11740a625fd2SDavid S. Miller }, 11750a625fd2SDavid S. Miller }, 11760a625fd2SDavid S. Miller { .name = "cbc(aes)", 11770a625fd2SDavid S. Miller .drv_name = "cbc-aes", 11780a625fd2SDavid S. Miller .block_size = AES_BLOCK_SIZE, 11790a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_AES128 | 11800a625fd2SDavid S. Miller ENC_TYPE_CHAINING_CBC), 11810a625fd2SDavid S. Miller .ablkcipher = { 11820a625fd2SDavid S. Miller .ivsize = AES_BLOCK_SIZE, 11830a625fd2SDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 11840a625fd2SDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 11850a625fd2SDavid S. Miller .setkey = n2_aes_setkey, 11860a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining, 11870a625fd2SDavid S. Miller .decrypt = n2_decrypt_chaining, 11880a625fd2SDavid S. Miller }, 11890a625fd2SDavid S. Miller }, 11900a625fd2SDavid S. Miller { .name = "ctr(aes)", 11910a625fd2SDavid S. Miller .drv_name = "ctr-aes", 11920a625fd2SDavid S. Miller .block_size = AES_BLOCK_SIZE, 11930a625fd2SDavid S. Miller .enc_type = (ENC_TYPE_ALG_AES128 | 11940a625fd2SDavid S. Miller ENC_TYPE_CHAINING_COUNTER), 11950a625fd2SDavid S. Miller .ablkcipher = { 11960a625fd2SDavid S. Miller .ivsize = AES_BLOCK_SIZE, 11970a625fd2SDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 11980a625fd2SDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 11990a625fd2SDavid S. Miller .setkey = n2_aes_setkey, 12000a625fd2SDavid S. Miller .encrypt = n2_encrypt_chaining, 12010a625fd2SDavid S. Miller .decrypt = n2_encrypt_chaining, 12020a625fd2SDavid S. Miller }, 12030a625fd2SDavid S. Miller }, 12040a625fd2SDavid S. Miller 12050a625fd2SDavid S. Miller }; 12060a625fd2SDavid S. Miller #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls) 12070a625fd2SDavid S. Miller 12080a625fd2SDavid S. Miller static LIST_HEAD(cipher_algs); 12090a625fd2SDavid S. Miller 12100a625fd2SDavid S. Miller struct n2_hash_tmpl { 12110a625fd2SDavid S. Miller const char *name; 12120a625fd2SDavid S. Miller int (*digest)(struct ahash_request *req); 12130a625fd2SDavid S. Miller u8 digest_size; 12140a625fd2SDavid S. Miller u8 block_size; 12150a625fd2SDavid S. Miller }; 12160a625fd2SDavid S. Miller static const struct n2_hash_tmpl hash_tmpls[] = { 12170a625fd2SDavid S. Miller { .name = "md5", 12180a625fd2SDavid S. Miller .digest = n2_md5_async_digest, 12190a625fd2SDavid S. Miller .digest_size = MD5_DIGEST_SIZE, 12200a625fd2SDavid S. Miller .block_size = MD5_HMAC_BLOCK_SIZE }, 12210a625fd2SDavid S. Miller { .name = "sha1", 12220a625fd2SDavid S. Miller .digest = n2_sha1_async_digest, 12230a625fd2SDavid S. Miller .digest_size = SHA1_DIGEST_SIZE, 12240a625fd2SDavid S. Miller .block_size = SHA1_BLOCK_SIZE }, 12250a625fd2SDavid S. Miller { .name = "sha256", 12260a625fd2SDavid S. Miller .digest = n2_sha256_async_digest, 12270a625fd2SDavid S. Miller .digest_size = SHA256_DIGEST_SIZE, 12280a625fd2SDavid S. Miller .block_size = SHA256_BLOCK_SIZE }, 12290a625fd2SDavid S. Miller { .name = "sha224", 12300a625fd2SDavid S. Miller .digest = n2_sha224_async_digest, 12310a625fd2SDavid S. Miller .digest_size = SHA224_DIGEST_SIZE, 12320a625fd2SDavid S. Miller .block_size = SHA224_BLOCK_SIZE }, 12330a625fd2SDavid S. Miller }; 12340a625fd2SDavid S. Miller #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) 12350a625fd2SDavid S. Miller 12360a625fd2SDavid S. Miller struct n2_ahash_alg { 12370a625fd2SDavid S. Miller struct list_head entry; 12380a625fd2SDavid S. Miller struct ahash_alg alg; 12390a625fd2SDavid S. Miller }; 12400a625fd2SDavid S. Miller static LIST_HEAD(ahash_algs); 12410a625fd2SDavid S. Miller 12420a625fd2SDavid S. Miller static int algs_registered; 12430a625fd2SDavid S. Miller 12440a625fd2SDavid S. Miller static void __n2_unregister_algs(void) 12450a625fd2SDavid S. Miller { 12460a625fd2SDavid S. Miller struct n2_cipher_alg *cipher, *cipher_tmp; 12470a625fd2SDavid S. Miller struct n2_ahash_alg *alg, *alg_tmp; 12480a625fd2SDavid S. Miller 12490a625fd2SDavid S. Miller list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) { 12500a625fd2SDavid S. Miller crypto_unregister_alg(&cipher->alg); 12510a625fd2SDavid S. Miller list_del(&cipher->entry); 12520a625fd2SDavid S. Miller kfree(cipher); 12530a625fd2SDavid S. Miller } 12540a625fd2SDavid S. Miller list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { 12550a625fd2SDavid S. Miller crypto_unregister_ahash(&alg->alg); 12560a625fd2SDavid S. Miller list_del(&alg->entry); 12570a625fd2SDavid S. Miller kfree(alg); 12580a625fd2SDavid S. Miller } 12590a625fd2SDavid S. Miller } 12600a625fd2SDavid S. Miller 12610a625fd2SDavid S. Miller static int n2_cipher_cra_init(struct crypto_tfm *tfm) 12620a625fd2SDavid S. Miller { 12630a625fd2SDavid S. Miller tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context); 12640a625fd2SDavid S. Miller return 0; 12650a625fd2SDavid S. Miller } 12660a625fd2SDavid S. Miller 12670a625fd2SDavid S. Miller static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl) 12680a625fd2SDavid S. Miller { 12690a625fd2SDavid S. Miller struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 12700a625fd2SDavid S. Miller struct crypto_alg *alg; 12710a625fd2SDavid S. Miller int err; 12720a625fd2SDavid S. Miller 12730a625fd2SDavid S. Miller if (!p) 12740a625fd2SDavid S. Miller return -ENOMEM; 12750a625fd2SDavid S. Miller 12760a625fd2SDavid S. Miller alg = &p->alg; 12770a625fd2SDavid S. Miller 12780a625fd2SDavid S. Miller snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 12790a625fd2SDavid S. Miller snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name); 12800a625fd2SDavid S. Miller alg->cra_priority = N2_CRA_PRIORITY; 12810a625fd2SDavid S. Miller alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; 12820a625fd2SDavid S. Miller alg->cra_blocksize = tmpl->block_size; 12830a625fd2SDavid S. Miller p->enc_type = tmpl->enc_type; 12840a625fd2SDavid S. Miller alg->cra_ctxsize = sizeof(struct n2_cipher_context); 12850a625fd2SDavid S. Miller alg->cra_type = &crypto_ablkcipher_type; 12860a625fd2SDavid S. Miller alg->cra_u.ablkcipher = tmpl->ablkcipher; 12870a625fd2SDavid S. Miller alg->cra_init = n2_cipher_cra_init; 12880a625fd2SDavid S. Miller alg->cra_module = THIS_MODULE; 12890a625fd2SDavid S. Miller 12900a625fd2SDavid S. Miller list_add(&p->entry, &cipher_algs); 12910a625fd2SDavid S. Miller err = crypto_register_alg(alg); 12920a625fd2SDavid S. Miller if (err) { 12930a625fd2SDavid S. Miller list_del(&p->entry); 12940a625fd2SDavid S. Miller kfree(p); 12950a625fd2SDavid S. Miller } 12960a625fd2SDavid S. Miller return err; 12970a625fd2SDavid S. Miller } 12980a625fd2SDavid S. Miller 12990a625fd2SDavid S. Miller static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) 13000a625fd2SDavid S. Miller { 13010a625fd2SDavid S. Miller struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 13020a625fd2SDavid S. Miller struct hash_alg_common *halg; 13030a625fd2SDavid S. Miller struct crypto_alg *base; 13040a625fd2SDavid S. Miller struct ahash_alg *ahash; 13050a625fd2SDavid S. Miller int err; 13060a625fd2SDavid S. Miller 13070a625fd2SDavid S. Miller if (!p) 13080a625fd2SDavid S. Miller return -ENOMEM; 13090a625fd2SDavid S. Miller 13100a625fd2SDavid S. Miller ahash = &p->alg; 13110a625fd2SDavid S. Miller ahash->init = n2_hash_async_init; 13120a625fd2SDavid S. Miller ahash->update = n2_hash_async_update; 13130a625fd2SDavid S. Miller ahash->final = n2_hash_async_final; 13140a625fd2SDavid S. Miller ahash->finup = n2_hash_async_finup; 13150a625fd2SDavid S. Miller ahash->digest = tmpl->digest; 13160a625fd2SDavid S. Miller 13170a625fd2SDavid S. Miller halg = &ahash->halg; 13180a625fd2SDavid S. Miller halg->digestsize = tmpl->digest_size; 13190a625fd2SDavid S. Miller 13200a625fd2SDavid S. Miller base = &halg->base; 13210a625fd2SDavid S. Miller snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 13220a625fd2SDavid S. Miller snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name); 13230a625fd2SDavid S. Miller base->cra_priority = N2_CRA_PRIORITY; 13240a625fd2SDavid S. Miller base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK; 13250a625fd2SDavid S. Miller base->cra_blocksize = tmpl->block_size; 13260a625fd2SDavid S. Miller base->cra_ctxsize = sizeof(struct n2_hash_ctx); 13270a625fd2SDavid S. Miller base->cra_module = THIS_MODULE; 13280a625fd2SDavid S. Miller base->cra_init = n2_hash_cra_init; 13290a625fd2SDavid S. Miller base->cra_exit = n2_hash_cra_exit; 13300a625fd2SDavid S. Miller 13310a625fd2SDavid S. Miller list_add(&p->entry, &ahash_algs); 13320a625fd2SDavid S. Miller err = crypto_register_ahash(ahash); 13330a625fd2SDavid S. Miller if (err) { 13340a625fd2SDavid S. Miller list_del(&p->entry); 13350a625fd2SDavid S. Miller kfree(p); 13360a625fd2SDavid S. Miller } 13370a625fd2SDavid S. Miller return err; 13380a625fd2SDavid S. Miller } 13390a625fd2SDavid S. Miller 13400a625fd2SDavid S. Miller static int __devinit n2_register_algs(void) 13410a625fd2SDavid S. Miller { 13420a625fd2SDavid S. Miller int i, err = 0; 13430a625fd2SDavid S. Miller 13440a625fd2SDavid S. Miller mutex_lock(&spu_lock); 13450a625fd2SDavid S. Miller if (algs_registered++) 13460a625fd2SDavid S. Miller goto out; 13470a625fd2SDavid S. Miller 13480a625fd2SDavid S. Miller for (i = 0; i < NUM_HASH_TMPLS; i++) { 13490a625fd2SDavid S. Miller err = __n2_register_one_ahash(&hash_tmpls[i]); 13500a625fd2SDavid S. Miller if (err) { 13510a625fd2SDavid S. Miller __n2_unregister_algs(); 13520a625fd2SDavid S. Miller goto out; 13530a625fd2SDavid S. Miller } 13540a625fd2SDavid S. Miller } 13550a625fd2SDavid S. Miller for (i = 0; i < NUM_CIPHER_TMPLS; i++) { 13560a625fd2SDavid S. Miller err = __n2_register_one_cipher(&cipher_tmpls[i]); 13570a625fd2SDavid S. Miller if (err) { 13580a625fd2SDavid S. Miller __n2_unregister_algs(); 13590a625fd2SDavid S. Miller goto out; 13600a625fd2SDavid S. Miller } 13610a625fd2SDavid S. Miller } 13620a625fd2SDavid S. Miller 13630a625fd2SDavid S. Miller out: 13640a625fd2SDavid S. Miller mutex_unlock(&spu_lock); 13650a625fd2SDavid S. Miller return err; 13660a625fd2SDavid S. Miller } 13670a625fd2SDavid S. Miller 13680a625fd2SDavid S. Miller static void __exit n2_unregister_algs(void) 13690a625fd2SDavid S. Miller { 13700a625fd2SDavid S. Miller mutex_lock(&spu_lock); 13710a625fd2SDavid S. Miller if (!--algs_registered) 13720a625fd2SDavid S. Miller __n2_unregister_algs(); 13730a625fd2SDavid S. Miller mutex_unlock(&spu_lock); 13740a625fd2SDavid S. Miller } 13750a625fd2SDavid S. Miller 13760a625fd2SDavid S. Miller /* To map CWQ queues to interrupt sources, the hypervisor API provides 13770a625fd2SDavid S. Miller * a devino. This isn't very useful to us because all of the 13780a625fd2SDavid S. Miller * interrupts listed in the of_device node have been translated to 13790a625fd2SDavid S. Miller * Linux virtual IRQ cookie numbers. 13800a625fd2SDavid S. Miller * 13810a625fd2SDavid S. Miller * So we have to back-translate, going through the 'intr' and 'ino' 13820a625fd2SDavid S. Miller * property tables of the n2cp MDESC node, matching it with the OF 13830a625fd2SDavid S. Miller * 'interrupts' property entries, in order to to figure out which 13840a625fd2SDavid S. Miller * devino goes to which already-translated IRQ. 13850a625fd2SDavid S. Miller */ 13860a625fd2SDavid S. Miller static int find_devino_index(struct of_device *dev, struct spu_mdesc_info *ip, 13870a625fd2SDavid S. Miller unsigned long dev_ino) 13880a625fd2SDavid S. Miller { 13890a625fd2SDavid S. Miller const unsigned int *dev_intrs; 13900a625fd2SDavid S. Miller unsigned int intr; 13910a625fd2SDavid S. Miller int i; 13920a625fd2SDavid S. Miller 13930a625fd2SDavid S. Miller for (i = 0; i < ip->num_intrs; i++) { 13940a625fd2SDavid S. Miller if (ip->ino_table[i].ino == dev_ino) 13950a625fd2SDavid S. Miller break; 13960a625fd2SDavid S. Miller } 13970a625fd2SDavid S. Miller if (i == ip->num_intrs) 13980a625fd2SDavid S. Miller return -ENODEV; 13990a625fd2SDavid S. Miller 14000a625fd2SDavid S. Miller intr = ip->ino_table[i].intr; 14010a625fd2SDavid S. Miller 1402ff6c7341SDavid S. Miller dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL); 14030a625fd2SDavid S. Miller if (!dev_intrs) 14040a625fd2SDavid S. Miller return -ENODEV; 14050a625fd2SDavid S. Miller 14060a625fd2SDavid S. Miller for (i = 0; i < dev->num_irqs; i++) { 14070a625fd2SDavid S. Miller if (dev_intrs[i] == intr) 14080a625fd2SDavid S. Miller return i; 14090a625fd2SDavid S. Miller } 14100a625fd2SDavid S. Miller 14110a625fd2SDavid S. Miller return -ENODEV; 14120a625fd2SDavid S. Miller } 14130a625fd2SDavid S. Miller 14140a625fd2SDavid S. Miller static int spu_map_ino(struct of_device *dev, struct spu_mdesc_info *ip, 14150a625fd2SDavid S. Miller const char *irq_name, struct spu_queue *p, 14160a625fd2SDavid S. Miller irq_handler_t handler) 14170a625fd2SDavid S. Miller { 14180a625fd2SDavid S. Miller unsigned long herr; 14190a625fd2SDavid S. Miller int index; 14200a625fd2SDavid S. Miller 14210a625fd2SDavid S. Miller herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino); 14220a625fd2SDavid S. Miller if (herr) 14230a625fd2SDavid S. Miller return -EINVAL; 14240a625fd2SDavid S. Miller 14250a625fd2SDavid S. Miller index = find_devino_index(dev, ip, p->devino); 14260a625fd2SDavid S. Miller if (index < 0) 14270a625fd2SDavid S. Miller return index; 14280a625fd2SDavid S. Miller 14290a625fd2SDavid S. Miller p->irq = dev->irqs[index]; 14300a625fd2SDavid S. Miller 14310a625fd2SDavid S. Miller sprintf(p->irq_name, "%s-%d", irq_name, index); 14320a625fd2SDavid S. Miller 14330a625fd2SDavid S. Miller return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM, 14340a625fd2SDavid S. Miller p->irq_name, p); 14350a625fd2SDavid S. Miller } 14360a625fd2SDavid S. Miller 14370a625fd2SDavid S. Miller static struct kmem_cache *queue_cache[2]; 14380a625fd2SDavid S. Miller 14390a625fd2SDavid S. Miller static void *new_queue(unsigned long q_type) 14400a625fd2SDavid S. Miller { 14410a625fd2SDavid S. Miller return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL); 14420a625fd2SDavid S. Miller } 14430a625fd2SDavid S. Miller 14440a625fd2SDavid S. Miller static void free_queue(void *p, unsigned long q_type) 14450a625fd2SDavid S. Miller { 14460a625fd2SDavid S. Miller return kmem_cache_free(queue_cache[q_type - 1], p); 14470a625fd2SDavid S. Miller } 14480a625fd2SDavid S. Miller 14490a625fd2SDavid S. Miller static int queue_cache_init(void) 14500a625fd2SDavid S. Miller { 14510a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 14520a625fd2SDavid S. Miller queue_cache[HV_NCS_QTYPE_MAU - 1] = 1453527b9525SDavid S. Miller kmem_cache_create("mau_queue", 14540a625fd2SDavid S. Miller (MAU_NUM_ENTRIES * 14550a625fd2SDavid S. Miller MAU_ENTRY_SIZE), 14560a625fd2SDavid S. Miller MAU_ENTRY_SIZE, 0, NULL); 14570a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 14580a625fd2SDavid S. Miller return -ENOMEM; 14590a625fd2SDavid S. Miller 14600a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) 14610a625fd2SDavid S. Miller queue_cache[HV_NCS_QTYPE_CWQ - 1] = 14620a625fd2SDavid S. Miller kmem_cache_create("cwq_queue", 14630a625fd2SDavid S. Miller (CWQ_NUM_ENTRIES * 14640a625fd2SDavid S. Miller CWQ_ENTRY_SIZE), 14650a625fd2SDavid S. Miller CWQ_ENTRY_SIZE, 0, NULL); 14660a625fd2SDavid S. Miller if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { 14670a625fd2SDavid S. Miller kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 14680a625fd2SDavid S. Miller return -ENOMEM; 14690a625fd2SDavid S. Miller } 14700a625fd2SDavid S. Miller return 0; 14710a625fd2SDavid S. Miller } 14720a625fd2SDavid S. Miller 14730a625fd2SDavid S. Miller static void queue_cache_destroy(void) 14740a625fd2SDavid S. Miller { 14750a625fd2SDavid S. Miller kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 14760a625fd2SDavid S. Miller kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]); 14770a625fd2SDavid S. Miller } 14780a625fd2SDavid S. Miller 14790a625fd2SDavid S. Miller static int spu_queue_register(struct spu_queue *p, unsigned long q_type) 14800a625fd2SDavid S. Miller { 14810a625fd2SDavid S. Miller cpumask_var_t old_allowed; 14820a625fd2SDavid S. Miller unsigned long hv_ret; 14830a625fd2SDavid S. Miller 14840a625fd2SDavid S. Miller if (cpumask_empty(&p->sharing)) 14850a625fd2SDavid S. Miller return -EINVAL; 14860a625fd2SDavid S. Miller 14870a625fd2SDavid S. Miller if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL)) 14880a625fd2SDavid S. Miller return -ENOMEM; 14890a625fd2SDavid S. Miller 14900a625fd2SDavid S. Miller cpumask_copy(old_allowed, ¤t->cpus_allowed); 14910a625fd2SDavid S. Miller 14920a625fd2SDavid S. Miller set_cpus_allowed_ptr(current, &p->sharing); 14930a625fd2SDavid S. Miller 14940a625fd2SDavid S. Miller hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q), 14950a625fd2SDavid S. Miller CWQ_NUM_ENTRIES, &p->qhandle); 14960a625fd2SDavid S. Miller if (!hv_ret) 14970a625fd2SDavid S. Miller sun4v_ncs_sethead_marker(p->qhandle, 0); 14980a625fd2SDavid S. Miller 14990a625fd2SDavid S. Miller set_cpus_allowed_ptr(current, old_allowed); 15000a625fd2SDavid S. Miller 15010a625fd2SDavid S. Miller free_cpumask_var(old_allowed); 15020a625fd2SDavid S. Miller 15030a625fd2SDavid S. Miller return (hv_ret ? -EINVAL : 0); 15040a625fd2SDavid S. Miller } 15050a625fd2SDavid S. Miller 15060a625fd2SDavid S. Miller static int spu_queue_setup(struct spu_queue *p) 15070a625fd2SDavid S. Miller { 15080a625fd2SDavid S. Miller int err; 15090a625fd2SDavid S. Miller 15100a625fd2SDavid S. Miller p->q = new_queue(p->q_type); 15110a625fd2SDavid S. Miller if (!p->q) 15120a625fd2SDavid S. Miller return -ENOMEM; 15130a625fd2SDavid S. Miller 15140a625fd2SDavid S. Miller err = spu_queue_register(p, p->q_type); 15150a625fd2SDavid S. Miller if (err) { 15160a625fd2SDavid S. Miller free_queue(p->q, p->q_type); 15170a625fd2SDavid S. Miller p->q = NULL; 15180a625fd2SDavid S. Miller } 15190a625fd2SDavid S. Miller 15200a625fd2SDavid S. Miller return err; 15210a625fd2SDavid S. Miller } 15220a625fd2SDavid S. Miller 15230a625fd2SDavid S. Miller static void spu_queue_destroy(struct spu_queue *p) 15240a625fd2SDavid S. Miller { 15250a625fd2SDavid S. Miller unsigned long hv_ret; 15260a625fd2SDavid S. Miller 15270a625fd2SDavid S. Miller if (!p->q) 15280a625fd2SDavid S. Miller return; 15290a625fd2SDavid S. Miller 15300a625fd2SDavid S. Miller hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle); 15310a625fd2SDavid S. Miller 15320a625fd2SDavid S. Miller if (!hv_ret) 15330a625fd2SDavid S. Miller free_queue(p->q, p->q_type); 15340a625fd2SDavid S. Miller } 15350a625fd2SDavid S. Miller 15360a625fd2SDavid S. Miller static void spu_list_destroy(struct list_head *list) 15370a625fd2SDavid S. Miller { 15380a625fd2SDavid S. Miller struct spu_queue *p, *n; 15390a625fd2SDavid S. Miller 15400a625fd2SDavid S. Miller list_for_each_entry_safe(p, n, list, list) { 15410a625fd2SDavid S. Miller int i; 15420a625fd2SDavid S. Miller 15430a625fd2SDavid S. Miller for (i = 0; i < NR_CPUS; i++) { 15440a625fd2SDavid S. Miller if (cpu_to_cwq[i] == p) 15450a625fd2SDavid S. Miller cpu_to_cwq[i] = NULL; 15460a625fd2SDavid S. Miller } 15470a625fd2SDavid S. Miller 15480a625fd2SDavid S. Miller if (p->irq) { 15490a625fd2SDavid S. Miller free_irq(p->irq, p); 15500a625fd2SDavid S. Miller p->irq = 0; 15510a625fd2SDavid S. Miller } 15520a625fd2SDavid S. Miller spu_queue_destroy(p); 15530a625fd2SDavid S. Miller list_del(&p->list); 15540a625fd2SDavid S. Miller kfree(p); 15550a625fd2SDavid S. Miller } 15560a625fd2SDavid S. Miller } 15570a625fd2SDavid S. Miller 15580a625fd2SDavid S. Miller /* Walk the backward arcs of a CWQ 'exec-unit' node, 15590a625fd2SDavid S. Miller * gathering cpu membership information. 15600a625fd2SDavid S. Miller */ 15610a625fd2SDavid S. Miller static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, 15620a625fd2SDavid S. Miller struct of_device *dev, 15630a625fd2SDavid S. Miller u64 node, struct spu_queue *p, 15640a625fd2SDavid S. Miller struct spu_queue **table) 15650a625fd2SDavid S. Miller { 15660a625fd2SDavid S. Miller u64 arc; 15670a625fd2SDavid S. Miller 15680a625fd2SDavid S. Miller mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { 15690a625fd2SDavid S. Miller u64 tgt = mdesc_arc_target(mdesc, arc); 15700a625fd2SDavid S. Miller const char *name = mdesc_node_name(mdesc, tgt); 15710a625fd2SDavid S. Miller const u64 *id; 15720a625fd2SDavid S. Miller 15730a625fd2SDavid S. Miller if (strcmp(name, "cpu")) 15740a625fd2SDavid S. Miller continue; 15750a625fd2SDavid S. Miller id = mdesc_get_property(mdesc, tgt, "id", NULL); 15760a625fd2SDavid S. Miller if (table[*id] != NULL) { 15770a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: SPU cpu slot already set.\n", 1578ff6c7341SDavid S. Miller dev->dev.of_node->full_name); 15790a625fd2SDavid S. Miller return -EINVAL; 15800a625fd2SDavid S. Miller } 15810a625fd2SDavid S. Miller cpu_set(*id, p->sharing); 15820a625fd2SDavid S. Miller table[*id] = p; 15830a625fd2SDavid S. Miller } 15840a625fd2SDavid S. Miller return 0; 15850a625fd2SDavid S. Miller } 15860a625fd2SDavid S. Miller 15870a625fd2SDavid S. Miller /* Process an 'exec-unit' MDESC node of type 'cwq'. */ 15880a625fd2SDavid S. Miller static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, 15890a625fd2SDavid S. Miller struct of_device *dev, struct mdesc_handle *mdesc, 15900a625fd2SDavid S. Miller u64 node, const char *iname, unsigned long q_type, 15910a625fd2SDavid S. Miller irq_handler_t handler, struct spu_queue **table) 15920a625fd2SDavid S. Miller { 15930a625fd2SDavid S. Miller struct spu_queue *p; 15940a625fd2SDavid S. Miller int err; 15950a625fd2SDavid S. Miller 15960a625fd2SDavid S. Miller p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); 15970a625fd2SDavid S. Miller if (!p) { 15980a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n", 1599ff6c7341SDavid S. Miller dev->dev.of_node->full_name); 16000a625fd2SDavid S. Miller return -ENOMEM; 16010a625fd2SDavid S. Miller } 16020a625fd2SDavid S. Miller 16030a625fd2SDavid S. Miller cpus_clear(p->sharing); 16040a625fd2SDavid S. Miller spin_lock_init(&p->lock); 16050a625fd2SDavid S. Miller p->q_type = q_type; 16060a625fd2SDavid S. Miller INIT_LIST_HEAD(&p->jobs); 16070a625fd2SDavid S. Miller list_add(&p->list, list); 16080a625fd2SDavid S. Miller 16090a625fd2SDavid S. Miller err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); 16100a625fd2SDavid S. Miller if (err) 16110a625fd2SDavid S. Miller return err; 16120a625fd2SDavid S. Miller 16130a625fd2SDavid S. Miller err = spu_queue_setup(p); 16140a625fd2SDavid S. Miller if (err) 16150a625fd2SDavid S. Miller return err; 16160a625fd2SDavid S. Miller 16170a625fd2SDavid S. Miller return spu_map_ino(dev, ip, iname, p, handler); 16180a625fd2SDavid S. Miller } 16190a625fd2SDavid S. Miller 16200a625fd2SDavid S. Miller static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct of_device *dev, 16210a625fd2SDavid S. Miller struct spu_mdesc_info *ip, struct list_head *list, 16220a625fd2SDavid S. Miller const char *exec_name, unsigned long q_type, 16230a625fd2SDavid S. Miller irq_handler_t handler, struct spu_queue **table) 16240a625fd2SDavid S. Miller { 16250a625fd2SDavid S. Miller int err = 0; 16260a625fd2SDavid S. Miller u64 node; 16270a625fd2SDavid S. Miller 16280a625fd2SDavid S. Miller mdesc_for_each_node_by_name(mdesc, node, "exec-unit") { 16290a625fd2SDavid S. Miller const char *type; 16300a625fd2SDavid S. Miller 16310a625fd2SDavid S. Miller type = mdesc_get_property(mdesc, node, "type", NULL); 16320a625fd2SDavid S. Miller if (!type || strcmp(type, exec_name)) 16330a625fd2SDavid S. Miller continue; 16340a625fd2SDavid S. Miller 16350a625fd2SDavid S. Miller err = handle_exec_unit(ip, list, dev, mdesc, node, 16360a625fd2SDavid S. Miller exec_name, q_type, handler, table); 16370a625fd2SDavid S. Miller if (err) { 16380a625fd2SDavid S. Miller spu_list_destroy(list); 16390a625fd2SDavid S. Miller break; 16400a625fd2SDavid S. Miller } 16410a625fd2SDavid S. Miller } 16420a625fd2SDavid S. Miller 16430a625fd2SDavid S. Miller return err; 16440a625fd2SDavid S. Miller } 16450a625fd2SDavid S. Miller 16460a625fd2SDavid S. Miller static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node, 16470a625fd2SDavid S. Miller struct spu_mdesc_info *ip) 16480a625fd2SDavid S. Miller { 16490a625fd2SDavid S. Miller const u64 *intr, *ino; 16500a625fd2SDavid S. Miller int intr_len, ino_len; 16510a625fd2SDavid S. Miller int i; 16520a625fd2SDavid S. Miller 16530a625fd2SDavid S. Miller intr = mdesc_get_property(mdesc, node, "intr", &intr_len); 16540a625fd2SDavid S. Miller if (!intr) 16550a625fd2SDavid S. Miller return -ENODEV; 16560a625fd2SDavid S. Miller 16570a625fd2SDavid S. Miller ino = mdesc_get_property(mdesc, node, "ino", &ino_len); 16580a625fd2SDavid S. Miller if (!intr) 16590a625fd2SDavid S. Miller return -ENODEV; 16600a625fd2SDavid S. Miller 16610a625fd2SDavid S. Miller if (intr_len != ino_len) 16620a625fd2SDavid S. Miller return -EINVAL; 16630a625fd2SDavid S. Miller 16640a625fd2SDavid S. Miller ip->num_intrs = intr_len / sizeof(u64); 16650a625fd2SDavid S. Miller ip->ino_table = kzalloc((sizeof(struct ino_blob) * 16660a625fd2SDavid S. Miller ip->num_intrs), 16670a625fd2SDavid S. Miller GFP_KERNEL); 16680a625fd2SDavid S. Miller if (!ip->ino_table) 16690a625fd2SDavid S. Miller return -ENOMEM; 16700a625fd2SDavid S. Miller 16710a625fd2SDavid S. Miller for (i = 0; i < ip->num_intrs; i++) { 16720a625fd2SDavid S. Miller struct ino_blob *b = &ip->ino_table[i]; 16730a625fd2SDavid S. Miller b->intr = intr[i]; 16740a625fd2SDavid S. Miller b->ino = ino[i]; 16750a625fd2SDavid S. Miller } 16760a625fd2SDavid S. Miller 16770a625fd2SDavid S. Miller return 0; 16780a625fd2SDavid S. Miller } 16790a625fd2SDavid S. Miller 16800a625fd2SDavid S. Miller static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc, 16810a625fd2SDavid S. Miller struct of_device *dev, 16820a625fd2SDavid S. Miller struct spu_mdesc_info *ip, 16830a625fd2SDavid S. Miller const char *node_name) 16840a625fd2SDavid S. Miller { 16850a625fd2SDavid S. Miller const unsigned int *reg; 16860a625fd2SDavid S. Miller u64 node; 16870a625fd2SDavid S. Miller 1688ff6c7341SDavid S. Miller reg = of_get_property(dev->dev.of_node, "reg", NULL); 16890a625fd2SDavid S. Miller if (!reg) 16900a625fd2SDavid S. Miller return -ENODEV; 16910a625fd2SDavid S. Miller 16920a625fd2SDavid S. Miller mdesc_for_each_node_by_name(mdesc, node, "virtual-device") { 16930a625fd2SDavid S. Miller const char *name; 16940a625fd2SDavid S. Miller const u64 *chdl; 16950a625fd2SDavid S. Miller 16960a625fd2SDavid S. Miller name = mdesc_get_property(mdesc, node, "name", NULL); 16970a625fd2SDavid S. Miller if (!name || strcmp(name, node_name)) 16980a625fd2SDavid S. Miller continue; 16990a625fd2SDavid S. Miller chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL); 17000a625fd2SDavid S. Miller if (!chdl || (*chdl != *reg)) 17010a625fd2SDavid S. Miller continue; 17020a625fd2SDavid S. Miller ip->cfg_handle = *chdl; 17030a625fd2SDavid S. Miller return get_irq_props(mdesc, node, ip); 17040a625fd2SDavid S. Miller } 17050a625fd2SDavid S. Miller 17060a625fd2SDavid S. Miller return -ENODEV; 17070a625fd2SDavid S. Miller } 17080a625fd2SDavid S. Miller 17090a625fd2SDavid S. Miller static unsigned long n2_spu_hvapi_major; 17100a625fd2SDavid S. Miller static unsigned long n2_spu_hvapi_minor; 17110a625fd2SDavid S. Miller 17120a625fd2SDavid S. Miller static int __devinit n2_spu_hvapi_register(void) 17130a625fd2SDavid S. Miller { 17140a625fd2SDavid S. Miller int err; 17150a625fd2SDavid S. Miller 17160a625fd2SDavid S. Miller n2_spu_hvapi_major = 2; 17170a625fd2SDavid S. Miller n2_spu_hvapi_minor = 0; 17180a625fd2SDavid S. Miller 17190a625fd2SDavid S. Miller err = sun4v_hvapi_register(HV_GRP_NCS, 17200a625fd2SDavid S. Miller n2_spu_hvapi_major, 17210a625fd2SDavid S. Miller &n2_spu_hvapi_minor); 17220a625fd2SDavid S. Miller 17230a625fd2SDavid S. Miller if (!err) 17240a625fd2SDavid S. Miller pr_info("Registered NCS HVAPI version %lu.%lu\n", 17250a625fd2SDavid S. Miller n2_spu_hvapi_major, 17260a625fd2SDavid S. Miller n2_spu_hvapi_minor); 17270a625fd2SDavid S. Miller 17280a625fd2SDavid S. Miller return err; 17290a625fd2SDavid S. Miller } 17300a625fd2SDavid S. Miller 17310a625fd2SDavid S. Miller static void n2_spu_hvapi_unregister(void) 17320a625fd2SDavid S. Miller { 17330a625fd2SDavid S. Miller sun4v_hvapi_unregister(HV_GRP_NCS); 17340a625fd2SDavid S. Miller } 17350a625fd2SDavid S. Miller 17360a625fd2SDavid S. Miller static int global_ref; 17370a625fd2SDavid S. Miller 17380a625fd2SDavid S. Miller static int __devinit grab_global_resources(void) 17390a625fd2SDavid S. Miller { 17400a625fd2SDavid S. Miller int err = 0; 17410a625fd2SDavid S. Miller 17420a625fd2SDavid S. Miller mutex_lock(&spu_lock); 17430a625fd2SDavid S. Miller 17440a625fd2SDavid S. Miller if (global_ref++) 17450a625fd2SDavid S. Miller goto out; 17460a625fd2SDavid S. Miller 17470a625fd2SDavid S. Miller err = n2_spu_hvapi_register(); 17480a625fd2SDavid S. Miller if (err) 17490a625fd2SDavid S. Miller goto out; 17500a625fd2SDavid S. Miller 17510a625fd2SDavid S. Miller err = queue_cache_init(); 17520a625fd2SDavid S. Miller if (err) 17530a625fd2SDavid S. Miller goto out_hvapi_release; 17540a625fd2SDavid S. Miller 17550a625fd2SDavid S. Miller err = -ENOMEM; 17560a625fd2SDavid S. Miller cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS, 17570a625fd2SDavid S. Miller GFP_KERNEL); 17580a625fd2SDavid S. Miller if (!cpu_to_cwq) 17590a625fd2SDavid S. Miller goto out_queue_cache_destroy; 17600a625fd2SDavid S. Miller 17610a625fd2SDavid S. Miller cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS, 17620a625fd2SDavid S. Miller GFP_KERNEL); 17630a625fd2SDavid S. Miller if (!cpu_to_mau) 17640a625fd2SDavid S. Miller goto out_free_cwq_table; 17650a625fd2SDavid S. Miller 17660a625fd2SDavid S. Miller err = 0; 17670a625fd2SDavid S. Miller 17680a625fd2SDavid S. Miller out: 17690a625fd2SDavid S. Miller if (err) 17700a625fd2SDavid S. Miller global_ref--; 17710a625fd2SDavid S. Miller mutex_unlock(&spu_lock); 17720a625fd2SDavid S. Miller return err; 17730a625fd2SDavid S. Miller 17740a625fd2SDavid S. Miller out_free_cwq_table: 17750a625fd2SDavid S. Miller kfree(cpu_to_cwq); 17760a625fd2SDavid S. Miller cpu_to_cwq = NULL; 17770a625fd2SDavid S. Miller 17780a625fd2SDavid S. Miller out_queue_cache_destroy: 17790a625fd2SDavid S. Miller queue_cache_destroy(); 17800a625fd2SDavid S. Miller 17810a625fd2SDavid S. Miller out_hvapi_release: 17820a625fd2SDavid S. Miller n2_spu_hvapi_unregister(); 17830a625fd2SDavid S. Miller goto out; 17840a625fd2SDavid S. Miller } 17850a625fd2SDavid S. Miller 17860a625fd2SDavid S. Miller static void release_global_resources(void) 17870a625fd2SDavid S. Miller { 17880a625fd2SDavid S. Miller mutex_lock(&spu_lock); 17890a625fd2SDavid S. Miller if (!--global_ref) { 17900a625fd2SDavid S. Miller kfree(cpu_to_cwq); 17910a625fd2SDavid S. Miller cpu_to_cwq = NULL; 17920a625fd2SDavid S. Miller 17930a625fd2SDavid S. Miller kfree(cpu_to_mau); 17940a625fd2SDavid S. Miller cpu_to_mau = NULL; 17950a625fd2SDavid S. Miller 17960a625fd2SDavid S. Miller queue_cache_destroy(); 17970a625fd2SDavid S. Miller n2_spu_hvapi_unregister(); 17980a625fd2SDavid S. Miller } 17990a625fd2SDavid S. Miller mutex_unlock(&spu_lock); 18000a625fd2SDavid S. Miller } 18010a625fd2SDavid S. Miller 18020a625fd2SDavid S. Miller static struct n2_crypto * __devinit alloc_n2cp(void) 18030a625fd2SDavid S. Miller { 18040a625fd2SDavid S. Miller struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL); 18050a625fd2SDavid S. Miller 18060a625fd2SDavid S. Miller if (np) 18070a625fd2SDavid S. Miller INIT_LIST_HEAD(&np->cwq_list); 18080a625fd2SDavid S. Miller 18090a625fd2SDavid S. Miller return np; 18100a625fd2SDavid S. Miller } 18110a625fd2SDavid S. Miller 18120a625fd2SDavid S. Miller static void free_n2cp(struct n2_crypto *np) 18130a625fd2SDavid S. Miller { 18140a625fd2SDavid S. Miller if (np->cwq_info.ino_table) { 18150a625fd2SDavid S. Miller kfree(np->cwq_info.ino_table); 18160a625fd2SDavid S. Miller np->cwq_info.ino_table = NULL; 18170a625fd2SDavid S. Miller } 18180a625fd2SDavid S. Miller 18190a625fd2SDavid S. Miller kfree(np); 18200a625fd2SDavid S. Miller } 18210a625fd2SDavid S. Miller 18220a625fd2SDavid S. Miller static void __devinit n2_spu_driver_version(void) 18230a625fd2SDavid S. Miller { 18240a625fd2SDavid S. Miller static int n2_spu_version_printed; 18250a625fd2SDavid S. Miller 18260a625fd2SDavid S. Miller if (n2_spu_version_printed++ == 0) 18270a625fd2SDavid S. Miller pr_info("%s", version); 18280a625fd2SDavid S. Miller } 18290a625fd2SDavid S. Miller 18300a625fd2SDavid S. Miller static int __devinit n2_crypto_probe(struct of_device *dev, 18310a625fd2SDavid S. Miller const struct of_device_id *match) 18320a625fd2SDavid S. Miller { 18330a625fd2SDavid S. Miller struct mdesc_handle *mdesc; 18340a625fd2SDavid S. Miller const char *full_name; 18350a625fd2SDavid S. Miller struct n2_crypto *np; 18360a625fd2SDavid S. Miller int err; 18370a625fd2SDavid S. Miller 18380a625fd2SDavid S. Miller n2_spu_driver_version(); 18390a625fd2SDavid S. Miller 1840ff6c7341SDavid S. Miller full_name = dev->dev.of_node->full_name; 18410a625fd2SDavid S. Miller pr_info("Found N2CP at %s\n", full_name); 18420a625fd2SDavid S. Miller 18430a625fd2SDavid S. Miller np = alloc_n2cp(); 18440a625fd2SDavid S. Miller if (!np) { 18450a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n", 18460a625fd2SDavid S. Miller full_name); 18470a625fd2SDavid S. Miller return -ENOMEM; 18480a625fd2SDavid S. Miller } 18490a625fd2SDavid S. Miller 18500a625fd2SDavid S. Miller err = grab_global_resources(); 18510a625fd2SDavid S. Miller if (err) { 18520a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to grab " 18530a625fd2SDavid S. Miller "global resources.\n", full_name); 18540a625fd2SDavid S. Miller goto out_free_n2cp; 18550a625fd2SDavid S. Miller } 18560a625fd2SDavid S. Miller 18570a625fd2SDavid S. Miller mdesc = mdesc_grab(); 18580a625fd2SDavid S. Miller 18590a625fd2SDavid S. Miller if (!mdesc) { 18600a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 18610a625fd2SDavid S. Miller full_name); 18620a625fd2SDavid S. Miller err = -ENODEV; 18630a625fd2SDavid S. Miller goto out_free_global; 18640a625fd2SDavid S. Miller } 18650a625fd2SDavid S. Miller err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); 18660a625fd2SDavid S. Miller if (err) { 18670a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 18680a625fd2SDavid S. Miller full_name); 18690a625fd2SDavid S. Miller mdesc_release(mdesc); 18700a625fd2SDavid S. Miller goto out_free_global; 18710a625fd2SDavid S. Miller } 18720a625fd2SDavid S. Miller 18730a625fd2SDavid S. Miller err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list, 18740a625fd2SDavid S. Miller "cwq", HV_NCS_QTYPE_CWQ, cwq_intr, 18750a625fd2SDavid S. Miller cpu_to_cwq); 18760a625fd2SDavid S. Miller mdesc_release(mdesc); 18770a625fd2SDavid S. Miller 18780a625fd2SDavid S. Miller if (err) { 18790a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n", 18800a625fd2SDavid S. Miller full_name); 18810a625fd2SDavid S. Miller goto out_free_global; 18820a625fd2SDavid S. Miller } 18830a625fd2SDavid S. Miller 18840a625fd2SDavid S. Miller err = n2_register_algs(); 18850a625fd2SDavid S. Miller if (err) { 18860a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to register algorithms.\n", 18870a625fd2SDavid S. Miller full_name); 18880a625fd2SDavid S. Miller goto out_free_spu_list; 18890a625fd2SDavid S. Miller } 18900a625fd2SDavid S. Miller 18910a625fd2SDavid S. Miller dev_set_drvdata(&dev->dev, np); 18920a625fd2SDavid S. Miller 18930a625fd2SDavid S. Miller return 0; 18940a625fd2SDavid S. Miller 18950a625fd2SDavid S. Miller out_free_spu_list: 18960a625fd2SDavid S. Miller spu_list_destroy(&np->cwq_list); 18970a625fd2SDavid S. Miller 18980a625fd2SDavid S. Miller out_free_global: 18990a625fd2SDavid S. Miller release_global_resources(); 19000a625fd2SDavid S. Miller 19010a625fd2SDavid S. Miller out_free_n2cp: 19020a625fd2SDavid S. Miller free_n2cp(np); 19030a625fd2SDavid S. Miller 19040a625fd2SDavid S. Miller return err; 19050a625fd2SDavid S. Miller } 19060a625fd2SDavid S. Miller 19070a625fd2SDavid S. Miller static int __devexit n2_crypto_remove(struct of_device *dev) 19080a625fd2SDavid S. Miller { 19090a625fd2SDavid S. Miller struct n2_crypto *np = dev_get_drvdata(&dev->dev); 19100a625fd2SDavid S. Miller 19110a625fd2SDavid S. Miller n2_unregister_algs(); 19120a625fd2SDavid S. Miller 19130a625fd2SDavid S. Miller spu_list_destroy(&np->cwq_list); 19140a625fd2SDavid S. Miller 19150a625fd2SDavid S. Miller release_global_resources(); 19160a625fd2SDavid S. Miller 19170a625fd2SDavid S. Miller free_n2cp(np); 19180a625fd2SDavid S. Miller 19190a625fd2SDavid S. Miller return 0; 19200a625fd2SDavid S. Miller } 19210a625fd2SDavid S. Miller 19220a625fd2SDavid S. Miller static struct n2_mau * __devinit alloc_ncp(void) 19230a625fd2SDavid S. Miller { 19240a625fd2SDavid S. Miller struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL); 19250a625fd2SDavid S. Miller 19260a625fd2SDavid S. Miller if (mp) 19270a625fd2SDavid S. Miller INIT_LIST_HEAD(&mp->mau_list); 19280a625fd2SDavid S. Miller 19290a625fd2SDavid S. Miller return mp; 19300a625fd2SDavid S. Miller } 19310a625fd2SDavid S. Miller 19320a625fd2SDavid S. Miller static void free_ncp(struct n2_mau *mp) 19330a625fd2SDavid S. Miller { 19340a625fd2SDavid S. Miller if (mp->mau_info.ino_table) { 19350a625fd2SDavid S. Miller kfree(mp->mau_info.ino_table); 19360a625fd2SDavid S. Miller mp->mau_info.ino_table = NULL; 19370a625fd2SDavid S. Miller } 19380a625fd2SDavid S. Miller 19390a625fd2SDavid S. Miller kfree(mp); 19400a625fd2SDavid S. Miller } 19410a625fd2SDavid S. Miller 19420a625fd2SDavid S. Miller static int __devinit n2_mau_probe(struct of_device *dev, 19430a625fd2SDavid S. Miller const struct of_device_id *match) 19440a625fd2SDavid S. Miller { 19450a625fd2SDavid S. Miller struct mdesc_handle *mdesc; 19460a625fd2SDavid S. Miller const char *full_name; 19470a625fd2SDavid S. Miller struct n2_mau *mp; 19480a625fd2SDavid S. Miller int err; 19490a625fd2SDavid S. Miller 19500a625fd2SDavid S. Miller n2_spu_driver_version(); 19510a625fd2SDavid S. Miller 1952ff6c7341SDavid S. Miller full_name = dev->dev.of_node->full_name; 19530a625fd2SDavid S. Miller pr_info("Found NCP at %s\n", full_name); 19540a625fd2SDavid S. Miller 19550a625fd2SDavid S. Miller mp = alloc_ncp(); 19560a625fd2SDavid S. Miller if (!mp) { 19570a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to allocate ncp.\n", 19580a625fd2SDavid S. Miller full_name); 19590a625fd2SDavid S. Miller return -ENOMEM; 19600a625fd2SDavid S. Miller } 19610a625fd2SDavid S. Miller 19620a625fd2SDavid S. Miller err = grab_global_resources(); 19630a625fd2SDavid S. Miller if (err) { 19640a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to grab " 19650a625fd2SDavid S. Miller "global resources.\n", full_name); 19660a625fd2SDavid S. Miller goto out_free_ncp; 19670a625fd2SDavid S. Miller } 19680a625fd2SDavid S. Miller 19690a625fd2SDavid S. Miller mdesc = mdesc_grab(); 19700a625fd2SDavid S. Miller 19710a625fd2SDavid S. Miller if (!mdesc) { 19720a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", 19730a625fd2SDavid S. Miller full_name); 19740a625fd2SDavid S. Miller err = -ENODEV; 19750a625fd2SDavid S. Miller goto out_free_global; 19760a625fd2SDavid S. Miller } 19770a625fd2SDavid S. Miller 19780a625fd2SDavid S. Miller err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); 19790a625fd2SDavid S. Miller if (err) { 19800a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", 19810a625fd2SDavid S. Miller full_name); 19820a625fd2SDavid S. Miller mdesc_release(mdesc); 19830a625fd2SDavid S. Miller goto out_free_global; 19840a625fd2SDavid S. Miller } 19850a625fd2SDavid S. Miller 19860a625fd2SDavid S. Miller err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list, 19870a625fd2SDavid S. Miller "mau", HV_NCS_QTYPE_MAU, mau_intr, 19880a625fd2SDavid S. Miller cpu_to_mau); 19890a625fd2SDavid S. Miller mdesc_release(mdesc); 19900a625fd2SDavid S. Miller 19910a625fd2SDavid S. Miller if (err) { 19920a625fd2SDavid S. Miller dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n", 19930a625fd2SDavid S. Miller full_name); 19940a625fd2SDavid S. Miller goto out_free_global; 19950a625fd2SDavid S. Miller } 19960a625fd2SDavid S. Miller 19970a625fd2SDavid S. Miller dev_set_drvdata(&dev->dev, mp); 19980a625fd2SDavid S. Miller 19990a625fd2SDavid S. Miller return 0; 20000a625fd2SDavid S. Miller 20010a625fd2SDavid S. Miller out_free_global: 20020a625fd2SDavid S. Miller release_global_resources(); 20030a625fd2SDavid S. Miller 20040a625fd2SDavid S. Miller out_free_ncp: 20050a625fd2SDavid S. Miller free_ncp(mp); 20060a625fd2SDavid S. Miller 20070a625fd2SDavid S. Miller return err; 20080a625fd2SDavid S. Miller } 20090a625fd2SDavid S. Miller 20100a625fd2SDavid S. Miller static int __devexit n2_mau_remove(struct of_device *dev) 20110a625fd2SDavid S. Miller { 20120a625fd2SDavid S. Miller struct n2_mau *mp = dev_get_drvdata(&dev->dev); 20130a625fd2SDavid S. Miller 20140a625fd2SDavid S. Miller spu_list_destroy(&mp->mau_list); 20150a625fd2SDavid S. Miller 20160a625fd2SDavid S. Miller release_global_resources(); 20170a625fd2SDavid S. Miller 20180a625fd2SDavid S. Miller free_ncp(mp); 20190a625fd2SDavid S. Miller 20200a625fd2SDavid S. Miller return 0; 20210a625fd2SDavid S. Miller } 20220a625fd2SDavid S. Miller 20230a625fd2SDavid S. Miller static struct of_device_id n2_crypto_match[] = { 20240a625fd2SDavid S. Miller { 20250a625fd2SDavid S. Miller .name = "n2cp", 20260a625fd2SDavid S. Miller .compatible = "SUNW,n2-cwq", 20270a625fd2SDavid S. Miller }, 20280a625fd2SDavid S. Miller { 20290a625fd2SDavid S. Miller .name = "n2cp", 20300a625fd2SDavid S. Miller .compatible = "SUNW,vf-cwq", 20310a625fd2SDavid S. Miller }, 20320a625fd2SDavid S. Miller {}, 20330a625fd2SDavid S. Miller }; 20340a625fd2SDavid S. Miller 20350a625fd2SDavid S. Miller MODULE_DEVICE_TABLE(of, n2_crypto_match); 20360a625fd2SDavid S. Miller 20370a625fd2SDavid S. Miller static struct of_platform_driver n2_crypto_driver = { 2038ff6c7341SDavid S. Miller .driver = { 20390a625fd2SDavid S. Miller .name = "n2cp", 2040ff6c7341SDavid S. Miller .owner = THIS_MODULE, 2041ff6c7341SDavid S. Miller .of_match_table = n2_crypto_match, 2042ff6c7341SDavid S. Miller }, 20430a625fd2SDavid S. Miller .probe = n2_crypto_probe, 20440a625fd2SDavid S. Miller .remove = __devexit_p(n2_crypto_remove), 20450a625fd2SDavid S. Miller }; 20460a625fd2SDavid S. Miller 20470a625fd2SDavid S. Miller static struct of_device_id n2_mau_match[] = { 20480a625fd2SDavid S. Miller { 20490a625fd2SDavid S. Miller .name = "ncp", 20500a625fd2SDavid S. Miller .compatible = "SUNW,n2-mau", 20510a625fd2SDavid S. Miller }, 20520a625fd2SDavid S. Miller { 20530a625fd2SDavid S. Miller .name = "ncp", 20540a625fd2SDavid S. Miller .compatible = "SUNW,vf-mau", 20550a625fd2SDavid S. Miller }, 20560a625fd2SDavid S. Miller {}, 20570a625fd2SDavid S. Miller }; 20580a625fd2SDavid S. Miller 20590a625fd2SDavid S. Miller MODULE_DEVICE_TABLE(of, n2_mau_match); 20600a625fd2SDavid S. Miller 20610a625fd2SDavid S. Miller static struct of_platform_driver n2_mau_driver = { 2062ff6c7341SDavid S. Miller .driver = { 20630a625fd2SDavid S. Miller .name = "ncp", 2064ff6c7341SDavid S. Miller .owner = THIS_MODULE, 2065ff6c7341SDavid S. Miller .of_match_table = n2_mau_match, 2066ff6c7341SDavid S. Miller }, 20670a625fd2SDavid S. Miller .probe = n2_mau_probe, 20680a625fd2SDavid S. Miller .remove = __devexit_p(n2_mau_remove), 20690a625fd2SDavid S. Miller }; 20700a625fd2SDavid S. Miller 20710a625fd2SDavid S. Miller static int __init n2_init(void) 20720a625fd2SDavid S. Miller { 20730a625fd2SDavid S. Miller int err = of_register_driver(&n2_crypto_driver, &of_bus_type); 20740a625fd2SDavid S. Miller 20750a625fd2SDavid S. Miller if (!err) { 20760a625fd2SDavid S. Miller err = of_register_driver(&n2_mau_driver, &of_bus_type); 20770a625fd2SDavid S. Miller if (err) 20780a625fd2SDavid S. Miller of_unregister_driver(&n2_crypto_driver); 20790a625fd2SDavid S. Miller } 20800a625fd2SDavid S. Miller return err; 20810a625fd2SDavid S. Miller } 20820a625fd2SDavid S. Miller 20830a625fd2SDavid S. Miller static void __exit n2_exit(void) 20840a625fd2SDavid S. Miller { 20850a625fd2SDavid S. Miller of_unregister_driver(&n2_mau_driver); 20860a625fd2SDavid S. Miller of_unregister_driver(&n2_crypto_driver); 20870a625fd2SDavid S. Miller } 20880a625fd2SDavid S. Miller 20890a625fd2SDavid S. Miller module_init(n2_init); 20900a625fd2SDavid S. Miller module_exit(n2_exit); 2091