1 /**
2  * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux.
3  *
4  * Copyright (C) 2011-2016 Chelsio Communications.  All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation.
9  *
10  * Written and Maintained by:
11  * Manoj Malviya (manojmalviya@chelsio.com)
12  * Atul Gupta (atul.gupta@chelsio.com)
13  * Jitendra Lulla (jlulla@chelsio.com)
14  * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
15  * Harsh Jain (harsh@chelsio.com)
16  */
17 
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/skbuff.h>
21 
22 #include <crypto/aes.h>
23 #include <crypto/hash.h>
24 
25 #include "t4_msg.h"
26 #include "chcr_core.h"
27 #include "cxgb4_uld.h"
28 
29 static LIST_HEAD(uld_ctx_list);
30 static DEFINE_MUTEX(dev_mutex);
31 static atomic_t dev_count;
32 static struct uld_ctx *ctx_rr;
33 
34 typedef int (*chcr_handler_func)(struct chcr_dev *dev, unsigned char *input);
35 static int cpl_fw6_pld_handler(struct chcr_dev *dev, unsigned char *input);
36 static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
37 static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
38 
39 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
40 	[CPL_FW6_PLD] = cpl_fw6_pld_handler,
41 };
42 
43 static struct cxgb4_uld_info chcr_uld_info = {
44 	.name = DRV_MODULE_NAME,
45 	.nrxq = MAX_ULD_QSETS,
46 	/* Max ntxq will be derived from fw config file*/
47 	.rxq_size = 1024,
48 	.add = chcr_uld_add,
49 	.state_change = chcr_uld_state_change,
50 	.rx_handler = chcr_uld_rx_handler,
51 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
52 	.tx_handler = chcr_uld_tx_handler,
53 #endif /* CONFIG_CHELSIO_IPSEC_INLINE */
54 };
55 
56 struct uld_ctx *assign_chcr_device(void)
57 {
58 	struct uld_ctx *u_ctx = NULL;
59 
60 	/*
61 	 * When multiple devices are present in system select
62 	 * device in round-robin fashion for crypto operations
63 	 * Although One session must use the same device to
64 	 * maintain request-response ordering.
65 	 */
66 	mutex_lock(&dev_mutex);
67 	if (!list_empty(&uld_ctx_list)) {
68 		u_ctx = ctx_rr;
69 		if (list_is_last(&ctx_rr->entry, &uld_ctx_list))
70 			ctx_rr = list_first_entry(&uld_ctx_list,
71 						  struct uld_ctx,
72 						  entry);
73 		else
74 			ctx_rr = list_next_entry(ctx_rr, entry);
75 	}
76 	mutex_unlock(&dev_mutex);
77 	return u_ctx;
78 }
79 
80 static int chcr_dev_add(struct uld_ctx *u_ctx)
81 {
82 	struct chcr_dev *dev;
83 
84 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
85 	if (!dev)
86 		return -ENXIO;
87 
88 	spin_lock_init(&dev->lock_chcr_dev);
89 	u_ctx->dev = dev;
90 	dev->u_ctx = u_ctx;
91 	atomic_inc(&dev_count);
92 	mutex_lock(&dev_mutex);
93 	list_add_tail(&u_ctx->entry, &uld_ctx_list);
94 	if (!ctx_rr)
95 		ctx_rr = u_ctx;
96 	mutex_unlock(&dev_mutex);
97 	return 0;
98 }
99 
100 static int chcr_dev_remove(struct uld_ctx *u_ctx)
101 {
102 	if (ctx_rr == u_ctx) {
103 		if (list_is_last(&ctx_rr->entry, &uld_ctx_list))
104 			ctx_rr = list_first_entry(&uld_ctx_list,
105 						  struct uld_ctx,
106 						  entry);
107 		else
108 			ctx_rr = list_next_entry(ctx_rr, entry);
109 	}
110 	list_del(&u_ctx->entry);
111 	if (list_empty(&uld_ctx_list))
112 		ctx_rr = NULL;
113 	kfree(u_ctx->dev);
114 	u_ctx->dev = NULL;
115 	atomic_dec(&dev_count);
116 	return 0;
117 }
118 
119 static int cpl_fw6_pld_handler(struct chcr_dev *dev,
120 			       unsigned char *input)
121 {
122 	struct crypto_async_request *req;
123 	struct cpl_fw6_pld *fw6_pld;
124 	u32 ack_err_status = 0;
125 	int error_status = 0;
126 	struct adapter *adap = padap(dev);
127 
128 	fw6_pld = (struct cpl_fw6_pld *)input;
129 	req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu(
130 						    fw6_pld->data[1]);
131 
132 	ack_err_status =
133 		ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4));
134 	if (ack_err_status) {
135 		if (CHK_MAC_ERR_BIT(ack_err_status) ||
136 		    CHK_PAD_ERR_BIT(ack_err_status))
137 			error_status = -EBADMSG;
138 		atomic_inc(&adap->chcr_stats.error);
139 	}
140 	/* call completion callback with failure status */
141 	if (req) {
142 		error_status = chcr_handle_resp(req, input, error_status);
143 	} else {
144 		pr_err("Incorrect request address from the firmware\n");
145 		return -EFAULT;
146 	}
147 	return 0;
148 }
149 
150 int chcr_send_wr(struct sk_buff *skb)
151 {
152 	return cxgb4_crypto_send(skb->dev, skb);
153 }
154 
155 static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
156 {
157 	struct uld_ctx *u_ctx;
158 
159 	/* Create the device and add it in the device list */
160 	if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE))
161 		return ERR_PTR(-EOPNOTSUPP);
162 
163 	/* Create the device and add it in the device list */
164 	u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
165 	if (!u_ctx) {
166 		u_ctx = ERR_PTR(-ENOMEM);
167 		goto out;
168 	}
169 	u_ctx->lldi = *lld;
170 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
171 	if (lld->crypto & ULP_CRYPTO_IPSEC_INLINE)
172 		chcr_add_xfrmops(lld);
173 #endif /* CONFIG_CHELSIO_IPSEC_INLINE */
174 out:
175 	return u_ctx;
176 }
177 
178 int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
179 			const struct pkt_gl *pgl)
180 {
181 	struct uld_ctx *u_ctx = (struct uld_ctx *)handle;
182 	struct chcr_dev *dev = u_ctx->dev;
183 	const struct cpl_fw6_pld *rpl = (struct cpl_fw6_pld *)rsp;
184 
185 	if (rpl->opcode != CPL_FW6_PLD) {
186 		pr_err("Unsupported opcode\n");
187 		return 0;
188 	}
189 
190 	if (!pgl)
191 		work_handlers[rpl->opcode](dev, (unsigned char *)&rsp[1]);
192 	else
193 		work_handlers[rpl->opcode](dev, pgl->va);
194 	return 0;
195 }
196 
197 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
198 int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev)
199 {
200 	return chcr_ipsec_xmit(skb, dev);
201 }
202 #endif /* CONFIG_CHELSIO_IPSEC_INLINE */
203 
204 static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
205 {
206 	struct uld_ctx *u_ctx = handle;
207 	int ret = 0;
208 
209 	switch (state) {
210 	case CXGB4_STATE_UP:
211 		if (!u_ctx->dev) {
212 			ret = chcr_dev_add(u_ctx);
213 			if (ret != 0)
214 				return ret;
215 		}
216 		if (atomic_read(&dev_count) == 1)
217 			ret = start_crypto();
218 		break;
219 
220 	case CXGB4_STATE_DETACH:
221 		if (u_ctx->dev) {
222 			mutex_lock(&dev_mutex);
223 			chcr_dev_remove(u_ctx);
224 			mutex_unlock(&dev_mutex);
225 		}
226 		if (!atomic_read(&dev_count))
227 			stop_crypto();
228 		break;
229 
230 	case CXGB4_STATE_START_RECOVERY:
231 	case CXGB4_STATE_DOWN:
232 	default:
233 		break;
234 	}
235 	return ret;
236 }
237 
238 static int __init chcr_crypto_init(void)
239 {
240 	cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info);
241 	return 0;
242 }
243 
244 static void __exit chcr_crypto_exit(void)
245 {
246 	struct uld_ctx *u_ctx, *tmp;
247 
248 	if (atomic_read(&dev_count))
249 		stop_crypto();
250 
251 	/* Remove all devices from list */
252 	mutex_lock(&dev_mutex);
253 	list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
254 		if (u_ctx->dev)
255 			chcr_dev_remove(u_ctx);
256 		kfree(u_ctx);
257 	}
258 	mutex_unlock(&dev_mutex);
259 	cxgb4_unregister_uld(CXGB4_ULD_CRYPTO);
260 }
261 
262 module_init(chcr_crypto_init);
263 module_exit(chcr_crypto_exit);
264 
265 MODULE_DESCRIPTION("Crypto Co-processor for Chelsio Terminator cards.");
266 MODULE_LICENSE("GPL");
267 MODULE_AUTHOR("Chelsio Communications");
268 MODULE_VERSION(DRV_VERSION);
269