129e5b878SLee Jones /*
2324429d7SHariprasad Shenai * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux.
3324429d7SHariprasad Shenai *
4324429d7SHariprasad Shenai * Copyright (C) 2011-2016 Chelsio Communications. All rights reserved.
5324429d7SHariprasad Shenai *
6324429d7SHariprasad Shenai * This program is free software; you can redistribute it and/or modify
7324429d7SHariprasad Shenai * it under the terms of the GNU General Public License as published by
8324429d7SHariprasad Shenai * the Free Software Foundation.
9324429d7SHariprasad Shenai *
10324429d7SHariprasad Shenai * Written and Maintained by:
11324429d7SHariprasad Shenai * Manoj Malviya (manojmalviya@chelsio.com)
12324429d7SHariprasad Shenai * Atul Gupta (atul.gupta@chelsio.com)
13324429d7SHariprasad Shenai * Jitendra Lulla (jlulla@chelsio.com)
14324429d7SHariprasad Shenai * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
15324429d7SHariprasad Shenai * Harsh Jain (harsh@chelsio.com)
16324429d7SHariprasad Shenai */
17324429d7SHariprasad Shenai
18324429d7SHariprasad Shenai #include <linux/kernel.h>
19324429d7SHariprasad Shenai #include <linux/module.h>
20324429d7SHariprasad Shenai #include <linux/skbuff.h>
21324429d7SHariprasad Shenai
22324429d7SHariprasad Shenai #include <crypto/aes.h>
23324429d7SHariprasad Shenai #include <crypto/hash.h>
24324429d7SHariprasad Shenai
25324429d7SHariprasad Shenai #include "t4_msg.h"
26324429d7SHariprasad Shenai #include "chcr_core.h"
27324429d7SHariprasad Shenai #include "cxgb4_uld.h"
28324429d7SHariprasad Shenai
29fef4912bSHarsh Jain static struct chcr_driver_data drv_data;
30324429d7SHariprasad Shenai
318a30923eSRohit Maheshwari typedef int (*chcr_handler_func)(struct adapter *adap, unsigned char *input);
328a30923eSRohit Maheshwari static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input);
33324429d7SHariprasad Shenai static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
34324429d7SHariprasad Shenai static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
35324429d7SHariprasad Shenai
36324429d7SHariprasad Shenai static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
37324429d7SHariprasad Shenai [CPL_FW6_PLD] = cpl_fw6_pld_handler,
38324429d7SHariprasad Shenai };
39324429d7SHariprasad Shenai
400fbc81b3SHariprasad Shenai static struct cxgb4_uld_info chcr_uld_info = {
41324429d7SHariprasad Shenai .name = DRV_MODULE_NAME,
420fbc81b3SHariprasad Shenai .nrxq = MAX_ULD_QSETS,
43a1c6fd43SHarsh Jain /* Max ntxq will be derived from fw config file*/
44324429d7SHariprasad Shenai .rxq_size = 1024,
45324429d7SHariprasad Shenai .add = chcr_uld_add,
46324429d7SHariprasad Shenai .state_change = chcr_uld_state_change,
47324429d7SHariprasad Shenai .rx_handler = chcr_uld_rx_handler,
48324429d7SHariprasad Shenai };
49324429d7SHariprasad Shenai
detach_work_fn(struct work_struct * work)50fef4912bSHarsh Jain static void detach_work_fn(struct work_struct *work)
51fef4912bSHarsh Jain {
52fef4912bSHarsh Jain struct chcr_dev *dev;
53fef4912bSHarsh Jain
54fef4912bSHarsh Jain dev = container_of(work, struct chcr_dev, detach_work.work);
55fef4912bSHarsh Jain
56fef4912bSHarsh Jain if (atomic_read(&dev->inflight)) {
57fef4912bSHarsh Jain dev->wqretry--;
58fef4912bSHarsh Jain if (dev->wqretry) {
59fef4912bSHarsh Jain pr_debug("Request Inflight Count %d\n",
60fef4912bSHarsh Jain atomic_read(&dev->inflight));
61fef4912bSHarsh Jain
62fef4912bSHarsh Jain schedule_delayed_work(&dev->detach_work, WQ_DETACH_TM);
63fef4912bSHarsh Jain } else {
64fef4912bSHarsh Jain WARN(1, "CHCR:%d request Still Pending\n",
65fef4912bSHarsh Jain atomic_read(&dev->inflight));
66fef4912bSHarsh Jain complete(&dev->detach_comp);
67fef4912bSHarsh Jain }
68fef4912bSHarsh Jain } else {
69fef4912bSHarsh Jain complete(&dev->detach_comp);
70fef4912bSHarsh Jain }
71fef4912bSHarsh Jain }
72fef4912bSHarsh Jain
assign_chcr_device(void)7314c19b17SHarsh Jain struct uld_ctx *assign_chcr_device(void)
74324429d7SHariprasad Shenai {
7514c19b17SHarsh Jain struct uld_ctx *u_ctx = NULL;
76324429d7SHariprasad Shenai
77324429d7SHariprasad Shenai /*
7814c19b17SHarsh Jain * When multiple devices are present in system select
7914c19b17SHarsh Jain * device in round-robin fashion for crypto operations
8014c19b17SHarsh Jain * Although One session must use the same device to
8114c19b17SHarsh Jain * maintain request-response ordering.
82324429d7SHariprasad Shenai */
83fef4912bSHarsh Jain mutex_lock(&drv_data.drv_mutex);
84fef4912bSHarsh Jain if (!list_empty(&drv_data.act_dev)) {
85fef4912bSHarsh Jain u_ctx = drv_data.last_dev;
86fef4912bSHarsh Jain if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev))
87fef4912bSHarsh Jain drv_data.last_dev = list_first_entry(&drv_data.act_dev,
88fef4912bSHarsh Jain struct uld_ctx, entry);
8914c19b17SHarsh Jain else
90fef4912bSHarsh Jain drv_data.last_dev =
91fef4912bSHarsh Jain list_next_entry(drv_data.last_dev, entry);
92f5f7bebcSHarsh Jain }
93fef4912bSHarsh Jain mutex_unlock(&drv_data.drv_mutex);
9414c19b17SHarsh Jain return u_ctx;
95324429d7SHariprasad Shenai }
96324429d7SHariprasad Shenai
chcr_dev_add(struct uld_ctx * u_ctx)97fef4912bSHarsh Jain static void chcr_dev_add(struct uld_ctx *u_ctx)
98324429d7SHariprasad Shenai {
99324429d7SHariprasad Shenai struct chcr_dev *dev;
100324429d7SHariprasad Shenai
101fef4912bSHarsh Jain dev = &u_ctx->dev;
102fef4912bSHarsh Jain dev->state = CHCR_ATTACH;
103fef4912bSHarsh Jain atomic_set(&dev->inflight, 0);
104fef4912bSHarsh Jain mutex_lock(&drv_data.drv_mutex);
105fef4912bSHarsh Jain list_move(&u_ctx->entry, &drv_data.act_dev);
106fef4912bSHarsh Jain if (!drv_data.last_dev)
107fef4912bSHarsh Jain drv_data.last_dev = u_ctx;
108fef4912bSHarsh Jain mutex_unlock(&drv_data.drv_mutex);
109324429d7SHariprasad Shenai }
110324429d7SHariprasad Shenai
chcr_dev_init(struct uld_ctx * u_ctx)111fef4912bSHarsh Jain static void chcr_dev_init(struct uld_ctx *u_ctx)
112324429d7SHariprasad Shenai {
113fef4912bSHarsh Jain struct chcr_dev *dev;
114fef4912bSHarsh Jain
115fef4912bSHarsh Jain dev = &u_ctx->dev;
116fef4912bSHarsh Jain spin_lock_init(&dev->lock_chcr_dev);
117fef4912bSHarsh Jain INIT_DELAYED_WORK(&dev->detach_work, detach_work_fn);
118fef4912bSHarsh Jain init_completion(&dev->detach_comp);
119fef4912bSHarsh Jain dev->state = CHCR_INIT;
120fef4912bSHarsh Jain dev->wqretry = WQ_RETRY;
121fef4912bSHarsh Jain atomic_inc(&drv_data.dev_count);
122fef4912bSHarsh Jain atomic_set(&dev->inflight, 0);
123fef4912bSHarsh Jain mutex_lock(&drv_data.drv_mutex);
124fef4912bSHarsh Jain list_add_tail(&u_ctx->entry, &drv_data.inact_dev);
125fef4912bSHarsh Jain mutex_unlock(&drv_data.drv_mutex);
12614c19b17SHarsh Jain }
127fef4912bSHarsh Jain
chcr_dev_move(struct uld_ctx * u_ctx)128fef4912bSHarsh Jain static int chcr_dev_move(struct uld_ctx *u_ctx)
129fef4912bSHarsh Jain {
130fef4912bSHarsh Jain mutex_lock(&drv_data.drv_mutex);
131fef4912bSHarsh Jain if (drv_data.last_dev == u_ctx) {
132fef4912bSHarsh Jain if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev))
133fef4912bSHarsh Jain drv_data.last_dev = list_first_entry(&drv_data.act_dev,
134fef4912bSHarsh Jain struct uld_ctx, entry);
135fef4912bSHarsh Jain else
136fef4912bSHarsh Jain drv_data.last_dev =
137fef4912bSHarsh Jain list_next_entry(drv_data.last_dev, entry);
138fef4912bSHarsh Jain }
139fef4912bSHarsh Jain list_move(&u_ctx->entry, &drv_data.inact_dev);
140fef4912bSHarsh Jain if (list_empty(&drv_data.act_dev))
141fef4912bSHarsh Jain drv_data.last_dev = NULL;
142fef4912bSHarsh Jain atomic_dec(&drv_data.dev_count);
143fef4912bSHarsh Jain mutex_unlock(&drv_data.drv_mutex);
144fef4912bSHarsh Jain
145324429d7SHariprasad Shenai return 0;
146324429d7SHariprasad Shenai }
147324429d7SHariprasad Shenai
cpl_fw6_pld_handler(struct adapter * adap,unsigned char * input)1488a30923eSRohit Maheshwari static int cpl_fw6_pld_handler(struct adapter *adap,
149324429d7SHariprasad Shenai unsigned char *input)
150324429d7SHariprasad Shenai {
151324429d7SHariprasad Shenai struct crypto_async_request *req;
152324429d7SHariprasad Shenai struct cpl_fw6_pld *fw6_pld;
153324429d7SHariprasad Shenai u32 ack_err_status = 0;
154324429d7SHariprasad Shenai int error_status = 0;
155324429d7SHariprasad Shenai
156324429d7SHariprasad Shenai fw6_pld = (struct cpl_fw6_pld *)input;
157324429d7SHariprasad Shenai req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu(
158324429d7SHariprasad Shenai fw6_pld->data[1]);
159324429d7SHariprasad Shenai
160324429d7SHariprasad Shenai ack_err_status =
161324429d7SHariprasad Shenai ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4));
162f31ba0f9SHarsh Jain if (CHK_MAC_ERR_BIT(ack_err_status) || CHK_PAD_ERR_BIT(ack_err_status))
1632debd332SHarsh Jain error_status = -EBADMSG;
164324429d7SHariprasad Shenai /* call completion callback with failure status */
165324429d7SHariprasad Shenai if (req) {
1662debd332SHarsh Jain error_status = chcr_handle_resp(req, input, error_status);
167324429d7SHariprasad Shenai } else {
168324429d7SHariprasad Shenai pr_err("Incorrect request address from the firmware\n");
169324429d7SHariprasad Shenai return -EFAULT;
170324429d7SHariprasad Shenai }
171f31ba0f9SHarsh Jain if (error_status)
172f31ba0f9SHarsh Jain atomic_inc(&adap->chcr_stats.error);
173f31ba0f9SHarsh Jain
174324429d7SHariprasad Shenai return 0;
175324429d7SHariprasad Shenai }
176324429d7SHariprasad Shenai
chcr_send_wr(struct sk_buff * skb)177324429d7SHariprasad Shenai int chcr_send_wr(struct sk_buff *skb)
178324429d7SHariprasad Shenai {
179ab677ff4SHariprasad Shenai return cxgb4_crypto_send(skb->dev, skb);
180324429d7SHariprasad Shenai }
181324429d7SHariprasad Shenai
chcr_uld_add(const struct cxgb4_lld_info * lld)182324429d7SHariprasad Shenai static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
183324429d7SHariprasad Shenai {
184324429d7SHariprasad Shenai struct uld_ctx *u_ctx;
185324429d7SHariprasad Shenai
186324429d7SHariprasad Shenai /* Create the device and add it in the device list */
187*66810912SVinay Kumar Yadav pr_info_once("%s\n", DRV_DESC);
188396d34f9SHarsh Jain if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE))
189396d34f9SHarsh Jain return ERR_PTR(-EOPNOTSUPP);
190396d34f9SHarsh Jain
191396d34f9SHarsh Jain /* Create the device and add it in the device list */
192324429d7SHariprasad Shenai u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
193324429d7SHariprasad Shenai if (!u_ctx) {
194324429d7SHariprasad Shenai u_ctx = ERR_PTR(-ENOMEM);
195324429d7SHariprasad Shenai goto out;
196324429d7SHariprasad Shenai }
197324429d7SHariprasad Shenai u_ctx->lldi = *lld;
198fef4912bSHarsh Jain chcr_dev_init(u_ctx);
199324429d7SHariprasad Shenai out:
200324429d7SHariprasad Shenai return u_ctx;
201324429d7SHariprasad Shenai }
202324429d7SHariprasad Shenai
chcr_uld_rx_handler(void * handle,const __be64 * rsp,const struct pkt_gl * pgl)203324429d7SHariprasad Shenai int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
204324429d7SHariprasad Shenai const struct pkt_gl *pgl)
205324429d7SHariprasad Shenai {
206324429d7SHariprasad Shenai struct uld_ctx *u_ctx = (struct uld_ctx *)handle;
207fef4912bSHarsh Jain struct chcr_dev *dev = &u_ctx->dev;
2088a30923eSRohit Maheshwari struct adapter *adap = padap(dev);
209d2826056SHarsh Jain const struct cpl_fw6_pld *rpl = (struct cpl_fw6_pld *)rsp;
210324429d7SHariprasad Shenai
2118a30923eSRohit Maheshwari if (!work_handlers[rpl->opcode]) {
2128a30923eSRohit Maheshwari pr_err("Unsupported opcode %d received\n", rpl->opcode);
213324429d7SHariprasad Shenai return 0;
214324429d7SHariprasad Shenai }
215324429d7SHariprasad Shenai
216324429d7SHariprasad Shenai if (!pgl)
2178a30923eSRohit Maheshwari work_handlers[rpl->opcode](adap, (unsigned char *)&rsp[1]);
218324429d7SHariprasad Shenai else
2198a30923eSRohit Maheshwari work_handlers[rpl->opcode](adap, pgl->va);
220324429d7SHariprasad Shenai return 0;
221324429d7SHariprasad Shenai }
222324429d7SHariprasad Shenai
chcr_detach_device(struct uld_ctx * u_ctx)223fef4912bSHarsh Jain static void chcr_detach_device(struct uld_ctx *u_ctx)
224fef4912bSHarsh Jain {
225fef4912bSHarsh Jain struct chcr_dev *dev = &u_ctx->dev;
226fef4912bSHarsh Jain
227fef4912bSHarsh Jain if (dev->state == CHCR_DETACH) {
228fef4912bSHarsh Jain pr_debug("Detached Event received for already detach device\n");
229fef4912bSHarsh Jain return;
230fef4912bSHarsh Jain }
231fef4912bSHarsh Jain dev->state = CHCR_DETACH;
232fef4912bSHarsh Jain if (atomic_read(&dev->inflight) != 0) {
233fef4912bSHarsh Jain schedule_delayed_work(&dev->detach_work, WQ_DETACH_TM);
234fef4912bSHarsh Jain wait_for_completion(&dev->detach_comp);
235fef4912bSHarsh Jain }
236fef4912bSHarsh Jain
237fef4912bSHarsh Jain // Move u_ctx to inactive_dev list
238fef4912bSHarsh Jain chcr_dev_move(u_ctx);
239fef4912bSHarsh Jain }
240fef4912bSHarsh Jain
chcr_uld_state_change(void * handle,enum cxgb4_state state)241324429d7SHariprasad Shenai static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
242324429d7SHariprasad Shenai {
243324429d7SHariprasad Shenai struct uld_ctx *u_ctx = handle;
244324429d7SHariprasad Shenai int ret = 0;
245324429d7SHariprasad Shenai
246324429d7SHariprasad Shenai switch (state) {
247324429d7SHariprasad Shenai case CXGB4_STATE_UP:
248fef4912bSHarsh Jain if (u_ctx->dev.state != CHCR_INIT) {
249fef4912bSHarsh Jain // ALready Initialised.
250fef4912bSHarsh Jain return 0;
251324429d7SHariprasad Shenai }
252fef4912bSHarsh Jain chcr_dev_add(u_ctx);
253324429d7SHariprasad Shenai ret = start_crypto();
254324429d7SHariprasad Shenai break;
255324429d7SHariprasad Shenai
256324429d7SHariprasad Shenai case CXGB4_STATE_DETACH:
257fef4912bSHarsh Jain chcr_detach_device(u_ctx);
2586ff78ffaSDevulapally Shiva Krishna if (!atomic_read(&drv_data.dev_count))
2596ff78ffaSDevulapally Shiva Krishna stop_crypto();
260324429d7SHariprasad Shenai break;
261324429d7SHariprasad Shenai
262324429d7SHariprasad Shenai case CXGB4_STATE_START_RECOVERY:
263324429d7SHariprasad Shenai case CXGB4_STATE_DOWN:
264324429d7SHariprasad Shenai default:
265324429d7SHariprasad Shenai break;
266324429d7SHariprasad Shenai }
267324429d7SHariprasad Shenai return ret;
268324429d7SHariprasad Shenai }
269324429d7SHariprasad Shenai
chcr_crypto_init(void)270324429d7SHariprasad Shenai static int __init chcr_crypto_init(void)
271324429d7SHariprasad Shenai {
272fef4912bSHarsh Jain INIT_LIST_HEAD(&drv_data.act_dev);
273fef4912bSHarsh Jain INIT_LIST_HEAD(&drv_data.inact_dev);
274fef4912bSHarsh Jain atomic_set(&drv_data.dev_count, 0);
275fef4912bSHarsh Jain mutex_init(&drv_data.drv_mutex);
276fef4912bSHarsh Jain drv_data.last_dev = NULL;
27740b06553SGanesh Goudar cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info);
278fef4912bSHarsh Jain
279324429d7SHariprasad Shenai return 0;
280324429d7SHariprasad Shenai }
281324429d7SHariprasad Shenai
chcr_crypto_exit(void)282324429d7SHariprasad Shenai static void __exit chcr_crypto_exit(void)
283324429d7SHariprasad Shenai {
284324429d7SHariprasad Shenai struct uld_ctx *u_ctx, *tmp;
285c0271a05SAyush Sawal struct adapter *adap;
286324429d7SHariprasad Shenai
287324429d7SHariprasad Shenai stop_crypto();
288fef4912bSHarsh Jain cxgb4_unregister_uld(CXGB4_ULD_CRYPTO);
289324429d7SHariprasad Shenai /* Remove all devices from list */
290fef4912bSHarsh Jain mutex_lock(&drv_data.drv_mutex);
291fef4912bSHarsh Jain list_for_each_entry_safe(u_ctx, tmp, &drv_data.act_dev, entry) {
292c0271a05SAyush Sawal adap = padap(&u_ctx->dev);
293c0271a05SAyush Sawal memset(&adap->chcr_stats, 0, sizeof(adap->chcr_stats));
294fef4912bSHarsh Jain list_del(&u_ctx->entry);
295324429d7SHariprasad Shenai kfree(u_ctx);
296324429d7SHariprasad Shenai }
297fef4912bSHarsh Jain list_for_each_entry_safe(u_ctx, tmp, &drv_data.inact_dev, entry) {
298c0271a05SAyush Sawal adap = padap(&u_ctx->dev);
299c0271a05SAyush Sawal memset(&adap->chcr_stats, 0, sizeof(adap->chcr_stats));
300fef4912bSHarsh Jain list_del(&u_ctx->entry);
301fef4912bSHarsh Jain kfree(u_ctx);
302fef4912bSHarsh Jain }
303fef4912bSHarsh Jain mutex_unlock(&drv_data.drv_mutex);
304324429d7SHariprasad Shenai }
305324429d7SHariprasad Shenai
306324429d7SHariprasad Shenai module_init(chcr_crypto_init);
307324429d7SHariprasad Shenai module_exit(chcr_crypto_exit);
308324429d7SHariprasad Shenai
309324429d7SHariprasad Shenai MODULE_DESCRIPTION("Crypto Co-processor for Chelsio Terminator cards.");
310324429d7SHariprasad Shenai MODULE_LICENSE("GPL");
311324429d7SHariprasad Shenai MODULE_AUTHOR("Chelsio Communications");
312