1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/cpumask.h>
3 #include <linux/dma-mapping.h>
4 #include <linux/dmapool.h>
5 #include <linux/delay.h>
6 #include <linux/gfp.h>
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/pci_regs.h>
10 #include <linux/vmalloc.h>
11 #include <linux/pci.h>
12 
13 #include "nitrox_dev.h"
14 #include "nitrox_common.h"
15 #include "nitrox_req.h"
16 #include "nitrox_csr.h"
17 
18 #define CRYPTO_CTX_SIZE	256
19 
20 /* packet inuput ring alignments */
21 #define PKTIN_Q_ALIGN_BYTES 16
22 /* AQM Queue input alignments */
23 #define AQM_Q_ALIGN_BYTES 32
24 
25 static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
26 {
27 	struct nitrox_device *ndev = cmdq->ndev;
28 
29 	cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes;
30 	cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize,
31 						&cmdq->unalign_dma,
32 						GFP_KERNEL);
33 	if (!cmdq->unalign_base)
34 		return -ENOMEM;
35 
36 	cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes);
37 	cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma);
38 	cmdq->write_idx = 0;
39 
40 	spin_lock_init(&cmdq->cmd_qlock);
41 	spin_lock_init(&cmdq->resp_qlock);
42 	spin_lock_init(&cmdq->backlog_qlock);
43 
44 	INIT_LIST_HEAD(&cmdq->response_head);
45 	INIT_LIST_HEAD(&cmdq->backlog_head);
46 	INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
47 
48 	atomic_set(&cmdq->pending_count, 0);
49 	atomic_set(&cmdq->backlog_count, 0);
50 	return 0;
51 }
52 
53 static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
54 {
55 	cmdq->write_idx = 0;
56 	atomic_set(&cmdq->pending_count, 0);
57 	atomic_set(&cmdq->backlog_count, 0);
58 }
59 
60 static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
61 {
62 	struct nitrox_device *ndev;
63 
64 	if (!cmdq)
65 		return;
66 
67 	if (!cmdq->unalign_base)
68 		return;
69 
70 	ndev = cmdq->ndev;
71 	cancel_work_sync(&cmdq->backlog_qflush);
72 
73 	dma_free_coherent(DEV(ndev), cmdq->qsize,
74 			  cmdq->unalign_base, cmdq->unalign_dma);
75 	nitrox_cmdq_reset(cmdq);
76 
77 	cmdq->dbell_csr_addr = NULL;
78 	cmdq->compl_cnt_csr_addr = NULL;
79 	cmdq->unalign_base = NULL;
80 	cmdq->base = NULL;
81 	cmdq->unalign_dma = 0;
82 	cmdq->dma = 0;
83 	cmdq->qsize = 0;
84 	cmdq->instr_size = 0;
85 }
86 
87 static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
88 {
89 	int i;
90 
91 	for (i = 0; i < ndev->nr_queues; i++) {
92 		nitrox_cmdq_cleanup(ndev->aqmq[i]);
93 		kfree_sensitive(ndev->aqmq[i]);
94 		ndev->aqmq[i] = NULL;
95 	}
96 }
97 
98 static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
99 {
100 	int i, err;
101 
102 	for (i = 0; i < ndev->nr_queues; i++) {
103 		struct nitrox_cmdq *cmdq;
104 		u64 offset;
105 
106 		cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node);
107 		if (!cmdq) {
108 			err = -ENOMEM;
109 			goto aqmq_fail;
110 		}
111 
112 		cmdq->ndev = ndev;
113 		cmdq->qno = i;
114 		cmdq->instr_size = sizeof(struct aqmq_command_s);
115 
116 		/* AQM Queue Doorbell Counter Register Address */
117 		offset = AQMQ_DRBLX(i);
118 		cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
119 		/* AQM Queue Commands Completed Count Register Address */
120 		offset = AQMQ_CMD_CNTX(i);
121 		cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
122 
123 		err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
124 		if (err) {
125 			kfree_sensitive(cmdq);
126 			goto aqmq_fail;
127 		}
128 		ndev->aqmq[i] = cmdq;
129 	}
130 
131 	return 0;
132 
133 aqmq_fail:
134 	nitrox_free_aqm_queues(ndev);
135 	return err;
136 }
137 
138 static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
139 {
140 	int i;
141 
142 	for (i = 0; i < ndev->nr_queues; i++) {
143 		struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
144 
145 		nitrox_cmdq_cleanup(cmdq);
146 	}
147 	kfree(ndev->pkt_inq);
148 	ndev->pkt_inq = NULL;
149 }
150 
151 static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev)
152 {
153 	int i, err;
154 
155 	ndev->pkt_inq = kcalloc_node(ndev->nr_queues,
156 				     sizeof(struct nitrox_cmdq),
157 				     GFP_KERNEL, ndev->node);
158 	if (!ndev->pkt_inq)
159 		return -ENOMEM;
160 
161 	for (i = 0; i < ndev->nr_queues; i++) {
162 		struct nitrox_cmdq *cmdq;
163 		u64 offset;
164 
165 		cmdq = &ndev->pkt_inq[i];
166 		cmdq->ndev = ndev;
167 		cmdq->qno = i;
168 		cmdq->instr_size = sizeof(struct nps_pkt_instr);
169 
170 		/* packet input ring doorbell address */
171 		offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
172 		cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
173 		/* packet solicit port completion count address */
174 		offset = NPS_PKT_SLC_CNTSX(i);
175 		cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
176 
177 		err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES);
178 		if (err)
179 			goto pktq_fail;
180 	}
181 	return 0;
182 
183 pktq_fail:
184 	nitrox_free_pktin_queues(ndev);
185 	return err;
186 }
187 
188 static int create_crypto_dma_pool(struct nitrox_device *ndev)
189 {
190 	size_t size;
191 
192 	/* Crypto context pool, 16 byte aligned */
193 	size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
194 	ndev->ctx_pool = dma_pool_create("nitrox-context",
195 					 DEV(ndev), size, 16, 0);
196 	if (!ndev->ctx_pool)
197 		return -ENOMEM;
198 
199 	return 0;
200 }
201 
202 static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
203 {
204 	if (!ndev->ctx_pool)
205 		return;
206 
207 	dma_pool_destroy(ndev->ctx_pool);
208 	ndev->ctx_pool = NULL;
209 }
210 
211 /*
212  * crypto_alloc_context - Allocate crypto context from pool
213  * @ndev: NITROX Device
214  */
215 void *crypto_alloc_context(struct nitrox_device *ndev)
216 {
217 	struct ctx_hdr *ctx;
218 	struct crypto_ctx_hdr *chdr;
219 	void *vaddr;
220 	dma_addr_t dma;
221 
222 	chdr = kmalloc(sizeof(*chdr), GFP_KERNEL);
223 	if (!chdr)
224 		return NULL;
225 
226 	vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma);
227 	if (!vaddr) {
228 		kfree(chdr);
229 		return NULL;
230 	}
231 
232 	/* fill meta data */
233 	ctx = vaddr;
234 	ctx->pool = ndev->ctx_pool;
235 	ctx->dma = dma;
236 	ctx->ctx_dma = dma + sizeof(struct ctx_hdr);
237 
238 	chdr->pool = ndev->ctx_pool;
239 	chdr->dma = dma;
240 	chdr->vaddr = vaddr;
241 
242 	return chdr;
243 }
244 
245 /**
246  * crypto_free_context - Free crypto context to pool
247  * @ctx: context to free
248  */
249 void crypto_free_context(void *ctx)
250 {
251 	struct crypto_ctx_hdr *ctxp;
252 
253 	if (!ctx)
254 		return;
255 
256 	ctxp = ctx;
257 	dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma);
258 	kfree(ctxp);
259 }
260 
261 /**
262  * nitrox_common_sw_init - allocate software resources.
263  * @ndev: NITROX device
264  *
265  * Allocates crypto context pools and command queues etc.
266  *
267  * Return: 0 on success, or a negative error code on error.
268  */
269 int nitrox_common_sw_init(struct nitrox_device *ndev)
270 {
271 	int err = 0;
272 
273 	/* per device crypto context pool */
274 	err = create_crypto_dma_pool(ndev);
275 	if (err)
276 		return err;
277 
278 	err = nitrox_alloc_pktin_queues(ndev);
279 	if (err)
280 		destroy_crypto_dma_pool(ndev);
281 
282 	err = nitrox_alloc_aqm_queues(ndev);
283 	if (err) {
284 		nitrox_free_pktin_queues(ndev);
285 		destroy_crypto_dma_pool(ndev);
286 	}
287 
288 	return err;
289 }
290 
291 /**
292  * nitrox_common_sw_cleanup - free software resources.
293  * @ndev: NITROX device
294  */
295 void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
296 {
297 	nitrox_free_aqm_queues(ndev);
298 	nitrox_free_pktin_queues(ndev);
299 	destroy_crypto_dma_pool(ndev);
300 }
301