1 /* SPDX-License-Identifier: GPL-2.0-only
2  * Copyright (C) 2020 Marvell.
3  */
4 #ifndef __OTX2_CPTLF_H
5 #define __OTX2_CPTLF_H
6 
7 #include <linux/soc/marvell/octeontx2/asm.h>
8 #include <mbox.h>
9 #include <rvu.h>
10 #include "otx2_cpt_common.h"
11 #include "otx2_cpt_reqmgr.h"
12 
13 /*
14  * CPT instruction and pending queues user requested length in CPT_INST_S msgs
15  */
16 #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200
17 
18 /*
19  * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S
20  * messages.
21  */
22 #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40)
23 
24 /*
25  * CPT instruction and pending queues length in CPT_INST_S messages
26  */
27 #define OTX2_CPT_INST_QLEN_MSGS	((OTX2_CPT_SIZE_DIV40 - 1) * 40)
28 
29 /*
30  * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction
31  * queue has less than 320 free entries. So, increase HW instruction queue
32  * size by 320 and give 320 entries less for SW/NIX RX as a workaround.
33  */
34 #define OTX2_CPT_INST_QLEN_EXTRA_BYTES  (320 * OTX2_CPT_INST_SIZE)
35 #define OTX2_CPT_EXTRA_SIZE_DIV40       (320/40)
36 
37 /* CPT instruction queue length in bytes */
38 #define OTX2_CPT_INST_QLEN_BYTES                                               \
39 		((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) +             \
40 		OTX2_CPT_INST_QLEN_EXTRA_BYTES)
41 
42 /* CPT instruction group queue length in bytes */
43 #define OTX2_CPT_INST_GRP_QLEN_BYTES                                           \
44 		((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16)
45 
46 /* CPT FC length in bytes */
47 #define OTX2_CPT_Q_FC_LEN 128
48 
49 /* CPT instruction queue alignment */
50 #define OTX2_CPT_INST_Q_ALIGNMENT  128
51 
52 /* Mask which selects all engine groups */
53 #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF
54 
55 /* Maximum LFs supported in OcteonTX2 for CPT */
56 #define OTX2_CPT_MAX_LFS_NUM    64
57 
58 /* Queue priority */
59 #define OTX2_CPT_QUEUE_HI_PRIO  0x1
60 #define OTX2_CPT_QUEUE_LOW_PRIO 0x0
61 
62 enum otx2_cptlf_state {
63 	OTX2_CPTLF_IN_RESET,
64 	OTX2_CPTLF_STARTED,
65 };
66 
67 struct otx2_cpt_inst_queue {
68 	u8 *vaddr;
69 	u8 *real_vaddr;
70 	dma_addr_t dma_addr;
71 	dma_addr_t real_dma_addr;
72 	u32 size;
73 };
74 
75 struct otx2_cptlfs_info;
76 struct otx2_cptlf_wqe {
77 	struct tasklet_struct work;
78 	struct otx2_cptlfs_info *lfs;
79 	u8 lf_num;
80 };
81 
82 struct otx2_cptlf_info {
83 	struct otx2_cptlfs_info *lfs;           /* Ptr to cptlfs_info struct */
84 	void __iomem *lmtline;                  /* Address of LMTLINE */
85 	void __iomem *ioreg;                    /* LMTLINE send register */
86 	int msix_offset;                        /* MSI-X interrupts offset */
87 	cpumask_var_t affinity_mask;            /* IRQs affinity mask */
88 	u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */
89 	u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS];  /* Is interrupt registered */
90 	u8 slot;                                /* Slot number of this LF */
91 
92 	struct otx2_cpt_inst_queue iqueue;/* Instruction queue */
93 	struct otx2_cpt_pending_queue pqueue; /* Pending queue */
94 	struct otx2_cptlf_wqe *wqe;       /* Tasklet work info */
95 };
96 
97 struct cpt_hw_ops {
98 	void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
99 			 struct otx2_cptlf_info *lf);
100 	u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
101 	u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
102 };
103 
104 struct otx2_cptlfs_info {
105 	/* Registers start address of VF/PF LFs are attached to */
106 	void __iomem *reg_base;
107 #define LMTLINE_SIZE  128
108 	void __iomem *lmt_base;
109 	struct pci_dev *pdev;   /* Device LFs are attached to */
110 	struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
111 	struct otx2_mbox *mbox;
112 	struct cpt_hw_ops *ops;
113 	u8 are_lfs_attached;	/* Whether CPT LFs are attached */
114 	u8 lfs_num;		/* Number of CPT LFs */
115 	u8 kcrypto_eng_grp_num;	/* Kernel crypto engine group number */
116 	u8 kvf_limits;          /* Kernel crypto limits */
117 	atomic_t state;         /* LF's state. started/reset */
118 	int blkaddr;            /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
119 };
120 
otx2_cpt_free_instruction_queues(struct otx2_cptlfs_info * lfs)121 static inline void otx2_cpt_free_instruction_queues(
122 					struct otx2_cptlfs_info *lfs)
123 {
124 	struct otx2_cpt_inst_queue *iq;
125 	int i;
126 
127 	for (i = 0; i < lfs->lfs_num; i++) {
128 		iq = &lfs->lf[i].iqueue;
129 		if (iq->real_vaddr)
130 			dma_free_coherent(&lfs->pdev->dev,
131 					  iq->size,
132 					  iq->real_vaddr,
133 					  iq->real_dma_addr);
134 		iq->real_vaddr = NULL;
135 		iq->vaddr = NULL;
136 	}
137 }
138 
otx2_cpt_alloc_instruction_queues(struct otx2_cptlfs_info * lfs)139 static inline int otx2_cpt_alloc_instruction_queues(
140 					struct otx2_cptlfs_info *lfs)
141 {
142 	struct otx2_cpt_inst_queue *iq;
143 	int ret = 0, i;
144 
145 	if (!lfs->lfs_num)
146 		return -EINVAL;
147 
148 	for (i = 0; i < lfs->lfs_num; i++) {
149 		iq = &lfs->lf[i].iqueue;
150 		iq->size = OTX2_CPT_INST_QLEN_BYTES +
151 			   OTX2_CPT_Q_FC_LEN +
152 			   OTX2_CPT_INST_GRP_QLEN_BYTES +
153 			   OTX2_CPT_INST_Q_ALIGNMENT;
154 		iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size,
155 					&iq->real_dma_addr, GFP_KERNEL);
156 		if (!iq->real_vaddr) {
157 			ret = -ENOMEM;
158 			goto error;
159 		}
160 		iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES;
161 		iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES;
162 
163 		/* Align pointers */
164 		iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT);
165 		iq->dma_addr = PTR_ALIGN(iq->dma_addr,
166 					 OTX2_CPT_INST_Q_ALIGNMENT);
167 	}
168 	return 0;
169 
170 error:
171 	otx2_cpt_free_instruction_queues(lfs);
172 	return ret;
173 }
174 
otx2_cptlf_set_iqueues_base_addr(struct otx2_cptlfs_info * lfs)175 static inline void otx2_cptlf_set_iqueues_base_addr(
176 					struct otx2_cptlfs_info *lfs)
177 {
178 	union otx2_cptx_lf_q_base lf_q_base;
179 	int slot;
180 
181 	for (slot = 0; slot < lfs->lfs_num; slot++) {
182 		lf_q_base.u = lfs->lf[slot].iqueue.dma_addr;
183 		otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot,
184 				 OTX2_CPT_LF_Q_BASE, lf_q_base.u);
185 	}
186 }
187 
otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info * lf)188 static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf)
189 {
190 	union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 };
191 
192 	lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 +
193 				 OTX2_CPT_EXTRA_SIZE_DIV40;
194 	otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot,
195 			 OTX2_CPT_LF_Q_SIZE, lf_q_size.u);
196 }
197 
otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info * lfs)198 static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs)
199 {
200 	int slot;
201 
202 	for (slot = 0; slot < lfs->lfs_num; slot++)
203 		otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]);
204 }
205 
otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info * lf)206 static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf)
207 {
208 	union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 };
209 	union otx2_cptx_lf_inprog lf_inprog;
210 	u8 blkaddr = lf->lfs->blkaddr;
211 	int timeout = 20;
212 
213 	/* Disable instructions enqueuing */
214 	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
215 			 OTX2_CPT_LF_CTL, lf_ctl.u);
216 
217 	/* Wait for instruction queue to become empty */
218 	do {
219 		lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr,
220 					      lf->slot, OTX2_CPT_LF_INPROG);
221 		if (!lf_inprog.s.inflight)
222 			break;
223 
224 		usleep_range(10000, 20000);
225 		if (timeout-- < 0) {
226 			dev_err(&lf->lfs->pdev->dev,
227 				"Error LF %d is still busy.\n", lf->slot);
228 			break;
229 		}
230 
231 	} while (1);
232 
233 	/*
234 	 * Disable executions in the LF's queue,
235 	 * the queue should be empty at this point
236 	 */
237 	lf_inprog.s.eena = 0x0;
238 	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
239 			 OTX2_CPT_LF_INPROG, lf_inprog.u);
240 }
241 
otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info * lfs)242 static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs)
243 {
244 	int slot;
245 
246 	for (slot = 0; slot < lfs->lfs_num; slot++)
247 		otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]);
248 }
249 
otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info * lf,bool enable)250 static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf,
251 					     bool enable)
252 {
253 	u8 blkaddr = lf->lfs->blkaddr;
254 	union otx2_cptx_lf_ctl lf_ctl;
255 
256 	lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
257 				   OTX2_CPT_LF_CTL);
258 
259 	/* Set iqueue's enqueuing */
260 	lf_ctl.s.ena = enable ? 0x1 : 0x0;
261 	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
262 			 OTX2_CPT_LF_CTL, lf_ctl.u);
263 }
264 
otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info * lf)265 static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf)
266 {
267 	otx2_cptlf_set_iqueue_enq(lf, true);
268 }
269 
otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info * lf,bool enable)270 static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
271 					      bool enable)
272 {
273 	union otx2_cptx_lf_inprog lf_inprog;
274 	u8 blkaddr = lf->lfs->blkaddr;
275 
276 	lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
277 				      OTX2_CPT_LF_INPROG);
278 
279 	/* Set iqueue's execution */
280 	lf_inprog.s.eena = enable ? 0x1 : 0x0;
281 	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
282 			 OTX2_CPT_LF_INPROG, lf_inprog.u);
283 }
284 
otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info * lf)285 static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
286 {
287 	otx2_cptlf_set_iqueue_exec(lf, true);
288 }
289 
otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info * lf)290 static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf)
291 {
292 	otx2_cptlf_set_iqueue_exec(lf, false);
293 }
294 
otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info * lfs)295 static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
296 {
297 	int slot;
298 
299 	for (slot = 0; slot < lfs->lfs_num; slot++) {
300 		otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
301 		otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
302 	}
303 }
304 
otx2_cpt_fill_inst(union otx2_cpt_inst_s * cptinst,struct otx2_cpt_iq_command * iq_cmd,u64 comp_baddr)305 static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst,
306 				      struct otx2_cpt_iq_command *iq_cmd,
307 				      u64 comp_baddr)
308 {
309 	cptinst->u[0] = 0x0;
310 	cptinst->s.doneint = true;
311 	cptinst->s.res_addr = comp_baddr;
312 	cptinst->u[2] = 0x0;
313 	cptinst->u[3] = 0x0;
314 	cptinst->s.ei0 = iq_cmd->cmd.u;
315 	cptinst->s.ei1 = iq_cmd->dptr;
316 	cptinst->s.ei2 = iq_cmd->rptr;
317 	cptinst->s.ei3 = iq_cmd->cptr.u;
318 }
319 
320 /*
321  * On OcteonTX2 platform the parameter insts_num is used as a count of
322  * instructions to be enqueued. The valid values for insts_num are:
323  * 1 - 1 CPT instruction will be enqueued during LMTST operation
324  * 2 - 2 CPT instructions will be enqueued during LMTST operation
325  */
otx2_cpt_send_cmd(union otx2_cpt_inst_s * cptinst,u32 insts_num,struct otx2_cptlf_info * lf)326 static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst,
327 				     u32 insts_num, struct otx2_cptlf_info *lf)
328 {
329 	void __iomem *lmtline = lf->lmtline;
330 	long ret;
331 
332 	/*
333 	 * Make sure memory areas pointed in CPT_INST_S
334 	 * are flushed before the instruction is sent to CPT
335 	 */
336 	dma_wmb();
337 
338 	do {
339 		/* Copy CPT command to LMTLINE */
340 		memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
341 
342 		/*
343 		 * LDEOR initiates atomic transfer to I/O device
344 		 * The following will cause the LMTST to fail (the LDEOR
345 		 * returns zero):
346 		 * - No stores have been performed to the LMTLINE since it was
347 		 * last invalidated.
348 		 * - The bytes which have been stored to LMTLINE since it was
349 		 * last invalidated form a pattern that is non-contiguous, does
350 		 * not start at byte 0, or does not end on a 8-byte boundary.
351 		 * (i.e.comprises a formation of other than 1–16 8-byte
352 		 * words.)
353 		 *
354 		 * These rules are designed such that an operating system
355 		 * context switch or hypervisor guest switch need have no
356 		 * knowledge of the LMTST operations; the switch code does not
357 		 * need to store to LMTCANCEL. Also note as LMTLINE data cannot
358 		 * be read, there is no information leakage between processes.
359 		 */
360 		ret = otx2_lmt_flush(lf->ioreg);
361 
362 	} while (!ret);
363 }
364 
otx2_cptlf_started(struct otx2_cptlfs_info * lfs)365 static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs)
366 {
367 	return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED;
368 }
369 
otx2_cptlf_set_dev_info(struct otx2_cptlfs_info * lfs,struct pci_dev * pdev,void __iomem * reg_base,struct otx2_mbox * mbox,int blkaddr)370 static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs,
371 					   struct pci_dev *pdev,
372 					   void __iomem *reg_base,
373 					   struct otx2_mbox *mbox,
374 					   int blkaddr)
375 {
376 	lfs->pdev = pdev;
377 	lfs->reg_base = reg_base;
378 	lfs->mbox = mbox;
379 	lfs->blkaddr = blkaddr;
380 }
381 
382 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
383 		    int lfs_num);
384 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
385 int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs);
386 void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs);
387 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
388 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
389 
390 #endif /* __OTX2_CPTLF_H */
391