1 /* SPDX-License-Identifier: GPL-2.0-only 2 * Copyright (C) 2020 Marvell. 3 */ 4 #ifndef __OTX2_CPTLF_H 5 #define __OTX2_CPTLF_H 6 7 #include <linux/soc/marvell/octeontx2/asm.h> 8 #include <mbox.h> 9 #include <rvu.h> 10 #include "otx2_cpt_common.h" 11 #include "otx2_cpt_reqmgr.h" 12 13 /* 14 * CPT instruction and pending queues user requested length in CPT_INST_S msgs 15 */ 16 #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 17 18 /* 19 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S 20 * messages. 21 */ 22 #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) 23 24 /* 25 * CPT instruction and pending queues length in CPT_INST_S messages 26 */ 27 #define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) 28 29 /* CPT instruction queue length in bytes */ 30 #define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \ 31 OTX2_CPT_INST_SIZE) 32 33 /* CPT instruction group queue length in bytes */ 34 #define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16) 35 36 /* CPT FC length in bytes */ 37 #define OTX2_CPT_Q_FC_LEN 128 38 39 /* CPT instruction queue alignment */ 40 #define OTX2_CPT_INST_Q_ALIGNMENT 128 41 42 /* Mask which selects all engine groups */ 43 #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF 44 45 /* Maximum LFs supported in OcteonTX2 for CPT */ 46 #define OTX2_CPT_MAX_LFS_NUM 64 47 48 /* Queue priority */ 49 #define OTX2_CPT_QUEUE_HI_PRIO 0x1 50 #define OTX2_CPT_QUEUE_LOW_PRIO 0x0 51 52 enum otx2_cptlf_state { 53 OTX2_CPTLF_IN_RESET, 54 OTX2_CPTLF_STARTED, 55 }; 56 57 struct otx2_cpt_inst_queue { 58 u8 *vaddr; 59 u8 *real_vaddr; 60 dma_addr_t dma_addr; 61 dma_addr_t real_dma_addr; 62 u32 size; 63 }; 64 65 struct otx2_cptlfs_info; 66 struct otx2_cptlf_wqe { 67 struct tasklet_struct work; 68 struct otx2_cptlfs_info *lfs; 69 u8 lf_num; 70 }; 71 72 struct otx2_cptlf_info { 73 struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ 74 void __iomem *lmtline; /* Address of LMTLINE */ 75 void __iomem *ioreg; /* LMTLINE send register */ 76 int msix_offset; /* MSI-X interrupts offset */ 77 cpumask_var_t affinity_mask; /* IRQs affinity mask */ 78 u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ 79 u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ 80 u8 slot; /* Slot number of this LF */ 81 82 struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ 83 struct otx2_cpt_pending_queue pqueue; /* Pending queue */ 84 struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ 85 }; 86 87 struct cpt_hw_ops { 88 void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num, 89 struct otx2_cptlf_info *lf); 90 u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result); 91 u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result); 92 }; 93 94 struct otx2_cptlfs_info { 95 /* Registers start address of VF/PF LFs are attached to */ 96 void __iomem *reg_base; 97 #define LMTLINE_SIZE 128 98 void __iomem *lmt_base; 99 struct pci_dev *pdev; /* Device LFs are attached to */ 100 struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; 101 struct otx2_mbox *mbox; 102 struct cpt_hw_ops *ops; 103 u8 are_lfs_attached; /* Whether CPT LFs are attached */ 104 u8 lfs_num; /* Number of CPT LFs */ 105 u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ 106 u8 kvf_limits; /* Kernel crypto limits */ 107 atomic_t state; /* LF's state. started/reset */ 108 int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */ 109 }; 110 111 static inline void otx2_cpt_free_instruction_queues( 112 struct otx2_cptlfs_info *lfs) 113 { 114 struct otx2_cpt_inst_queue *iq; 115 int i; 116 117 for (i = 0; i < lfs->lfs_num; i++) { 118 iq = &lfs->lf[i].iqueue; 119 if (iq->real_vaddr) 120 dma_free_coherent(&lfs->pdev->dev, 121 iq->size, 122 iq->real_vaddr, 123 iq->real_dma_addr); 124 iq->real_vaddr = NULL; 125 iq->vaddr = NULL; 126 } 127 } 128 129 static inline int otx2_cpt_alloc_instruction_queues( 130 struct otx2_cptlfs_info *lfs) 131 { 132 struct otx2_cpt_inst_queue *iq; 133 int ret = 0, i; 134 135 if (!lfs->lfs_num) 136 return -EINVAL; 137 138 for (i = 0; i < lfs->lfs_num; i++) { 139 iq = &lfs->lf[i].iqueue; 140 iq->size = OTX2_CPT_INST_QLEN_BYTES + 141 OTX2_CPT_Q_FC_LEN + 142 OTX2_CPT_INST_GRP_QLEN_BYTES + 143 OTX2_CPT_INST_Q_ALIGNMENT; 144 iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, 145 &iq->real_dma_addr, GFP_KERNEL); 146 if (!iq->real_vaddr) { 147 ret = -ENOMEM; 148 goto error; 149 } 150 iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; 151 iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; 152 153 /* Align pointers */ 154 iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); 155 iq->dma_addr = PTR_ALIGN(iq->dma_addr, 156 OTX2_CPT_INST_Q_ALIGNMENT); 157 } 158 return 0; 159 160 error: 161 otx2_cpt_free_instruction_queues(lfs); 162 return ret; 163 } 164 165 static inline void otx2_cptlf_set_iqueues_base_addr( 166 struct otx2_cptlfs_info *lfs) 167 { 168 union otx2_cptx_lf_q_base lf_q_base; 169 int slot; 170 171 for (slot = 0; slot < lfs->lfs_num; slot++) { 172 lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; 173 otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, 174 OTX2_CPT_LF_Q_BASE, lf_q_base.u); 175 } 176 } 177 178 static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) 179 { 180 union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; 181 182 lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40; 183 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 184 OTX2_CPT_LF_Q_SIZE, lf_q_size.u); 185 } 186 187 static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) 188 { 189 int slot; 190 191 for (slot = 0; slot < lfs->lfs_num; slot++) 192 otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); 193 } 194 195 static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) 196 { 197 union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; 198 union otx2_cptx_lf_inprog lf_inprog; 199 int timeout = 20; 200 201 /* Disable instructions enqueuing */ 202 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 203 OTX2_CPT_LF_CTL, lf_ctl.u); 204 205 /* Wait for instruction queue to become empty */ 206 do { 207 lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, 208 lf->slot, OTX2_CPT_LF_INPROG); 209 if (!lf_inprog.s.inflight) 210 break; 211 212 usleep_range(10000, 20000); 213 if (timeout-- < 0) { 214 dev_err(&lf->lfs->pdev->dev, 215 "Error LF %d is still busy.\n", lf->slot); 216 break; 217 } 218 219 } while (1); 220 221 /* 222 * Disable executions in the LF's queue, 223 * the queue should be empty at this point 224 */ 225 lf_inprog.s.eena = 0x0; 226 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 227 OTX2_CPT_LF_INPROG, lf_inprog.u); 228 } 229 230 static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) 231 { 232 int slot; 233 234 for (slot = 0; slot < lfs->lfs_num; slot++) 235 otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); 236 } 237 238 static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, 239 bool enable) 240 { 241 union otx2_cptx_lf_ctl lf_ctl; 242 243 lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 244 OTX2_CPT_LF_CTL); 245 246 /* Set iqueue's enqueuing */ 247 lf_ctl.s.ena = enable ? 0x1 : 0x0; 248 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 249 OTX2_CPT_LF_CTL, lf_ctl.u); 250 } 251 252 static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) 253 { 254 otx2_cptlf_set_iqueue_enq(lf, true); 255 } 256 257 static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, 258 bool enable) 259 { 260 union otx2_cptx_lf_inprog lf_inprog; 261 262 lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 263 OTX2_CPT_LF_INPROG); 264 265 /* Set iqueue's execution */ 266 lf_inprog.s.eena = enable ? 0x1 : 0x0; 267 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 268 OTX2_CPT_LF_INPROG, lf_inprog.u); 269 } 270 271 static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) 272 { 273 otx2_cptlf_set_iqueue_exec(lf, true); 274 } 275 276 static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) 277 { 278 otx2_cptlf_set_iqueue_exec(lf, false); 279 } 280 281 static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) 282 { 283 int slot; 284 285 for (slot = 0; slot < lfs->lfs_num; slot++) { 286 otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); 287 otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); 288 } 289 } 290 291 static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, 292 struct otx2_cpt_iq_command *iq_cmd, 293 u64 comp_baddr) 294 { 295 cptinst->u[0] = 0x0; 296 cptinst->s.doneint = true; 297 cptinst->s.res_addr = comp_baddr; 298 cptinst->u[2] = 0x0; 299 cptinst->u[3] = 0x0; 300 cptinst->s.ei0 = iq_cmd->cmd.u; 301 cptinst->s.ei1 = iq_cmd->dptr; 302 cptinst->s.ei2 = iq_cmd->rptr; 303 cptinst->s.ei3 = iq_cmd->cptr.u; 304 } 305 306 /* 307 * On OcteonTX2 platform the parameter insts_num is used as a count of 308 * instructions to be enqueued. The valid values for insts_num are: 309 * 1 - 1 CPT instruction will be enqueued during LMTST operation 310 * 2 - 2 CPT instructions will be enqueued during LMTST operation 311 */ 312 static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, 313 u32 insts_num, struct otx2_cptlf_info *lf) 314 { 315 void __iomem *lmtline = lf->lmtline; 316 long ret; 317 318 /* 319 * Make sure memory areas pointed in CPT_INST_S 320 * are flushed before the instruction is sent to CPT 321 */ 322 dma_wmb(); 323 324 do { 325 /* Copy CPT command to LMTLINE */ 326 memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); 327 328 /* 329 * LDEOR initiates atomic transfer to I/O device 330 * The following will cause the LMTST to fail (the LDEOR 331 * returns zero): 332 * - No stores have been performed to the LMTLINE since it was 333 * last invalidated. 334 * - The bytes which have been stored to LMTLINE since it was 335 * last invalidated form a pattern that is non-contiguous, does 336 * not start at byte 0, or does not end on a 8-byte boundary. 337 * (i.e.comprises a formation of other than 1–16 8-byte 338 * words.) 339 * 340 * These rules are designed such that an operating system 341 * context switch or hypervisor guest switch need have no 342 * knowledge of the LMTST operations; the switch code does not 343 * need to store to LMTCANCEL. Also note as LMTLINE data cannot 344 * be read, there is no information leakage between processes. 345 */ 346 ret = otx2_lmt_flush(lf->ioreg); 347 348 } while (!ret); 349 } 350 351 static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) 352 { 353 return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; 354 } 355 356 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, 357 int lfs_num); 358 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); 359 int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); 360 void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); 361 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); 362 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); 363 364 #endif /* __OTX2_CPTLF_H */ 365