1 /* SPDX-License-Identifier: GPL-2.0-only 2 * Copyright (C) 2020 Marvell. 3 */ 4 #ifndef __OTX2_CPTLF_H 5 #define __OTX2_CPTLF_H 6 7 #include <linux/soc/marvell/octeontx2/asm.h> 8 #include <mbox.h> 9 #include <rvu.h> 10 #include "otx2_cpt_common.h" 11 #include "otx2_cpt_reqmgr.h" 12 13 /* 14 * CPT instruction and pending queues user requested length in CPT_INST_S msgs 15 */ 16 #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 17 18 /* 19 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S 20 * messages. 21 */ 22 #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) 23 24 /* 25 * CPT instruction and pending queues length in CPT_INST_S messages 26 */ 27 #define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) 28 29 /* 30 * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction 31 * queue has less than 320 free entries. So, increase HW instruction queue 32 * size by 320 and give 320 entries less for SW/NIX RX as a workaround. 33 */ 34 #define OTX2_CPT_INST_QLEN_EXTRA_BYTES (320 * OTX2_CPT_INST_SIZE) 35 #define OTX2_CPT_EXTRA_SIZE_DIV40 (320/40) 36 37 /* CPT instruction queue length in bytes */ 38 #define OTX2_CPT_INST_QLEN_BYTES \ 39 ((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) + \ 40 OTX2_CPT_INST_QLEN_EXTRA_BYTES) 41 42 /* CPT instruction group queue length in bytes */ 43 #define OTX2_CPT_INST_GRP_QLEN_BYTES \ 44 ((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16) 45 46 /* CPT FC length in bytes */ 47 #define OTX2_CPT_Q_FC_LEN 128 48 49 /* CPT instruction queue alignment */ 50 #define OTX2_CPT_INST_Q_ALIGNMENT 128 51 52 /* Mask which selects all engine groups */ 53 #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF 54 55 /* Maximum LFs supported in OcteonTX2 for CPT */ 56 #define OTX2_CPT_MAX_LFS_NUM 64 57 58 /* Queue priority */ 59 #define OTX2_CPT_QUEUE_HI_PRIO 0x1 60 #define OTX2_CPT_QUEUE_LOW_PRIO 0x0 61 62 enum otx2_cptlf_state { 63 OTX2_CPTLF_IN_RESET, 64 OTX2_CPTLF_STARTED, 65 }; 66 67 struct otx2_cpt_inst_queue { 68 u8 *vaddr; 69 u8 *real_vaddr; 70 dma_addr_t dma_addr; 71 dma_addr_t real_dma_addr; 72 u32 size; 73 }; 74 75 struct otx2_cptlfs_info; 76 struct otx2_cptlf_wqe { 77 struct tasklet_struct work; 78 struct otx2_cptlfs_info *lfs; 79 u8 lf_num; 80 }; 81 82 struct otx2_cptlf_info { 83 struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ 84 void __iomem *lmtline; /* Address of LMTLINE */ 85 void __iomem *ioreg; /* LMTLINE send register */ 86 int msix_offset; /* MSI-X interrupts offset */ 87 cpumask_var_t affinity_mask; /* IRQs affinity mask */ 88 u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ 89 u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ 90 u8 slot; /* Slot number of this LF */ 91 92 struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ 93 struct otx2_cpt_pending_queue pqueue; /* Pending queue */ 94 struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ 95 }; 96 97 struct cpt_hw_ops { 98 void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num, 99 struct otx2_cptlf_info *lf); 100 u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result); 101 u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result); 102 }; 103 104 struct otx2_cptlfs_info { 105 /* Registers start address of VF/PF LFs are attached to */ 106 void __iomem *reg_base; 107 #define LMTLINE_SIZE 128 108 void __iomem *lmt_base; 109 struct pci_dev *pdev; /* Device LFs are attached to */ 110 struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; 111 struct otx2_mbox *mbox; 112 struct cpt_hw_ops *ops; 113 u8 are_lfs_attached; /* Whether CPT LFs are attached */ 114 u8 lfs_num; /* Number of CPT LFs */ 115 u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ 116 u8 kvf_limits; /* Kernel crypto limits */ 117 atomic_t state; /* LF's state. started/reset */ 118 int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */ 119 }; 120 121 static inline void otx2_cpt_free_instruction_queues( 122 struct otx2_cptlfs_info *lfs) 123 { 124 struct otx2_cpt_inst_queue *iq; 125 int i; 126 127 for (i = 0; i < lfs->lfs_num; i++) { 128 iq = &lfs->lf[i].iqueue; 129 if (iq->real_vaddr) 130 dma_free_coherent(&lfs->pdev->dev, 131 iq->size, 132 iq->real_vaddr, 133 iq->real_dma_addr); 134 iq->real_vaddr = NULL; 135 iq->vaddr = NULL; 136 } 137 } 138 139 static inline int otx2_cpt_alloc_instruction_queues( 140 struct otx2_cptlfs_info *lfs) 141 { 142 struct otx2_cpt_inst_queue *iq; 143 int ret = 0, i; 144 145 if (!lfs->lfs_num) 146 return -EINVAL; 147 148 for (i = 0; i < lfs->lfs_num; i++) { 149 iq = &lfs->lf[i].iqueue; 150 iq->size = OTX2_CPT_INST_QLEN_BYTES + 151 OTX2_CPT_Q_FC_LEN + 152 OTX2_CPT_INST_GRP_QLEN_BYTES + 153 OTX2_CPT_INST_Q_ALIGNMENT; 154 iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, 155 &iq->real_dma_addr, GFP_KERNEL); 156 if (!iq->real_vaddr) { 157 ret = -ENOMEM; 158 goto error; 159 } 160 iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; 161 iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; 162 163 /* Align pointers */ 164 iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); 165 iq->dma_addr = PTR_ALIGN(iq->dma_addr, 166 OTX2_CPT_INST_Q_ALIGNMENT); 167 } 168 return 0; 169 170 error: 171 otx2_cpt_free_instruction_queues(lfs); 172 return ret; 173 } 174 175 static inline void otx2_cptlf_set_iqueues_base_addr( 176 struct otx2_cptlfs_info *lfs) 177 { 178 union otx2_cptx_lf_q_base lf_q_base; 179 int slot; 180 181 for (slot = 0; slot < lfs->lfs_num; slot++) { 182 lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; 183 otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, 184 OTX2_CPT_LF_Q_BASE, lf_q_base.u); 185 } 186 } 187 188 static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) 189 { 190 union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; 191 192 lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 + 193 OTX2_CPT_EXTRA_SIZE_DIV40; 194 otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot, 195 OTX2_CPT_LF_Q_SIZE, lf_q_size.u); 196 } 197 198 static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) 199 { 200 int slot; 201 202 for (slot = 0; slot < lfs->lfs_num; slot++) 203 otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); 204 } 205 206 static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) 207 { 208 union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; 209 union otx2_cptx_lf_inprog lf_inprog; 210 u8 blkaddr = lf->lfs->blkaddr; 211 int timeout = 20; 212 213 /* Disable instructions enqueuing */ 214 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 215 OTX2_CPT_LF_CTL, lf_ctl.u); 216 217 /* Wait for instruction queue to become empty */ 218 do { 219 lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, 220 lf->slot, OTX2_CPT_LF_INPROG); 221 if (!lf_inprog.s.inflight) 222 break; 223 224 usleep_range(10000, 20000); 225 if (timeout-- < 0) { 226 dev_err(&lf->lfs->pdev->dev, 227 "Error LF %d is still busy.\n", lf->slot); 228 break; 229 } 230 231 } while (1); 232 233 /* 234 * Disable executions in the LF's queue, 235 * the queue should be empty at this point 236 */ 237 lf_inprog.s.eena = 0x0; 238 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 239 OTX2_CPT_LF_INPROG, lf_inprog.u); 240 } 241 242 static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) 243 { 244 int slot; 245 246 for (slot = 0; slot < lfs->lfs_num; slot++) 247 otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); 248 } 249 250 static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, 251 bool enable) 252 { 253 u8 blkaddr = lf->lfs->blkaddr; 254 union otx2_cptx_lf_ctl lf_ctl; 255 256 lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, 257 OTX2_CPT_LF_CTL); 258 259 /* Set iqueue's enqueuing */ 260 lf_ctl.s.ena = enable ? 0x1 : 0x0; 261 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 262 OTX2_CPT_LF_CTL, lf_ctl.u); 263 } 264 265 static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) 266 { 267 otx2_cptlf_set_iqueue_enq(lf, true); 268 } 269 270 static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, 271 bool enable) 272 { 273 union otx2_cptx_lf_inprog lf_inprog; 274 u8 blkaddr = lf->lfs->blkaddr; 275 276 lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, 277 OTX2_CPT_LF_INPROG); 278 279 /* Set iqueue's execution */ 280 lf_inprog.s.eena = enable ? 0x1 : 0x0; 281 otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, 282 OTX2_CPT_LF_INPROG, lf_inprog.u); 283 } 284 285 static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) 286 { 287 otx2_cptlf_set_iqueue_exec(lf, true); 288 } 289 290 static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) 291 { 292 otx2_cptlf_set_iqueue_exec(lf, false); 293 } 294 295 static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) 296 { 297 int slot; 298 299 for (slot = 0; slot < lfs->lfs_num; slot++) { 300 otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); 301 otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); 302 } 303 } 304 305 static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, 306 struct otx2_cpt_iq_command *iq_cmd, 307 u64 comp_baddr) 308 { 309 cptinst->u[0] = 0x0; 310 cptinst->s.doneint = true; 311 cptinst->s.res_addr = comp_baddr; 312 cptinst->u[2] = 0x0; 313 cptinst->u[3] = 0x0; 314 cptinst->s.ei0 = iq_cmd->cmd.u; 315 cptinst->s.ei1 = iq_cmd->dptr; 316 cptinst->s.ei2 = iq_cmd->rptr; 317 cptinst->s.ei3 = iq_cmd->cptr.u; 318 } 319 320 /* 321 * On OcteonTX2 platform the parameter insts_num is used as a count of 322 * instructions to be enqueued. The valid values for insts_num are: 323 * 1 - 1 CPT instruction will be enqueued during LMTST operation 324 * 2 - 2 CPT instructions will be enqueued during LMTST operation 325 */ 326 static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, 327 u32 insts_num, struct otx2_cptlf_info *lf) 328 { 329 void __iomem *lmtline = lf->lmtline; 330 long ret; 331 332 /* 333 * Make sure memory areas pointed in CPT_INST_S 334 * are flushed before the instruction is sent to CPT 335 */ 336 dma_wmb(); 337 338 do { 339 /* Copy CPT command to LMTLINE */ 340 memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); 341 342 /* 343 * LDEOR initiates atomic transfer to I/O device 344 * The following will cause the LMTST to fail (the LDEOR 345 * returns zero): 346 * - No stores have been performed to the LMTLINE since it was 347 * last invalidated. 348 * - The bytes which have been stored to LMTLINE since it was 349 * last invalidated form a pattern that is non-contiguous, does 350 * not start at byte 0, or does not end on a 8-byte boundary. 351 * (i.e.comprises a formation of other than 1–16 8-byte 352 * words.) 353 * 354 * These rules are designed such that an operating system 355 * context switch or hypervisor guest switch need have no 356 * knowledge of the LMTST operations; the switch code does not 357 * need to store to LMTCANCEL. Also note as LMTLINE data cannot 358 * be read, there is no information leakage between processes. 359 */ 360 ret = otx2_lmt_flush(lf->ioreg); 361 362 } while (!ret); 363 } 364 365 static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) 366 { 367 return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; 368 } 369 370 static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs, 371 struct pci_dev *pdev, 372 void __iomem *reg_base, 373 struct otx2_mbox *mbox, 374 int blkaddr) 375 { 376 lfs->pdev = pdev; 377 lfs->reg_base = reg_base; 378 lfs->mbox = mbox; 379 lfs->blkaddr = blkaddr; 380 } 381 382 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, 383 int lfs_num); 384 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); 385 int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); 386 void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); 387 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); 388 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); 389 390 #endif /* __OTX2_CPTLF_H */ 391