164506017SSrujana Challa /* SPDX-License-Identifier: GPL-2.0-only 264506017SSrujana Challa * Copyright (C) 2020 Marvell. 364506017SSrujana Challa */ 464506017SSrujana Challa #ifndef __OTX2_CPTLF_H 564506017SSrujana Challa #define __OTX2_CPTLF_H 664506017SSrujana Challa 7*78506c2aSSrujana Challa #include <linux/soc/marvell/octeontx2/asm.h> 864506017SSrujana Challa #include <mbox.h> 964506017SSrujana Challa #include <rvu.h> 1064506017SSrujana Challa #include "otx2_cpt_common.h" 11*78506c2aSSrujana Challa #include "otx2_cpt_reqmgr.h" 1264506017SSrujana Challa 1364506017SSrujana Challa /* 1464506017SSrujana Challa * CPT instruction and pending queues user requested length in CPT_INST_S msgs 1564506017SSrujana Challa */ 1664506017SSrujana Challa #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 1764506017SSrujana Challa 1864506017SSrujana Challa /* 1964506017SSrujana Challa * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S 2064506017SSrujana Challa * messages. 2164506017SSrujana Challa */ 2264506017SSrujana Challa #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) 2364506017SSrujana Challa 2464506017SSrujana Challa /* 2564506017SSrujana Challa * CPT instruction and pending queues length in CPT_INST_S messages 2664506017SSrujana Challa */ 2764506017SSrujana Challa #define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) 2864506017SSrujana Challa 2964506017SSrujana Challa /* CPT instruction queue length in bytes */ 3064506017SSrujana Challa #define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \ 3164506017SSrujana Challa OTX2_CPT_INST_SIZE) 3264506017SSrujana Challa 3364506017SSrujana Challa /* CPT instruction group queue length in bytes */ 3464506017SSrujana Challa #define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16) 3564506017SSrujana Challa 3664506017SSrujana Challa /* CPT FC length in bytes */ 3764506017SSrujana Challa #define OTX2_CPT_Q_FC_LEN 128 3864506017SSrujana Challa 3964506017SSrujana Challa /* CPT instruction queue alignment */ 4064506017SSrujana Challa #define OTX2_CPT_INST_Q_ALIGNMENT 128 4164506017SSrujana Challa 4264506017SSrujana Challa /* Mask which selects all engine groups */ 4364506017SSrujana Challa #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF 4464506017SSrujana Challa 4564506017SSrujana Challa /* Maximum LFs supported in OcteonTX2 for CPT */ 4664506017SSrujana Challa #define OTX2_CPT_MAX_LFS_NUM 64 4764506017SSrujana Challa 4864506017SSrujana Challa /* Queue priority */ 4964506017SSrujana Challa #define OTX2_CPT_QUEUE_HI_PRIO 0x1 5064506017SSrujana Challa #define OTX2_CPT_QUEUE_LOW_PRIO 0x0 5164506017SSrujana Challa 5264506017SSrujana Challa enum otx2_cptlf_state { 5364506017SSrujana Challa OTX2_CPTLF_IN_RESET, 5464506017SSrujana Challa OTX2_CPTLF_STARTED, 5564506017SSrujana Challa }; 5664506017SSrujana Challa 5764506017SSrujana Challa struct otx2_cpt_inst_queue { 5864506017SSrujana Challa u8 *vaddr; 5964506017SSrujana Challa u8 *real_vaddr; 6064506017SSrujana Challa dma_addr_t dma_addr; 6164506017SSrujana Challa dma_addr_t real_dma_addr; 6264506017SSrujana Challa u32 size; 6364506017SSrujana Challa }; 6464506017SSrujana Challa 6564506017SSrujana Challa struct otx2_cptlfs_info; 6664506017SSrujana Challa struct otx2_cptlf_wqe { 6764506017SSrujana Challa struct tasklet_struct work; 6864506017SSrujana Challa struct otx2_cptlfs_info *lfs; 6964506017SSrujana Challa u8 lf_num; 7064506017SSrujana Challa }; 7164506017SSrujana Challa 7264506017SSrujana Challa struct otx2_cptlf_info { 7364506017SSrujana Challa struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ 7464506017SSrujana Challa void __iomem *lmtline; /* Address of LMTLINE */ 7564506017SSrujana Challa void __iomem *ioreg; /* LMTLINE send register */ 7664506017SSrujana Challa int msix_offset; /* MSI-X interrupts offset */ 7764506017SSrujana Challa cpumask_var_t affinity_mask; /* IRQs affinity mask */ 7864506017SSrujana Challa u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ 7964506017SSrujana Challa u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ 8064506017SSrujana Challa u8 slot; /* Slot number of this LF */ 8164506017SSrujana Challa 8264506017SSrujana Challa struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ 8364506017SSrujana Challa struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ 8464506017SSrujana Challa }; 8564506017SSrujana Challa 8664506017SSrujana Challa struct otx2_cptlfs_info { 8764506017SSrujana Challa /* Registers start address of VF/PF LFs are attached to */ 8864506017SSrujana Challa void __iomem *reg_base; 8964506017SSrujana Challa struct pci_dev *pdev; /* Device LFs are attached to */ 9064506017SSrujana Challa struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; 9164506017SSrujana Challa struct otx2_mbox *mbox; 9264506017SSrujana Challa u8 are_lfs_attached; /* Whether CPT LFs are attached */ 9364506017SSrujana Challa u8 lfs_num; /* Number of CPT LFs */ 9464506017SSrujana Challa atomic_t state; /* LF's state. started/reset */ 9564506017SSrujana Challa }; 9664506017SSrujana Challa 9764506017SSrujana Challa static inline void otx2_cpt_free_instruction_queues( 9864506017SSrujana Challa struct otx2_cptlfs_info *lfs) 9964506017SSrujana Challa { 10064506017SSrujana Challa struct otx2_cpt_inst_queue *iq; 10164506017SSrujana Challa int i; 10264506017SSrujana Challa 10364506017SSrujana Challa for (i = 0; i < lfs->lfs_num; i++) { 10464506017SSrujana Challa iq = &lfs->lf[i].iqueue; 10564506017SSrujana Challa if (iq->real_vaddr) 10664506017SSrujana Challa dma_free_coherent(&lfs->pdev->dev, 10764506017SSrujana Challa iq->size, 10864506017SSrujana Challa iq->real_vaddr, 10964506017SSrujana Challa iq->real_dma_addr); 11064506017SSrujana Challa iq->real_vaddr = NULL; 11164506017SSrujana Challa iq->vaddr = NULL; 11264506017SSrujana Challa } 11364506017SSrujana Challa } 11464506017SSrujana Challa 11564506017SSrujana Challa static inline int otx2_cpt_alloc_instruction_queues( 11664506017SSrujana Challa struct otx2_cptlfs_info *lfs) 11764506017SSrujana Challa { 11864506017SSrujana Challa struct otx2_cpt_inst_queue *iq; 11964506017SSrujana Challa int ret = 0, i; 12064506017SSrujana Challa 12164506017SSrujana Challa if (!lfs->lfs_num) 12264506017SSrujana Challa return -EINVAL; 12364506017SSrujana Challa 12464506017SSrujana Challa for (i = 0; i < lfs->lfs_num; i++) { 12564506017SSrujana Challa iq = &lfs->lf[i].iqueue; 12664506017SSrujana Challa iq->size = OTX2_CPT_INST_QLEN_BYTES + 12764506017SSrujana Challa OTX2_CPT_Q_FC_LEN + 12864506017SSrujana Challa OTX2_CPT_INST_GRP_QLEN_BYTES + 12964506017SSrujana Challa OTX2_CPT_INST_Q_ALIGNMENT; 13064506017SSrujana Challa iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, 13164506017SSrujana Challa &iq->real_dma_addr, GFP_KERNEL); 13264506017SSrujana Challa if (!iq->real_vaddr) { 13364506017SSrujana Challa ret = -ENOMEM; 13464506017SSrujana Challa goto error; 13564506017SSrujana Challa } 13664506017SSrujana Challa iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; 13764506017SSrujana Challa iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; 13864506017SSrujana Challa 13964506017SSrujana Challa /* Align pointers */ 14064506017SSrujana Challa iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); 14164506017SSrujana Challa iq->dma_addr = PTR_ALIGN(iq->dma_addr, 14264506017SSrujana Challa OTX2_CPT_INST_Q_ALIGNMENT); 14364506017SSrujana Challa } 14464506017SSrujana Challa return 0; 14564506017SSrujana Challa 14664506017SSrujana Challa error: 14764506017SSrujana Challa otx2_cpt_free_instruction_queues(lfs); 14864506017SSrujana Challa return ret; 14964506017SSrujana Challa } 15064506017SSrujana Challa 15164506017SSrujana Challa static inline void otx2_cptlf_set_iqueues_base_addr( 15264506017SSrujana Challa struct otx2_cptlfs_info *lfs) 15364506017SSrujana Challa { 15464506017SSrujana Challa union otx2_cptx_lf_q_base lf_q_base; 15564506017SSrujana Challa int slot; 15664506017SSrujana Challa 15764506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) { 15864506017SSrujana Challa lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; 15964506017SSrujana Challa otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, 16064506017SSrujana Challa OTX2_CPT_LF_Q_BASE, lf_q_base.u); 16164506017SSrujana Challa } 16264506017SSrujana Challa } 16364506017SSrujana Challa 16464506017SSrujana Challa static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) 16564506017SSrujana Challa { 16664506017SSrujana Challa union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; 16764506017SSrujana Challa 16864506017SSrujana Challa lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40; 16964506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 17064506017SSrujana Challa OTX2_CPT_LF_Q_SIZE, lf_q_size.u); 17164506017SSrujana Challa } 17264506017SSrujana Challa 17364506017SSrujana Challa static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) 17464506017SSrujana Challa { 17564506017SSrujana Challa int slot; 17664506017SSrujana Challa 17764506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) 17864506017SSrujana Challa otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); 17964506017SSrujana Challa } 18064506017SSrujana Challa 18164506017SSrujana Challa static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) 18264506017SSrujana Challa { 18364506017SSrujana Challa union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; 18464506017SSrujana Challa union otx2_cptx_lf_inprog lf_inprog; 18564506017SSrujana Challa int timeout = 20; 18664506017SSrujana Challa 18764506017SSrujana Challa /* Disable instructions enqueuing */ 18864506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 18964506017SSrujana Challa OTX2_CPT_LF_CTL, lf_ctl.u); 19064506017SSrujana Challa 19164506017SSrujana Challa /* Wait for instruction queue to become empty */ 19264506017SSrujana Challa do { 19364506017SSrujana Challa lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, 19464506017SSrujana Challa lf->slot, OTX2_CPT_LF_INPROG); 19564506017SSrujana Challa if (!lf_inprog.s.inflight) 19664506017SSrujana Challa break; 19764506017SSrujana Challa 19864506017SSrujana Challa usleep_range(10000, 20000); 19964506017SSrujana Challa if (timeout-- < 0) { 20064506017SSrujana Challa dev_err(&lf->lfs->pdev->dev, 20164506017SSrujana Challa "Error LF %d is still busy.\n", lf->slot); 20264506017SSrujana Challa break; 20364506017SSrujana Challa } 20464506017SSrujana Challa 20564506017SSrujana Challa } while (1); 20664506017SSrujana Challa 20764506017SSrujana Challa /* 20864506017SSrujana Challa * Disable executions in the LF's queue, 20964506017SSrujana Challa * the queue should be empty at this point 21064506017SSrujana Challa */ 21164506017SSrujana Challa lf_inprog.s.eena = 0x0; 21264506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 21364506017SSrujana Challa OTX2_CPT_LF_INPROG, lf_inprog.u); 21464506017SSrujana Challa } 21564506017SSrujana Challa 21664506017SSrujana Challa static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) 21764506017SSrujana Challa { 21864506017SSrujana Challa int slot; 21964506017SSrujana Challa 22064506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) 22164506017SSrujana Challa otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); 22264506017SSrujana Challa } 22364506017SSrujana Challa 22464506017SSrujana Challa static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, 22564506017SSrujana Challa bool enable) 22664506017SSrujana Challa { 22764506017SSrujana Challa union otx2_cptx_lf_ctl lf_ctl; 22864506017SSrujana Challa 22964506017SSrujana Challa lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 23064506017SSrujana Challa OTX2_CPT_LF_CTL); 23164506017SSrujana Challa 23264506017SSrujana Challa /* Set iqueue's enqueuing */ 23364506017SSrujana Challa lf_ctl.s.ena = enable ? 0x1 : 0x0; 23464506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 23564506017SSrujana Challa OTX2_CPT_LF_CTL, lf_ctl.u); 23664506017SSrujana Challa } 23764506017SSrujana Challa 23864506017SSrujana Challa static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) 23964506017SSrujana Challa { 24064506017SSrujana Challa otx2_cptlf_set_iqueue_enq(lf, true); 24164506017SSrujana Challa } 24264506017SSrujana Challa 24364506017SSrujana Challa static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, 24464506017SSrujana Challa bool enable) 24564506017SSrujana Challa { 24664506017SSrujana Challa union otx2_cptx_lf_inprog lf_inprog; 24764506017SSrujana Challa 24864506017SSrujana Challa lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 24964506017SSrujana Challa OTX2_CPT_LF_INPROG); 25064506017SSrujana Challa 25164506017SSrujana Challa /* Set iqueue's execution */ 25264506017SSrujana Challa lf_inprog.s.eena = enable ? 0x1 : 0x0; 25364506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 25464506017SSrujana Challa OTX2_CPT_LF_INPROG, lf_inprog.u); 25564506017SSrujana Challa } 25664506017SSrujana Challa 25764506017SSrujana Challa static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) 25864506017SSrujana Challa { 25964506017SSrujana Challa otx2_cptlf_set_iqueue_exec(lf, true); 26064506017SSrujana Challa } 26164506017SSrujana Challa 26264506017SSrujana Challa static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) 26364506017SSrujana Challa { 26464506017SSrujana Challa otx2_cptlf_set_iqueue_exec(lf, false); 26564506017SSrujana Challa } 26664506017SSrujana Challa 26764506017SSrujana Challa static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) 26864506017SSrujana Challa { 26964506017SSrujana Challa int slot; 27064506017SSrujana Challa 27164506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) { 27264506017SSrujana Challa otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); 27364506017SSrujana Challa otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); 27464506017SSrujana Challa } 27564506017SSrujana Challa } 27664506017SSrujana Challa 277*78506c2aSSrujana Challa static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, 278*78506c2aSSrujana Challa struct otx2_cpt_iq_command *iq_cmd, 279*78506c2aSSrujana Challa u64 comp_baddr) 280*78506c2aSSrujana Challa { 281*78506c2aSSrujana Challa cptinst->u[0] = 0x0; 282*78506c2aSSrujana Challa cptinst->s.doneint = true; 283*78506c2aSSrujana Challa cptinst->s.res_addr = comp_baddr; 284*78506c2aSSrujana Challa cptinst->u[2] = 0x0; 285*78506c2aSSrujana Challa cptinst->u[3] = 0x0; 286*78506c2aSSrujana Challa cptinst->s.ei0 = iq_cmd->cmd.u; 287*78506c2aSSrujana Challa cptinst->s.ei1 = iq_cmd->dptr; 288*78506c2aSSrujana Challa cptinst->s.ei2 = iq_cmd->rptr; 289*78506c2aSSrujana Challa cptinst->s.ei3 = iq_cmd->cptr.u; 290*78506c2aSSrujana Challa } 291*78506c2aSSrujana Challa 292*78506c2aSSrujana Challa /* 293*78506c2aSSrujana Challa * On OcteonTX2 platform the parameter insts_num is used as a count of 294*78506c2aSSrujana Challa * instructions to be enqueued. The valid values for insts_num are: 295*78506c2aSSrujana Challa * 1 - 1 CPT instruction will be enqueued during LMTST operation 296*78506c2aSSrujana Challa * 2 - 2 CPT instructions will be enqueued during LMTST operation 297*78506c2aSSrujana Challa */ 298*78506c2aSSrujana Challa static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, 299*78506c2aSSrujana Challa u32 insts_num, struct otx2_cptlf_info *lf) 300*78506c2aSSrujana Challa { 301*78506c2aSSrujana Challa void __iomem *lmtline = lf->lmtline; 302*78506c2aSSrujana Challa long ret; 303*78506c2aSSrujana Challa 304*78506c2aSSrujana Challa /* 305*78506c2aSSrujana Challa * Make sure memory areas pointed in CPT_INST_S 306*78506c2aSSrujana Challa * are flushed before the instruction is sent to CPT 307*78506c2aSSrujana Challa */ 308*78506c2aSSrujana Challa dma_wmb(); 309*78506c2aSSrujana Challa 310*78506c2aSSrujana Challa do { 311*78506c2aSSrujana Challa /* Copy CPT command to LMTLINE */ 312*78506c2aSSrujana Challa memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); 313*78506c2aSSrujana Challa 314*78506c2aSSrujana Challa /* 315*78506c2aSSrujana Challa * LDEOR initiates atomic transfer to I/O device 316*78506c2aSSrujana Challa * The following will cause the LMTST to fail (the LDEOR 317*78506c2aSSrujana Challa * returns zero): 318*78506c2aSSrujana Challa * - No stores have been performed to the LMTLINE since it was 319*78506c2aSSrujana Challa * last invalidated. 320*78506c2aSSrujana Challa * - The bytes which have been stored to LMTLINE since it was 321*78506c2aSSrujana Challa * last invalidated form a pattern that is non-contiguous, does 322*78506c2aSSrujana Challa * not start at byte 0, or does not end on a 8-byte boundary. 323*78506c2aSSrujana Challa * (i.e.comprises a formation of other than 1–16 8-byte 324*78506c2aSSrujana Challa * words.) 325*78506c2aSSrujana Challa * 326*78506c2aSSrujana Challa * These rules are designed such that an operating system 327*78506c2aSSrujana Challa * context switch or hypervisor guest switch need have no 328*78506c2aSSrujana Challa * knowledge of the LMTST operations; the switch code does not 329*78506c2aSSrujana Challa * need to store to LMTCANCEL. Also note as LMTLINE data cannot 330*78506c2aSSrujana Challa * be read, there is no information leakage between processes. 331*78506c2aSSrujana Challa */ 332*78506c2aSSrujana Challa ret = otx2_lmt_flush(lf->ioreg); 333*78506c2aSSrujana Challa 334*78506c2aSSrujana Challa } while (!ret); 335*78506c2aSSrujana Challa } 336*78506c2aSSrujana Challa 33764506017SSrujana Challa int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, 33864506017SSrujana Challa int lfs_num); 33964506017SSrujana Challa void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); 34064506017SSrujana Challa int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); 34164506017SSrujana Challa void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); 34264506017SSrujana Challa void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); 34364506017SSrujana Challa int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); 34464506017SSrujana Challa 34564506017SSrujana Challa #endif /* __OTX2_CPTLF_H */ 346