164506017SSrujana Challa /* SPDX-License-Identifier: GPL-2.0-only 264506017SSrujana Challa * Copyright (C) 2020 Marvell. 364506017SSrujana Challa */ 464506017SSrujana Challa #ifndef __OTX2_CPTLF_H 564506017SSrujana Challa #define __OTX2_CPTLF_H 664506017SSrujana Challa 778506c2aSSrujana Challa #include <linux/soc/marvell/octeontx2/asm.h> 864506017SSrujana Challa #include <mbox.h> 964506017SSrujana Challa #include <rvu.h> 1064506017SSrujana Challa #include "otx2_cpt_common.h" 1178506c2aSSrujana Challa #include "otx2_cpt_reqmgr.h" 1264506017SSrujana Challa 1364506017SSrujana Challa /* 1464506017SSrujana Challa * CPT instruction and pending queues user requested length in CPT_INST_S msgs 1564506017SSrujana Challa */ 1664506017SSrujana Challa #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 1764506017SSrujana Challa 1864506017SSrujana Challa /* 1964506017SSrujana Challa * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S 2064506017SSrujana Challa * messages. 2164506017SSrujana Challa */ 2264506017SSrujana Challa #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) 2364506017SSrujana Challa 2464506017SSrujana Challa /* 2564506017SSrujana Challa * CPT instruction and pending queues length in CPT_INST_S messages 2664506017SSrujana Challa */ 2764506017SSrujana Challa #define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) 2864506017SSrujana Challa 2964506017SSrujana Challa /* CPT instruction queue length in bytes */ 3064506017SSrujana Challa #define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \ 3164506017SSrujana Challa OTX2_CPT_INST_SIZE) 3264506017SSrujana Challa 3364506017SSrujana Challa /* CPT instruction group queue length in bytes */ 3464506017SSrujana Challa #define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16) 3564506017SSrujana Challa 3664506017SSrujana Challa /* CPT FC length in bytes */ 3764506017SSrujana Challa #define OTX2_CPT_Q_FC_LEN 128 3864506017SSrujana Challa 3964506017SSrujana Challa /* CPT instruction queue alignment */ 4064506017SSrujana Challa #define OTX2_CPT_INST_Q_ALIGNMENT 128 4164506017SSrujana Challa 4264506017SSrujana Challa /* Mask which selects all engine groups */ 4364506017SSrujana Challa #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF 4464506017SSrujana Challa 4564506017SSrujana Challa /* Maximum LFs supported in OcteonTX2 for CPT */ 4664506017SSrujana Challa #define OTX2_CPT_MAX_LFS_NUM 64 4764506017SSrujana Challa 4864506017SSrujana Challa /* Queue priority */ 4964506017SSrujana Challa #define OTX2_CPT_QUEUE_HI_PRIO 0x1 5064506017SSrujana Challa #define OTX2_CPT_QUEUE_LOW_PRIO 0x0 5164506017SSrujana Challa 5264506017SSrujana Challa enum otx2_cptlf_state { 5364506017SSrujana Challa OTX2_CPTLF_IN_RESET, 5464506017SSrujana Challa OTX2_CPTLF_STARTED, 5564506017SSrujana Challa }; 5664506017SSrujana Challa 5764506017SSrujana Challa struct otx2_cpt_inst_queue { 5864506017SSrujana Challa u8 *vaddr; 5964506017SSrujana Challa u8 *real_vaddr; 6064506017SSrujana Challa dma_addr_t dma_addr; 6164506017SSrujana Challa dma_addr_t real_dma_addr; 6264506017SSrujana Challa u32 size; 6364506017SSrujana Challa }; 6464506017SSrujana Challa 6564506017SSrujana Challa struct otx2_cptlfs_info; 6664506017SSrujana Challa struct otx2_cptlf_wqe { 6764506017SSrujana Challa struct tasklet_struct work; 6864506017SSrujana Challa struct otx2_cptlfs_info *lfs; 6964506017SSrujana Challa u8 lf_num; 7064506017SSrujana Challa }; 7164506017SSrujana Challa 7264506017SSrujana Challa struct otx2_cptlf_info { 7364506017SSrujana Challa struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ 7464506017SSrujana Challa void __iomem *lmtline; /* Address of LMTLINE */ 7564506017SSrujana Challa void __iomem *ioreg; /* LMTLINE send register */ 7664506017SSrujana Challa int msix_offset; /* MSI-X interrupts offset */ 7764506017SSrujana Challa cpumask_var_t affinity_mask; /* IRQs affinity mask */ 7864506017SSrujana Challa u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ 7964506017SSrujana Challa u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ 8064506017SSrujana Challa u8 slot; /* Slot number of this LF */ 8164506017SSrujana Challa 8264506017SSrujana Challa struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ 83*8ec8015aSSrujana Challa struct otx2_cpt_pending_queue pqueue; /* Pending queue */ 8464506017SSrujana Challa struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ 8564506017SSrujana Challa }; 8664506017SSrujana Challa 8764506017SSrujana Challa struct otx2_cptlfs_info { 8864506017SSrujana Challa /* Registers start address of VF/PF LFs are attached to */ 8964506017SSrujana Challa void __iomem *reg_base; 9064506017SSrujana Challa struct pci_dev *pdev; /* Device LFs are attached to */ 9164506017SSrujana Challa struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; 9264506017SSrujana Challa struct otx2_mbox *mbox; 9364506017SSrujana Challa u8 are_lfs_attached; /* Whether CPT LFs are attached */ 9464506017SSrujana Challa u8 lfs_num; /* Number of CPT LFs */ 95*8ec8015aSSrujana Challa u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ 96*8ec8015aSSrujana Challa u8 kvf_limits; /* Kernel crypto limits */ 9764506017SSrujana Challa atomic_t state; /* LF's state. started/reset */ 9864506017SSrujana Challa }; 9964506017SSrujana Challa 10064506017SSrujana Challa static inline void otx2_cpt_free_instruction_queues( 10164506017SSrujana Challa struct otx2_cptlfs_info *lfs) 10264506017SSrujana Challa { 10364506017SSrujana Challa struct otx2_cpt_inst_queue *iq; 10464506017SSrujana Challa int i; 10564506017SSrujana Challa 10664506017SSrujana Challa for (i = 0; i < lfs->lfs_num; i++) { 10764506017SSrujana Challa iq = &lfs->lf[i].iqueue; 10864506017SSrujana Challa if (iq->real_vaddr) 10964506017SSrujana Challa dma_free_coherent(&lfs->pdev->dev, 11064506017SSrujana Challa iq->size, 11164506017SSrujana Challa iq->real_vaddr, 11264506017SSrujana Challa iq->real_dma_addr); 11364506017SSrujana Challa iq->real_vaddr = NULL; 11464506017SSrujana Challa iq->vaddr = NULL; 11564506017SSrujana Challa } 11664506017SSrujana Challa } 11764506017SSrujana Challa 11864506017SSrujana Challa static inline int otx2_cpt_alloc_instruction_queues( 11964506017SSrujana Challa struct otx2_cptlfs_info *lfs) 12064506017SSrujana Challa { 12164506017SSrujana Challa struct otx2_cpt_inst_queue *iq; 12264506017SSrujana Challa int ret = 0, i; 12364506017SSrujana Challa 12464506017SSrujana Challa if (!lfs->lfs_num) 12564506017SSrujana Challa return -EINVAL; 12664506017SSrujana Challa 12764506017SSrujana Challa for (i = 0; i < lfs->lfs_num; i++) { 12864506017SSrujana Challa iq = &lfs->lf[i].iqueue; 12964506017SSrujana Challa iq->size = OTX2_CPT_INST_QLEN_BYTES + 13064506017SSrujana Challa OTX2_CPT_Q_FC_LEN + 13164506017SSrujana Challa OTX2_CPT_INST_GRP_QLEN_BYTES + 13264506017SSrujana Challa OTX2_CPT_INST_Q_ALIGNMENT; 13364506017SSrujana Challa iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, 13464506017SSrujana Challa &iq->real_dma_addr, GFP_KERNEL); 13564506017SSrujana Challa if (!iq->real_vaddr) { 13664506017SSrujana Challa ret = -ENOMEM; 13764506017SSrujana Challa goto error; 13864506017SSrujana Challa } 13964506017SSrujana Challa iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; 14064506017SSrujana Challa iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; 14164506017SSrujana Challa 14264506017SSrujana Challa /* Align pointers */ 14364506017SSrujana Challa iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); 14464506017SSrujana Challa iq->dma_addr = PTR_ALIGN(iq->dma_addr, 14564506017SSrujana Challa OTX2_CPT_INST_Q_ALIGNMENT); 14664506017SSrujana Challa } 14764506017SSrujana Challa return 0; 14864506017SSrujana Challa 14964506017SSrujana Challa error: 15064506017SSrujana Challa otx2_cpt_free_instruction_queues(lfs); 15164506017SSrujana Challa return ret; 15264506017SSrujana Challa } 15364506017SSrujana Challa 15464506017SSrujana Challa static inline void otx2_cptlf_set_iqueues_base_addr( 15564506017SSrujana Challa struct otx2_cptlfs_info *lfs) 15664506017SSrujana Challa { 15764506017SSrujana Challa union otx2_cptx_lf_q_base lf_q_base; 15864506017SSrujana Challa int slot; 15964506017SSrujana Challa 16064506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) { 16164506017SSrujana Challa lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; 16264506017SSrujana Challa otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, 16364506017SSrujana Challa OTX2_CPT_LF_Q_BASE, lf_q_base.u); 16464506017SSrujana Challa } 16564506017SSrujana Challa } 16664506017SSrujana Challa 16764506017SSrujana Challa static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) 16864506017SSrujana Challa { 16964506017SSrujana Challa union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; 17064506017SSrujana Challa 17164506017SSrujana Challa lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40; 17264506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 17364506017SSrujana Challa OTX2_CPT_LF_Q_SIZE, lf_q_size.u); 17464506017SSrujana Challa } 17564506017SSrujana Challa 17664506017SSrujana Challa static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) 17764506017SSrujana Challa { 17864506017SSrujana Challa int slot; 17964506017SSrujana Challa 18064506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) 18164506017SSrujana Challa otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); 18264506017SSrujana Challa } 18364506017SSrujana Challa 18464506017SSrujana Challa static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) 18564506017SSrujana Challa { 18664506017SSrujana Challa union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; 18764506017SSrujana Challa union otx2_cptx_lf_inprog lf_inprog; 18864506017SSrujana Challa int timeout = 20; 18964506017SSrujana Challa 19064506017SSrujana Challa /* Disable instructions enqueuing */ 19164506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 19264506017SSrujana Challa OTX2_CPT_LF_CTL, lf_ctl.u); 19364506017SSrujana Challa 19464506017SSrujana Challa /* Wait for instruction queue to become empty */ 19564506017SSrujana Challa do { 19664506017SSrujana Challa lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, 19764506017SSrujana Challa lf->slot, OTX2_CPT_LF_INPROG); 19864506017SSrujana Challa if (!lf_inprog.s.inflight) 19964506017SSrujana Challa break; 20064506017SSrujana Challa 20164506017SSrujana Challa usleep_range(10000, 20000); 20264506017SSrujana Challa if (timeout-- < 0) { 20364506017SSrujana Challa dev_err(&lf->lfs->pdev->dev, 20464506017SSrujana Challa "Error LF %d is still busy.\n", lf->slot); 20564506017SSrujana Challa break; 20664506017SSrujana Challa } 20764506017SSrujana Challa 20864506017SSrujana Challa } while (1); 20964506017SSrujana Challa 21064506017SSrujana Challa /* 21164506017SSrujana Challa * Disable executions in the LF's queue, 21264506017SSrujana Challa * the queue should be empty at this point 21364506017SSrujana Challa */ 21464506017SSrujana Challa lf_inprog.s.eena = 0x0; 21564506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 21664506017SSrujana Challa OTX2_CPT_LF_INPROG, lf_inprog.u); 21764506017SSrujana Challa } 21864506017SSrujana Challa 21964506017SSrujana Challa static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) 22064506017SSrujana Challa { 22164506017SSrujana Challa int slot; 22264506017SSrujana Challa 22364506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) 22464506017SSrujana Challa otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); 22564506017SSrujana Challa } 22664506017SSrujana Challa 22764506017SSrujana Challa static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, 22864506017SSrujana Challa bool enable) 22964506017SSrujana Challa { 23064506017SSrujana Challa union otx2_cptx_lf_ctl lf_ctl; 23164506017SSrujana Challa 23264506017SSrujana Challa lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 23364506017SSrujana Challa OTX2_CPT_LF_CTL); 23464506017SSrujana Challa 23564506017SSrujana Challa /* Set iqueue's enqueuing */ 23664506017SSrujana Challa lf_ctl.s.ena = enable ? 0x1 : 0x0; 23764506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 23864506017SSrujana Challa OTX2_CPT_LF_CTL, lf_ctl.u); 23964506017SSrujana Challa } 24064506017SSrujana Challa 24164506017SSrujana Challa static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) 24264506017SSrujana Challa { 24364506017SSrujana Challa otx2_cptlf_set_iqueue_enq(lf, true); 24464506017SSrujana Challa } 24564506017SSrujana Challa 24664506017SSrujana Challa static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, 24764506017SSrujana Challa bool enable) 24864506017SSrujana Challa { 24964506017SSrujana Challa union otx2_cptx_lf_inprog lf_inprog; 25064506017SSrujana Challa 25164506017SSrujana Challa lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 25264506017SSrujana Challa OTX2_CPT_LF_INPROG); 25364506017SSrujana Challa 25464506017SSrujana Challa /* Set iqueue's execution */ 25564506017SSrujana Challa lf_inprog.s.eena = enable ? 0x1 : 0x0; 25664506017SSrujana Challa otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 25764506017SSrujana Challa OTX2_CPT_LF_INPROG, lf_inprog.u); 25864506017SSrujana Challa } 25964506017SSrujana Challa 26064506017SSrujana Challa static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) 26164506017SSrujana Challa { 26264506017SSrujana Challa otx2_cptlf_set_iqueue_exec(lf, true); 26364506017SSrujana Challa } 26464506017SSrujana Challa 26564506017SSrujana Challa static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) 26664506017SSrujana Challa { 26764506017SSrujana Challa otx2_cptlf_set_iqueue_exec(lf, false); 26864506017SSrujana Challa } 26964506017SSrujana Challa 27064506017SSrujana Challa static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) 27164506017SSrujana Challa { 27264506017SSrujana Challa int slot; 27364506017SSrujana Challa 27464506017SSrujana Challa for (slot = 0; slot < lfs->lfs_num; slot++) { 27564506017SSrujana Challa otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); 27664506017SSrujana Challa otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); 27764506017SSrujana Challa } 27864506017SSrujana Challa } 27964506017SSrujana Challa 28078506c2aSSrujana Challa static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, 28178506c2aSSrujana Challa struct otx2_cpt_iq_command *iq_cmd, 28278506c2aSSrujana Challa u64 comp_baddr) 28378506c2aSSrujana Challa { 28478506c2aSSrujana Challa cptinst->u[0] = 0x0; 28578506c2aSSrujana Challa cptinst->s.doneint = true; 28678506c2aSSrujana Challa cptinst->s.res_addr = comp_baddr; 28778506c2aSSrujana Challa cptinst->u[2] = 0x0; 28878506c2aSSrujana Challa cptinst->u[3] = 0x0; 28978506c2aSSrujana Challa cptinst->s.ei0 = iq_cmd->cmd.u; 29078506c2aSSrujana Challa cptinst->s.ei1 = iq_cmd->dptr; 29178506c2aSSrujana Challa cptinst->s.ei2 = iq_cmd->rptr; 29278506c2aSSrujana Challa cptinst->s.ei3 = iq_cmd->cptr.u; 29378506c2aSSrujana Challa } 29478506c2aSSrujana Challa 29578506c2aSSrujana Challa /* 29678506c2aSSrujana Challa * On OcteonTX2 platform the parameter insts_num is used as a count of 29778506c2aSSrujana Challa * instructions to be enqueued. The valid values for insts_num are: 29878506c2aSSrujana Challa * 1 - 1 CPT instruction will be enqueued during LMTST operation 29978506c2aSSrujana Challa * 2 - 2 CPT instructions will be enqueued during LMTST operation 30078506c2aSSrujana Challa */ 30178506c2aSSrujana Challa static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, 30278506c2aSSrujana Challa u32 insts_num, struct otx2_cptlf_info *lf) 30378506c2aSSrujana Challa { 30478506c2aSSrujana Challa void __iomem *lmtline = lf->lmtline; 30578506c2aSSrujana Challa long ret; 30678506c2aSSrujana Challa 30778506c2aSSrujana Challa /* 30878506c2aSSrujana Challa * Make sure memory areas pointed in CPT_INST_S 30978506c2aSSrujana Challa * are flushed before the instruction is sent to CPT 31078506c2aSSrujana Challa */ 31178506c2aSSrujana Challa dma_wmb(); 31278506c2aSSrujana Challa 31378506c2aSSrujana Challa do { 31478506c2aSSrujana Challa /* Copy CPT command to LMTLINE */ 31578506c2aSSrujana Challa memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); 31678506c2aSSrujana Challa 31778506c2aSSrujana Challa /* 31878506c2aSSrujana Challa * LDEOR initiates atomic transfer to I/O device 31978506c2aSSrujana Challa * The following will cause the LMTST to fail (the LDEOR 32078506c2aSSrujana Challa * returns zero): 32178506c2aSSrujana Challa * - No stores have been performed to the LMTLINE since it was 32278506c2aSSrujana Challa * last invalidated. 32378506c2aSSrujana Challa * - The bytes which have been stored to LMTLINE since it was 32478506c2aSSrujana Challa * last invalidated form a pattern that is non-contiguous, does 32578506c2aSSrujana Challa * not start at byte 0, or does not end on a 8-byte boundary. 32678506c2aSSrujana Challa * (i.e.comprises a formation of other than 1–16 8-byte 32778506c2aSSrujana Challa * words.) 32878506c2aSSrujana Challa * 32978506c2aSSrujana Challa * These rules are designed such that an operating system 33078506c2aSSrujana Challa * context switch or hypervisor guest switch need have no 33178506c2aSSrujana Challa * knowledge of the LMTST operations; the switch code does not 33278506c2aSSrujana Challa * need to store to LMTCANCEL. Also note as LMTLINE data cannot 33378506c2aSSrujana Challa * be read, there is no information leakage between processes. 33478506c2aSSrujana Challa */ 33578506c2aSSrujana Challa ret = otx2_lmt_flush(lf->ioreg); 33678506c2aSSrujana Challa 33778506c2aSSrujana Challa } while (!ret); 33878506c2aSSrujana Challa } 33978506c2aSSrujana Challa 340*8ec8015aSSrujana Challa static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) 341*8ec8015aSSrujana Challa { 342*8ec8015aSSrujana Challa return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; 343*8ec8015aSSrujana Challa } 344*8ec8015aSSrujana Challa 34564506017SSrujana Challa int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, 34664506017SSrujana Challa int lfs_num); 34764506017SSrujana Challa void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); 34864506017SSrujana Challa int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); 34964506017SSrujana Challa void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); 35064506017SSrujana Challa void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); 35164506017SSrujana Challa int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); 35264506017SSrujana Challa 35364506017SSrujana Challa #endif /* __OTX2_CPTLF_H */ 354