1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2020-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include "gaudi2P.h" 9 #include "gaudi2_masks.h" 10 #include "../include/gaudi2/gaudi2_special_blocks.h" 11 #include "../include/hw_ip/mmu/mmu_general.h" 12 #include "../include/hw_ip/mmu/mmu_v2_0.h" 13 #include "../include/gaudi2/gaudi2_packets.h" 14 #include "../include/gaudi2/gaudi2_reg_map.h" 15 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h" 16 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h" 17 18 #include <linux/module.h> 19 #include <linux/pci.h> 20 #include <linux/hwmon.h> 21 #include <linux/iommu.h> 22 23 #define GAUDI2_DMA_POOL_BLK_SIZE SZ_256 /* 256 bytes */ 24 25 #define GAUDI2_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ 26 27 #define GAUDI2_RESET_POLL_TIMEOUT_USEC 500000 /* 500ms */ 28 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC 25000 /* 25s */ 29 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC 25000 /* 25s */ 30 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC 3000000 /* 3s */ 31 #define GAUDI2_RESET_POLL_CNT 3 32 #define GAUDI2_RESET_WAIT_MSEC 1 /* 1ms */ 33 #define GAUDI2_CPU_RESET_WAIT_MSEC 100 /* 100ms */ 34 #define GAUDI2_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 35 #define GAUDI2_CB_POOL_CB_CNT 512 36 #define GAUDI2_CB_POOL_CB_SIZE SZ_128K /* 128KB */ 37 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 38 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC 25000000 /* 25s */ 39 #define GAUDI2_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */ 41 42 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT 3 43 44 /* 45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs 46 * and the code relies on that value (for array size etc..) we define another value 47 * for MAX faulty TPCs which reflects the cluster binning requirements 48 */ 49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS 1 50 #define MAX_FAULTY_XBARS 1 51 #define MAX_FAULTY_EDMAS 1 52 #define MAX_FAULTY_DECODERS 1 53 54 #define GAUDI2_TPC_FULL_MASK 0x1FFFFFF 55 #define GAUDI2_HIF_HMMU_FULL_MASK 0xFFFF 56 #define GAUDI2_DECODER_FULL_MASK 0x3FF 57 58 #define GAUDI2_NA_EVENT_CAUSE 0xFF 59 #define GAUDI2_NUM_OF_QM_ERR_CAUSE 18 60 #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE 25 61 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE 3 62 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE 14 63 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE 3 64 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE 2 65 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE 22 66 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE 31 67 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE 25 68 #define GAUDI2_NUM_OF_MME_ERR_CAUSE 16 69 #define GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE 5 70 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE 7 71 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE 8 72 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE 19 73 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE 9 74 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE 3 75 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE 3 76 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE 2 77 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE 2 78 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE 2 79 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE 5 80 81 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 10) 82 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 200) 83 #define GAUDI2_ARB_WDT_TIMEOUT (0x1000000) 84 85 #define GAUDI2_VDEC_TIMEOUT_USEC 10000 /* 10ms */ 86 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC (GAUDI2_VDEC_TIMEOUT_USEC * 100) 87 88 #define KDMA_TIMEOUT_USEC USEC_PER_SEC 89 90 #define IS_DMA_IDLE(dma_core_sts0) \ 91 (!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK))) 92 93 #define IS_DMA_HALTED(dma_core_sts1) \ 94 ((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK)) 95 96 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) 97 98 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK)) 99 100 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \ 101 ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \ 102 (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \ 103 (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK))) 104 105 #define PCIE_DEC_EN_MASK 0x300 106 #define DEC_WORK_STATE_IDLE 0 107 #define DEC_WORK_STATE_PEND 3 108 #define IS_DEC_IDLE(dec_swreg15) \ 109 (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \ 110 ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_PEND) 111 112 /* HBM MMU address scrambling parameters */ 113 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE SZ_8M 114 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT 26 115 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT 0 116 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK DRAM_VA_HINT_MASK 117 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR 16 118 #define MMU_RANGE_INV_VA_LSB_SHIFT 12 119 #define MMU_RANGE_INV_VA_MSB_SHIFT 44 120 #define MMU_RANGE_INV_EN_SHIFT 0 121 #define MMU_RANGE_INV_ASID_EN_SHIFT 1 122 #define MMU_RANGE_INV_ASID_SHIFT 2 123 124 /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has 125 * a 2 entries FIFO, and hence it is not enabled for it. 126 */ 127 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0) 128 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0) 129 130 #define GAUDI2_MAX_STRING_LEN 64 131 132 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \ 133 GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1) 134 135 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) 136 137 /* RAZWI initiator coordinates */ 138 #define RAZWI_GET_AXUSER_XY(x) \ 139 ((x & 0xF8001FF0) >> 4) 140 141 #define RAZWI_GET_AXUSER_LOW_XY(x) \ 142 ((x & 0x00001FF0) >> 4) 143 144 #define RAZWI_INITIATOR_AXUER_L_X_SHIFT 0 145 #define RAZWI_INITIATOR_AXUER_L_X_MASK 0x1F 146 #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT 5 147 #define RAZWI_INITIATOR_AXUER_L_Y_MASK 0xF 148 149 #define RAZWI_INITIATOR_AXUER_H_X_SHIFT 23 150 #define RAZWI_INITIATOR_AXUER_H_X_MASK 0x1F 151 152 #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \ 153 ((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \ 154 (((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT)) 155 156 #define RAZWI_INITIATOR_ID_X_HIGH(x) \ 157 (((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT) 158 159 #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \ 160 (RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh)) 161 162 #define PSOC_RAZWI_ENG_STR_SIZE 128 163 #define PSOC_RAZWI_MAX_ENG_PER_RTR 5 164 165 /* HW scrambles only bits 0-25 */ 166 #define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26) 167 168 struct gaudi2_razwi_info { 169 u32 axuser_xy; 170 u32 rtr_ctrl; 171 u16 eng_id; 172 char *eng_name; 173 }; 174 175 static struct gaudi2_razwi_info common_razwi_info[] = { 176 {RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE, 177 GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"}, 178 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 179 GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"}, 180 {RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE, 181 GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"}, 182 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 183 GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"}, 184 {RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE, 185 GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"}, 186 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 187 GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"}, 188 {RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE, 189 GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"}, 190 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 191 GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"}, 192 {RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE, 193 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"}, 194 {RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE, 195 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"}, 196 {RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE, 197 GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"}, 198 {RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE, 199 GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"}, 200 {RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE, 201 GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"}, 202 {RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE, 203 GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"}, 204 {RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE, 205 GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"}, 206 {RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE, 207 GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"}, 208 {RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE, 209 GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"}, 210 {RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE, 211 GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"}, 212 {RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE, 213 GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"}, 214 {RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE, 215 GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"}, 216 {RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE, 217 GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"}, 218 {RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE, 219 GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"}, 220 {RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE, 221 GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"}, 222 {RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE, 223 GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"}, 224 {RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE, 225 GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"}, 226 {RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE, 227 GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"}, 228 {RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE, 229 GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"}, 230 {RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE, 231 GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"}, 232 {RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE, 233 GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"}, 234 {RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE, 235 GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"}, 236 {RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE, 237 GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"}, 238 {RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE, 239 GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"}, 240 {RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE, 241 GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"}, 242 {RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE, 243 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"}, 244 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 245 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"}, 246 {RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE, 247 GAUDI2_ENGINE_ID_NIC0_0, "NIC0"}, 248 {RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE, 249 GAUDI2_ENGINE_ID_NIC0_1, "NIC1"}, 250 {RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE, 251 GAUDI2_ENGINE_ID_NIC1_0, "NIC2"}, 252 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 253 GAUDI2_ENGINE_ID_NIC1_1, "NIC3"}, 254 {RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE, 255 GAUDI2_ENGINE_ID_NIC2_0, "NIC4"}, 256 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 257 GAUDI2_ENGINE_ID_NIC2_1, "NIC5"}, 258 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 259 GAUDI2_ENGINE_ID_NIC3_0, "NIC6"}, 260 {RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE, 261 GAUDI2_ENGINE_ID_NIC3_1, "NIC7"}, 262 {RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE, 263 GAUDI2_ENGINE_ID_NIC4_0, "NIC8"}, 264 {RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE, 265 GAUDI2_ENGINE_ID_NIC4_1, "NIC9"}, 266 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 267 GAUDI2_ENGINE_ID_NIC5_0, "NIC10"}, 268 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 269 GAUDI2_ENGINE_ID_NIC5_1, "NIC11"}, 270 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 271 GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"}, 272 {RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE, 273 GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"}, 274 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 275 GAUDI2_ENGINE_ID_SIZE, "PMMU"}, 276 {RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE, 277 GAUDI2_ENGINE_ID_SIZE, "PCIE"}, 278 {RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE, 279 GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"}, 280 {RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE, 281 GAUDI2_ENGINE_ID_KDMA, "KDMA"}, 282 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE, 283 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"}, 284 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE, 285 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"}, 286 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE, 287 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"}, 288 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE, 289 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"}, 290 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 291 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"}, 292 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 293 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"}, 294 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 295 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"}, 296 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 297 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"}, 298 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 299 GAUDI2_ENGINE_ID_SIZE, "HMMU0"}, 300 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 301 GAUDI2_ENGINE_ID_SIZE, "HMMU1"}, 302 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 303 GAUDI2_ENGINE_ID_SIZE, "HMMU2"}, 304 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 305 GAUDI2_ENGINE_ID_SIZE, "HMMU3"}, 306 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 307 GAUDI2_ENGINE_ID_SIZE, "HMMU4"}, 308 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 309 GAUDI2_ENGINE_ID_SIZE, "HMMU5"}, 310 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 311 GAUDI2_ENGINE_ID_SIZE, "HMMU6"}, 312 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 313 GAUDI2_ENGINE_ID_SIZE, "HMMU7"}, 314 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 315 GAUDI2_ENGINE_ID_SIZE, "HMMU8"}, 316 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 317 GAUDI2_ENGINE_ID_SIZE, "HMMU9"}, 318 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 319 GAUDI2_ENGINE_ID_SIZE, "HMMU10"}, 320 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 321 GAUDI2_ENGINE_ID_SIZE, "HMMU11"}, 322 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 323 GAUDI2_ENGINE_ID_SIZE, "HMMU12"}, 324 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 325 GAUDI2_ENGINE_ID_SIZE, "HMMU13"}, 326 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 327 GAUDI2_ENGINE_ID_SIZE, "HMMU14"}, 328 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 329 GAUDI2_ENGINE_ID_SIZE, "HMMU15"}, 330 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 331 GAUDI2_ENGINE_ID_ROT_0, "ROT0"}, 332 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 333 GAUDI2_ENGINE_ID_ROT_1, "ROT1"}, 334 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 335 GAUDI2_ENGINE_ID_PSOC, "CPU"}, 336 {RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE, 337 GAUDI2_ENGINE_ID_PSOC, "PSOC"} 338 }; 339 340 static struct gaudi2_razwi_info mme_razwi_info[] = { 341 /* MME X high coordinate is N/A, hence using only low coordinates */ 342 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 343 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"}, 344 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 345 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"}, 346 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 347 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"}, 348 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 349 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"}, 350 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 351 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"}, 352 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 353 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"}, 354 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 355 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"}, 356 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 357 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"}, 358 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 359 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"}, 360 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 361 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"}, 362 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 363 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"}, 364 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 365 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"}, 366 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 367 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"}, 368 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 369 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"}, 370 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 371 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"}, 372 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 373 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"}, 374 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 375 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"}, 376 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 377 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"}, 378 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 379 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"}, 380 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 381 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"}, 382 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 383 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"}, 384 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 385 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"}, 386 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 387 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"}, 388 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 389 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"}, 390 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 391 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"}, 392 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 393 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"}, 394 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 395 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"}, 396 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 397 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"}, 398 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 399 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"}, 400 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 401 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"}, 402 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 403 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"}, 404 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 405 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"}, 406 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 407 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"}, 408 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 409 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"}, 410 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 411 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"}, 412 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 413 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"} 414 }; 415 416 enum hl_pmmu_fatal_cause { 417 LATENCY_RD_OUT_FIFO_OVERRUN, 418 LATENCY_WR_OUT_FIFO_OVERRUN, 419 }; 420 421 enum hl_pcie_drain_ind_cause { 422 LBW_AXI_DRAIN_IND, 423 HBW_AXI_DRAIN_IND 424 }; 425 426 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = { 427 [HBM_ID0] = 0xFFFC, 428 [HBM_ID1] = 0xFFCF, 429 [HBM_ID2] = 0xF7F7, 430 [HBM_ID3] = 0x7F7F, 431 [HBM_ID4] = 0xFCFF, 432 [HBM_ID5] = 0xCFFF, 433 }; 434 435 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = { 436 [0] = HBM_ID0, 437 [1] = HBM_ID1, 438 [2] = HBM_ID4, 439 [3] = HBM_ID5, 440 }; 441 442 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = { 443 [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0, 444 [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2, 445 [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1, 446 [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3, 447 [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2, 448 [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4, 449 [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3, 450 [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5, 451 }; 452 453 static const int gaudi2_qman_async_event_id[] = { 454 [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM, 455 [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM, 456 [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM, 457 [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM, 458 [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM, 459 [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM, 460 [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM, 461 [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM, 462 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM, 463 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM, 464 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM, 465 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM, 466 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM, 467 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM, 468 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM, 469 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM, 470 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM, 471 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM, 472 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM, 473 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM, 474 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM, 475 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM, 476 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM, 477 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM, 478 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM, 479 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM, 480 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM, 481 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM, 482 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM, 483 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM, 484 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM, 485 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM, 486 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM, 487 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM, 488 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM, 489 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM, 490 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM, 491 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM, 492 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM, 493 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM, 494 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM, 495 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM, 496 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM, 497 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM, 498 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM, 499 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM, 500 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM, 501 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM, 502 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM, 503 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM, 504 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM, 505 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM, 506 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM, 507 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM, 508 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM, 509 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM, 510 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM, 511 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM, 512 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM, 513 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM, 514 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM, 515 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM, 516 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM, 517 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM, 518 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM, 519 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM, 520 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM, 521 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM, 522 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM, 523 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM, 524 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM, 525 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM, 526 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM, 527 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM, 528 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM, 529 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM, 530 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM, 531 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM, 532 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM, 533 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM, 534 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM, 535 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM, 536 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM, 537 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM, 538 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM, 539 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM, 540 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM, 541 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM, 542 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM, 543 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM, 544 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM, 545 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM, 546 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM, 547 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM, 548 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM, 549 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM, 550 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM, 551 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM, 552 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM, 553 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM, 554 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM, 555 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM, 556 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM, 557 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM, 558 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM, 559 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM, 560 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM, 561 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM, 562 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM, 563 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM, 564 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM, 565 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM, 566 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM, 567 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM, 568 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM, 569 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM, 570 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM, 571 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM, 572 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM, 573 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM, 574 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM, 575 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM, 576 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM, 577 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM, 578 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM, 579 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM, 580 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM, 581 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM, 582 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM, 583 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM, 584 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM, 585 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM, 586 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM, 587 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM, 588 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM, 589 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM, 590 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM, 591 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM, 592 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM, 593 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM, 594 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM, 595 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM, 596 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM, 597 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM, 598 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM, 599 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM, 600 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM, 601 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM, 602 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM, 603 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM, 604 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM, 605 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM, 606 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM, 607 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM, 608 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM, 609 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM, 610 [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0, 611 [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0, 612 [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0, 613 [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0, 614 [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1, 615 [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1, 616 [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1, 617 [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1, 618 [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0, 619 [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0, 620 [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0, 621 [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0, 622 [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1, 623 [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1, 624 [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1, 625 [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1, 626 [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0, 627 [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0, 628 [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0, 629 [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0, 630 [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1, 631 [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1, 632 [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1, 633 [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1, 634 [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0, 635 [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0, 636 [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0, 637 [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0, 638 [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1, 639 [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1, 640 [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1, 641 [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1, 642 [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0, 643 [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0, 644 [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0, 645 [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0, 646 [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1, 647 [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1, 648 [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1, 649 [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1, 650 [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0, 651 [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0, 652 [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0, 653 [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0, 654 [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1, 655 [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1, 656 [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1, 657 [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1, 658 [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0, 659 [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0, 660 [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0, 661 [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0, 662 [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1, 663 [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1, 664 [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1, 665 [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1, 666 [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0, 667 [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0, 668 [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0, 669 [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0, 670 [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1, 671 [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1, 672 [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1, 673 [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1, 674 [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0, 675 [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0, 676 [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0, 677 [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0, 678 [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1, 679 [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1, 680 [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1, 681 [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1, 682 [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0, 683 [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0, 684 [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0, 685 [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0, 686 [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1, 687 [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1, 688 [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1, 689 [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1, 690 [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0, 691 [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0, 692 [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0, 693 [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0, 694 [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1, 695 [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1, 696 [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1, 697 [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1, 698 [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0, 699 [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0, 700 [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0, 701 [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0, 702 [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1, 703 [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1, 704 [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1, 705 [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1, 706 [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 707 [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 708 [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 709 [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 710 [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 711 [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 712 [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 713 [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM 714 }; 715 716 static const int gaudi2_dma_core_async_event_id[] = { 717 [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE, 718 [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE, 719 [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE, 720 [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE, 721 [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE, 722 [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE, 723 [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE, 724 [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE, 725 [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE, 726 [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE, 727 [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE, 728 }; 729 730 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = { 731 "qman sei intr", 732 "arc sei intr" 733 }; 734 735 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = { 736 "AXI_TERMINATOR WR", 737 "AXI_TERMINATOR RD", 738 "AXI SPLIT SEI Status" 739 }; 740 741 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = { 742 "cbu_bresp_sei_intr_cause", 743 "cbu_rresp_sei_intr_cause", 744 "lbu_bresp_sei_intr_cause", 745 "lbu_rresp_sei_intr_cause", 746 "cbu_axi_split_intr_cause", 747 "lbu_axi_split_intr_cause", 748 "arc_ip_excptn_sei_intr_cause", 749 "dmi_bresp_sei_intr_cause", 750 "aux2apb_err_sei_intr_cause", 751 "cfg_lbw_wr_terminated_intr_cause", 752 "cfg_lbw_rd_terminated_intr_cause", 753 "cfg_dccm_wr_terminated_intr_cause", 754 "cfg_dccm_rd_terminated_intr_cause", 755 "cfg_hbw_rd_terminated_intr_cause" 756 }; 757 758 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = { 759 "msix_vcd_hbw_sei", 760 "msix_l2c_hbw_sei", 761 "msix_nrm_hbw_sei", 762 "msix_abnrm_hbw_sei", 763 "msix_vcd_lbw_sei", 764 "msix_l2c_lbw_sei", 765 "msix_nrm_lbw_sei", 766 "msix_abnrm_lbw_sei", 767 "apb_vcd_lbw_sei", 768 "apb_l2c_lbw_sei", 769 "apb_nrm_lbw_sei", 770 "apb_abnrm_lbw_sei", 771 "dec_sei", 772 "dec_apb_sei", 773 "trc_apb_sei", 774 "lbw_mstr_if_sei", 775 "axi_split_bresp_err_sei", 776 "hbw_axi_wr_viol_sei", 777 "hbw_axi_rd_viol_sei", 778 "lbw_axi_wr_viol_sei", 779 "lbw_axi_rd_viol_sei", 780 "vcd_spi", 781 "l2c_spi", 782 "nrm_spi", 783 "abnrm_spi", 784 }; 785 786 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = { 787 "PQ AXI HBW error", 788 "CQ AXI HBW error", 789 "CP AXI HBW error", 790 "CP error due to undefined OPCODE", 791 "CP encountered STOP OPCODE", 792 "CP AXI LBW error", 793 "CP WRREG32 or WRBULK returned error", 794 "N/A", 795 "FENCE 0 inc over max value and clipped", 796 "FENCE 1 inc over max value and clipped", 797 "FENCE 2 inc over max value and clipped", 798 "FENCE 3 inc over max value and clipped", 799 "FENCE 0 dec under min value and clipped", 800 "FENCE 1 dec under min value and clipped", 801 "FENCE 2 dec under min value and clipped", 802 "FENCE 3 dec under min value and clipped", 803 "CPDMA Up overflow", 804 "PQC L2H error" 805 }; 806 807 static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = { 808 "RSVD0", 809 "CQ AXI HBW error", 810 "CP AXI HBW error", 811 "CP error due to undefined OPCODE", 812 "CP encountered STOP OPCODE", 813 "CP AXI LBW error", 814 "CP WRREG32 or WRBULK returned error", 815 "N/A", 816 "FENCE 0 inc over max value and clipped", 817 "FENCE 1 inc over max value and clipped", 818 "FENCE 2 inc over max value and clipped", 819 "FENCE 3 inc over max value and clipped", 820 "FENCE 0 dec under min value and clipped", 821 "FENCE 1 dec under min value and clipped", 822 "FENCE 2 dec under min value and clipped", 823 "FENCE 3 dec under min value and clipped", 824 "CPDMA Up overflow", 825 "RSVD17", 826 "CQ_WR_IFIFO_CI_ERR", 827 "CQ_WR_CTL_CI_ERR", 828 "ARC_CQF_RD_ERR", 829 "ARC_CQ_WR_IFIFO_CI_ERR", 830 "ARC_CQ_WR_CTL_CI_ERR", 831 "ARC_AXI_ERR", 832 "CP_SWITCH_WDT_ERR" 833 }; 834 835 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = { 836 "Choice push while full error", 837 "Choice Q watchdog error", 838 "MSG AXI LBW returned with error" 839 }; 840 841 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = { 842 "qm_axi_err", 843 "qm_trace_fence_events", 844 "qm_sw_err", 845 "qm_cp_sw_stop", 846 "lbw_mstr_rresp_err", 847 "lbw_mstr_bresp_err", 848 "lbw_msg_slverr", 849 "hbw_msg_slverr", 850 "wbc_slverr", 851 "hbw_mstr_rresp_err", 852 "hbw_mstr_bresp_err", 853 "sb_resp_intr", 854 "mrsb_resp_intr", 855 "core_dw_status_0", 856 "core_dw_status_1", 857 "core_dw_status_2", 858 "core_dw_status_3", 859 "core_dw_status_4", 860 "core_dw_status_5", 861 "core_dw_status_6", 862 "core_dw_status_7", 863 "async_arc2cpu_sei_intr", 864 }; 865 866 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = { 867 "tpc_address_exceed_slm", 868 "tpc_div_by_0", 869 "tpc_spu_mac_overflow", 870 "tpc_spu_addsub_overflow", 871 "tpc_spu_abs_overflow", 872 "tpc_spu_fma_fp_dst_nan", 873 "tpc_spu_fma_fp_dst_inf", 874 "tpc_spu_convert_fp_dst_nan", 875 "tpc_spu_convert_fp_dst_inf", 876 "tpc_spu_fp_dst_denorm", 877 "tpc_vpu_mac_overflow", 878 "tpc_vpu_addsub_overflow", 879 "tpc_vpu_abs_overflow", 880 "tpc_vpu_convert_fp_dst_nan", 881 "tpc_vpu_convert_fp_dst_inf", 882 "tpc_vpu_fma_fp_dst_nan", 883 "tpc_vpu_fma_fp_dst_inf", 884 "tpc_vpu_fp_dst_denorm", 885 "tpc_assertions", 886 "tpc_illegal_instruction", 887 "tpc_pc_wrap_around", 888 "tpc_qm_sw_err", 889 "tpc_hbw_rresp_err", 890 "tpc_hbw_bresp_err", 891 "tpc_lbw_rresp_err", 892 "tpc_lbw_bresp_err", 893 "st_unlock_already_locked", 894 "invalid_lock_access", 895 "LD_L protection violation", 896 "ST_L protection violation", 897 "D$ L0CS mismatch", 898 }; 899 900 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = { 901 "agu_resp_intr", 902 "qman_axi_err", 903 "wap sei (wbc axi err)", 904 "arc sei", 905 "cfg access error", 906 "qm_sw_err", 907 "sbte_dbg_intr_0", 908 "sbte_dbg_intr_1", 909 "sbte_dbg_intr_2", 910 "sbte_dbg_intr_3", 911 "sbte_dbg_intr_4", 912 "sbte_prtn_intr_0", 913 "sbte_prtn_intr_1", 914 "sbte_prtn_intr_2", 915 "sbte_prtn_intr_3", 916 "sbte_prtn_intr_4", 917 }; 918 919 static const char * const guadi2_mme_sbte_error_cause[GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE] = { 920 "i0", 921 "i1", 922 "i2", 923 "i3", 924 "i4", 925 }; 926 927 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = { 928 "WBC ERR RESP_0", 929 "WBC ERR RESP_1", 930 "AP SOURCE POS INF", 931 "AP SOURCE NEG INF", 932 "AP SOURCE NAN", 933 "AP RESULT POS INF", 934 "AP RESULT NEG INF", 935 }; 936 937 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 938 "HBW Read returned with error RRESP", 939 "HBW write returned with error BRESP", 940 "LBW write returned with error BRESP", 941 "descriptor_fifo_overflow", 942 "KDMA SB LBW Read returned with error", 943 "KDMA WBC LBW Write returned with error", 944 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 945 "WRONG CFG FOR COMMIT IN LIN DMA" 946 }; 947 948 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 949 "HBW/LBW Read returned with error RRESP", 950 "HBW/LBW write returned with error BRESP", 951 "LBW write returned with error BRESP", 952 "descriptor_fifo_overflow", 953 "KDMA SB LBW Read returned with error", 954 "KDMA WBC LBW Write returned with error", 955 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 956 "WRONG CFG FOR COMMIT IN LIN DMA" 957 }; 958 959 struct gaudi2_sm_sei_cause_data { 960 const char *cause_name; 961 const char *log_name; 962 }; 963 964 static const struct gaudi2_sm_sei_cause_data 965 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = { 966 {"calculated SO value overflow/underflow", "SOB ID"}, 967 {"payload address of monitor is not aligned to 4B", "monitor addr"}, 968 {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"}, 969 }; 970 971 static const char * const 972 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = { 973 "LATENCY_RD_OUT_FIFO_OVERRUN", 974 "LATENCY_WR_OUT_FIFO_OVERRUN", 975 }; 976 977 static const char * const 978 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = { 979 "LATENCY_RD_OUT_FIFO_OVERRUN", 980 "LATENCY_WR_OUT_FIFO_OVERRUN", 981 }; 982 983 static const char * const 984 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = { 985 "AXI drain HBW", 986 "AXI drain LBW", 987 }; 988 989 static const char * const 990 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = { 991 "HBW error response", 992 "LBW error response", 993 "TLP is blocked by RR" 994 }; 995 996 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = { 997 [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE, 998 [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE, 999 [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE, 1000 [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE, 1001 [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE, 1002 [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE, 1003 [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE, 1004 [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE, 1005 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE, 1006 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE, 1007 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE, 1008 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE, 1009 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE, 1010 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE, 1011 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE, 1012 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE, 1013 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE, 1014 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE, 1015 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE, 1016 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE, 1017 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE, 1018 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE, 1019 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE, 1020 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE, 1021 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE, 1022 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE, 1023 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE, 1024 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE, 1025 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE, 1026 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE, 1027 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE, 1028 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE, 1029 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE, 1030 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE, 1031 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE, 1032 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE, 1033 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE, 1034 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE, 1035 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE, 1036 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE, 1037 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE, 1038 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE, 1039 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE, 1040 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE, 1041 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE, 1042 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE, 1043 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE, 1044 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE, 1045 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE, 1046 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE, 1047 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE, 1048 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE, 1049 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE, 1050 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE, 1051 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE, 1052 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE, 1053 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE, 1054 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE, 1055 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE, 1056 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE, 1057 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE, 1058 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE, 1059 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE, 1060 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE, 1061 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE, 1062 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE, 1063 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE, 1064 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE, 1065 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE, 1066 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE, 1067 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE, 1068 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE, 1069 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE, 1070 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE, 1071 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE, 1072 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE, 1073 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE, 1074 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE, 1075 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE, 1076 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE, 1077 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE, 1078 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE, 1079 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE, 1080 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE, 1081 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE, 1082 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE, 1083 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE, 1084 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE, 1085 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE, 1086 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE, 1087 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE, 1088 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE, 1089 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE, 1090 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE, 1091 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE, 1092 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE, 1093 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE, 1094 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE, 1095 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE, 1096 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE, 1097 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE, 1098 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE, 1099 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE, 1100 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE, 1101 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE, 1102 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE, 1103 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE, 1104 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE, 1105 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE, 1106 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE, 1107 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE, 1108 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE, 1109 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE, 1110 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE, 1111 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE, 1112 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE, 1113 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE, 1114 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE, 1115 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE, 1116 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE, 1117 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE, 1118 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE, 1119 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE, 1120 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE, 1121 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE, 1122 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE, 1123 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE, 1124 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE, 1125 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE, 1126 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE, 1127 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE, 1128 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE, 1129 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE, 1130 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE, 1131 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE, 1132 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE, 1133 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE, 1134 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE, 1135 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE, 1136 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE, 1137 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE, 1138 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE, 1139 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE, 1140 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE, 1141 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE, 1142 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE, 1143 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE, 1144 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE, 1145 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE, 1146 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE, 1147 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE, 1148 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE, 1149 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE, 1150 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE, 1151 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE, 1152 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE, 1153 [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE, 1154 [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE, 1155 [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE, 1156 [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE, 1157 [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE, 1158 [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE, 1159 [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE, 1160 [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE, 1161 [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE, 1162 [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE, 1163 [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE, 1164 [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE, 1165 [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE, 1166 [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE, 1167 [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE, 1168 [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE, 1169 [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE, 1170 [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE, 1171 [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE, 1172 [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE, 1173 [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE, 1174 [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE, 1175 [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE, 1176 [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE, 1177 [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE, 1178 [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE, 1179 [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE, 1180 [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE, 1181 [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE, 1182 [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE, 1183 [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE, 1184 [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE, 1185 [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE, 1186 [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE, 1187 [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE, 1188 [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE, 1189 [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE, 1190 [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE, 1191 [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE, 1192 [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE, 1193 [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE, 1194 [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE, 1195 [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE, 1196 [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE, 1197 [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE, 1198 [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE, 1199 [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE, 1200 [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE, 1201 [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE, 1202 [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE, 1203 [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE, 1204 [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE, 1205 [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE, 1206 [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE, 1207 [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE, 1208 [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE, 1209 [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE, 1210 [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE, 1211 [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE, 1212 [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE, 1213 [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE, 1214 [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE, 1215 [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE, 1216 [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE, 1217 [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE, 1218 [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE, 1219 [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE, 1220 [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE, 1221 [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE, 1222 [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE, 1223 [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE, 1224 [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE, 1225 [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE, 1226 [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE, 1227 [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE, 1228 [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE, 1229 [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE, 1230 [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE, 1231 [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE, 1232 [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE, 1233 [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE, 1234 [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE, 1235 [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE, 1236 [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE, 1237 [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE, 1238 [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE, 1239 [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE, 1240 [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE, 1241 [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE, 1242 [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE, 1243 [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE, 1244 [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE, 1245 [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE, 1246 [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE, 1247 [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE, 1248 [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE, 1249 [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE, 1250 [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE, 1251 [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE, 1252 [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE, 1253 [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE, 1254 [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE, 1255 [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE, 1256 [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE 1257 }; 1258 1259 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = { 1260 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE, 1261 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE, 1262 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE, 1263 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE, 1264 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE, 1265 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE, 1266 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE, 1267 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE, 1268 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE, 1269 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE, 1270 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE, 1271 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE, 1272 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE, 1273 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE, 1274 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE, 1275 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE, 1276 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE, 1277 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE, 1278 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE, 1279 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE, 1280 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE, 1281 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE, 1282 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE, 1283 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE, 1284 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE, 1285 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE, 1286 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE, 1287 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE, 1288 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE, 1289 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE, 1290 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE, 1291 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE, 1292 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE, 1293 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE, 1294 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE, 1295 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE, 1296 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE, 1297 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE, 1298 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE, 1299 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE, 1300 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE, 1301 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE, 1302 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE, 1303 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE, 1304 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE, 1305 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE, 1306 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE, 1307 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE, 1308 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE, 1309 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE, 1310 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE, 1311 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE, 1312 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE, 1313 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE, 1314 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE, 1315 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE, 1316 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE, 1317 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE, 1318 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE, 1319 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE, 1320 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE, 1321 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE, 1322 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE, 1323 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE, 1324 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE, 1325 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE, 1326 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE, 1327 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE, 1328 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE, 1329 }; 1330 1331 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = { 1332 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE, 1333 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE, 1334 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE, 1335 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE, 1336 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE, 1337 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE, 1338 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE, 1339 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE, 1340 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE, 1341 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE, 1342 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE, 1343 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE, 1344 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE, 1345 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE, 1346 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE, 1347 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE, 1348 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE, 1349 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE, 1350 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE, 1351 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE, 1352 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE, 1353 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE, 1354 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE, 1355 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE, 1356 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE, 1357 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE, 1358 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE, 1359 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE, 1360 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE, 1361 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE, 1362 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE, 1363 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE, 1364 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE, 1365 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE, 1366 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE, 1367 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE, 1368 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE, 1369 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE, 1370 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE, 1371 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE, 1372 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE, 1373 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE, 1374 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE, 1375 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE, 1376 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE, 1377 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE, 1378 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE, 1379 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE, 1380 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE, 1381 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE, 1382 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE, 1383 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE, 1384 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE, 1385 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE, 1386 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE, 1387 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE, 1388 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE, 1389 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE, 1390 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE, 1391 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE, 1392 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE, 1393 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE, 1394 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE, 1395 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE, 1396 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE, 1397 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE, 1398 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE, 1399 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE, 1400 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE, 1401 }; 1402 1403 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = { 1404 [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE, 1405 [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE, 1406 [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE, 1407 [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE, 1408 }; 1409 1410 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = { 1411 [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0, 1412 [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0, 1413 [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0, 1414 [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0, 1415 [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1, 1416 [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1, 1417 [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1, 1418 [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1, 1419 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0, 1420 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0, 1421 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0, 1422 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0, 1423 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1, 1424 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1, 1425 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1, 1426 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1, 1427 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0, 1428 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0, 1429 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0, 1430 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0, 1431 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0, 1432 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0, 1433 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0, 1434 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0, 1435 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1, 1436 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1, 1437 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1, 1438 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1, 1439 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2, 1440 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2, 1441 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2, 1442 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2, 1443 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3, 1444 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3, 1445 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3, 1446 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3, 1447 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4, 1448 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4, 1449 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4, 1450 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4, 1451 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5, 1452 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5, 1453 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5, 1454 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5, 1455 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24, 1456 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24, 1457 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24, 1458 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24, 1459 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2, 1460 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2, 1461 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2, 1462 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2, 1463 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3, 1464 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3, 1465 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3, 1466 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3, 1467 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4, 1468 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4, 1469 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4, 1470 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4, 1471 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6, 1472 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6, 1473 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6, 1474 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6, 1475 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7, 1476 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7, 1477 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7, 1478 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7, 1479 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8, 1480 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8, 1481 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8, 1482 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8, 1483 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9, 1484 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9, 1485 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9, 1486 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9, 1487 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10, 1488 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10, 1489 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10, 1490 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10, 1491 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11, 1492 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11, 1493 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11, 1494 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11, 1495 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4, 1496 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4, 1497 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4, 1498 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4, 1499 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5, 1500 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5, 1501 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5, 1502 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5, 1503 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1, 1504 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1, 1505 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1, 1506 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1, 1507 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12, 1508 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12, 1509 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12, 1510 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12, 1511 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13, 1512 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13, 1513 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13, 1514 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13, 1515 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14, 1516 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14, 1517 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14, 1518 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14, 1519 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15, 1520 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15, 1521 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15, 1522 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15, 1523 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16, 1524 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16, 1525 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16, 1526 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16, 1527 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17, 1528 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17, 1529 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17, 1530 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17, 1531 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6, 1532 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6, 1533 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6, 1534 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6, 1535 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7, 1536 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7, 1537 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7, 1538 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7, 1539 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5, 1540 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5, 1541 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5, 1542 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5, 1543 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18, 1544 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18, 1545 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18, 1546 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18, 1547 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19, 1548 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19, 1549 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19, 1550 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19, 1551 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20, 1552 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20, 1553 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20, 1554 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20, 1555 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21, 1556 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21, 1557 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21, 1558 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21, 1559 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22, 1560 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22, 1561 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22, 1562 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22, 1563 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23, 1564 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23, 1565 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23, 1566 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23, 1567 [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0, 1568 [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0, 1569 [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0, 1570 [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0, 1571 [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1, 1572 [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1, 1573 [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1, 1574 [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1, 1575 [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2, 1576 [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2, 1577 [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2, 1578 [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2, 1579 [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3, 1580 [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3, 1581 [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3, 1582 [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3, 1583 [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4, 1584 [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4, 1585 [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4, 1586 [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4, 1587 [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5, 1588 [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5, 1589 [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5, 1590 [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5, 1591 [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6, 1592 [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6, 1593 [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6, 1594 [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6, 1595 [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7, 1596 [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7, 1597 [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7, 1598 [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7, 1599 [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8, 1600 [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8, 1601 [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8, 1602 [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8, 1603 [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9, 1604 [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9, 1605 [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9, 1606 [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9, 1607 [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10, 1608 [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10, 1609 [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10, 1610 [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10, 1611 [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11, 1612 [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11, 1613 [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11, 1614 [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11, 1615 [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12, 1616 [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12, 1617 [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12, 1618 [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12, 1619 [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13, 1620 [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13, 1621 [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13, 1622 [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13, 1623 [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14, 1624 [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14, 1625 [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14, 1626 [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14, 1627 [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15, 1628 [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15, 1629 [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15, 1630 [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15, 1631 [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16, 1632 [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16, 1633 [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16, 1634 [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16, 1635 [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17, 1636 [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17, 1637 [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17, 1638 [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17, 1639 [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18, 1640 [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18, 1641 [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18, 1642 [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18, 1643 [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19, 1644 [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19, 1645 [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19, 1646 [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19, 1647 [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20, 1648 [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20, 1649 [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20, 1650 [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20, 1651 [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21, 1652 [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21, 1653 [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21, 1654 [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21, 1655 [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22, 1656 [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22, 1657 [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22, 1658 [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22, 1659 [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23, 1660 [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23, 1661 [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23, 1662 [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23, 1663 [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0, 1664 [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0, 1665 [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0, 1666 [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0, 1667 [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1, 1668 [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1, 1669 [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1, 1670 [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1 1671 }; 1672 1673 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = { 1674 [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE, 1675 [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE, 1676 [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE, 1677 [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE, 1678 [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE, 1679 [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE, 1680 [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE, 1681 [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE, 1682 [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE, 1683 [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE, 1684 [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE 1685 }; 1686 1687 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = { 1688 [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE, 1689 [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE, 1690 [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE, 1691 [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE 1692 }; 1693 1694 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = { 1695 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE, 1696 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE, 1697 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE, 1698 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE, 1699 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE, 1700 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE, 1701 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE, 1702 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE, 1703 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE, 1704 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE, 1705 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE, 1706 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE, 1707 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE, 1708 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE, 1709 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE, 1710 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE, 1711 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE, 1712 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE, 1713 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE, 1714 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE, 1715 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE, 1716 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE, 1717 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE, 1718 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE, 1719 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE, 1720 }; 1721 1722 static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = { 1723 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE, 1724 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE, 1725 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE, 1726 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE, 1727 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE, 1728 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE, 1729 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE, 1730 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE, 1731 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE, 1732 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE, 1733 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE, 1734 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE, 1735 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE, 1736 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE, 1737 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE, 1738 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE, 1739 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE, 1740 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE, 1741 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE, 1742 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE, 1743 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE, 1744 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE, 1745 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE, 1746 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE, 1747 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE, 1748 }; 1749 1750 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = { 1751 [ROTATOR_ID_0] = mmROT0_BASE, 1752 [ROTATOR_ID_1] = mmROT1_BASE 1753 }; 1754 1755 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = { 1756 [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0, 1757 [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0, 1758 [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0, 1759 [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0, 1760 [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0, 1761 [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0, 1762 [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0, 1763 [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0, 1764 [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0, 1765 [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0, 1766 [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0, 1767 [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0, 1768 [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0, 1769 [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0, 1770 [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0, 1771 [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0, 1772 [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0, 1773 [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0, 1774 [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0, 1775 [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0, 1776 [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0, 1777 [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0, 1778 [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0, 1779 [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0, 1780 [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0, 1781 }; 1782 1783 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = { 1784 [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0, 1785 [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0, 1786 }; 1787 1788 static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = { 1789 [GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0, 1790 [GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1, 1791 [GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2, 1792 [GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3, 1793 [GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4, 1794 [GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5, 1795 [GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0, 1796 [GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1, 1797 [GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2, 1798 [GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3, 1799 [GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4, 1800 [GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5, 1801 [GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0, 1802 [GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1, 1803 [GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2, 1804 [GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3, 1805 [GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4, 1806 [GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5, 1807 [GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0, 1808 [GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1, 1809 [GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2, 1810 [GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3, 1811 [GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4, 1812 [GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5, 1813 /* the PCI TPC is placed last (mapped liked HW) */ 1814 [GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6, 1815 }; 1816 1817 static const u32 gaudi2_mme_engine_id_to_mme_id[] = { 1818 [GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0, 1819 [GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1, 1820 [GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2, 1821 [GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3, 1822 }; 1823 1824 static const u32 gaudi2_edma_engine_id_to_edma_id[] = { 1825 [GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0, 1826 [GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1, 1827 [GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0, 1828 [GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1, 1829 [GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2, 1830 [GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3, 1831 [GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4, 1832 [GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5, 1833 [GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6, 1834 [GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7, 1835 [GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA, 1836 }; 1837 1838 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 1839 GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 1840 GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0, 1841 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 1842 GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0, 1843 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 1844 GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0, 1845 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0, 1846 GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0, 1847 }; 1848 1849 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = { 1850 "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal", 1851 "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal", 1852 "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal", 1853 "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal", 1854 "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal", 1855 "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal", 1856 "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal", 1857 "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal", 1858 "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal", 1859 "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal" 1860 }; 1861 1862 enum rtr_id { 1863 DCORE0_RTR0, 1864 DCORE0_RTR1, 1865 DCORE0_RTR2, 1866 DCORE0_RTR3, 1867 DCORE0_RTR4, 1868 DCORE0_RTR5, 1869 DCORE0_RTR6, 1870 DCORE0_RTR7, 1871 DCORE1_RTR0, 1872 DCORE1_RTR1, 1873 DCORE1_RTR2, 1874 DCORE1_RTR3, 1875 DCORE1_RTR4, 1876 DCORE1_RTR5, 1877 DCORE1_RTR6, 1878 DCORE1_RTR7, 1879 DCORE2_RTR0, 1880 DCORE2_RTR1, 1881 DCORE2_RTR2, 1882 DCORE2_RTR3, 1883 DCORE2_RTR4, 1884 DCORE2_RTR5, 1885 DCORE2_RTR6, 1886 DCORE2_RTR7, 1887 DCORE3_RTR0, 1888 DCORE3_RTR1, 1889 DCORE3_RTR2, 1890 DCORE3_RTR3, 1891 DCORE3_RTR4, 1892 DCORE3_RTR5, 1893 DCORE3_RTR6, 1894 DCORE3_RTR7, 1895 }; 1896 1897 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 1898 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3, 1899 DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4, 1900 DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, 1901 DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, 1902 DCORE0_RTR0 1903 }; 1904 1905 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 1906 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, 1907 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, 1908 DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0, 1909 DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7, 1910 DCORE0_RTR0 1911 }; 1912 1913 static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = { 1914 DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0, 1915 DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0 1916 }; 1917 1918 static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = { 1919 DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1, 1920 DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0 1921 }; 1922 1923 static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 1924 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 1925 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 1926 }; 1927 1928 static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 1929 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 1930 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 1931 }; 1932 1933 static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 1934 mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 1935 mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 1936 mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 1937 mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 1938 mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 1939 mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 1940 mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 1941 mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE 1942 }; 1943 1944 static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = { 1945 DCORE0_RTR0, DCORE0_RTR0 1946 }; 1947 1948 static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = { 1949 DCORE0_RTR2, DCORE0_RTR2 1950 }; 1951 1952 static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = { 1953 DCORE2_RTR0, DCORE3_RTR7 1954 }; 1955 1956 static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = { 1957 DCORE2_RTR2, DCORE3_RTR5 1958 }; 1959 1960 struct mme_initiators_rtr_id { 1961 u32 wap0; 1962 u32 wap1; 1963 u32 write; 1964 u32 read; 1965 u32 sbte0; 1966 u32 sbte1; 1967 u32 sbte2; 1968 u32 sbte3; 1969 u32 sbte4; 1970 }; 1971 1972 enum mme_initiators { 1973 MME_WAP0 = 0, 1974 MME_WAP1, 1975 MME_WRITE, 1976 MME_READ, 1977 MME_SBTE0, 1978 MME_SBTE1, 1979 MME_SBTE2, 1980 MME_SBTE3, 1981 MME_SBTE4, 1982 MME_INITIATORS_MAX 1983 }; 1984 1985 static const struct mme_initiators_rtr_id 1986 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = { 1987 { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7, 1988 .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6}, 1989 { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8, 1990 .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8}, 1991 { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23, 1992 .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23}, 1993 { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30, 1994 .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28}, 1995 }; 1996 1997 enum razwi_event_sources { 1998 RAZWI_TPC, 1999 RAZWI_MME, 2000 RAZWI_EDMA, 2001 RAZWI_PDMA, 2002 RAZWI_NIC, 2003 RAZWI_DEC, 2004 RAZWI_ROT 2005 }; 2006 2007 struct hbm_mc_error_causes { 2008 u32 mask; 2009 char cause[50]; 2010 }; 2011 2012 static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS; 2013 2014 /* Special blocks iterator is currently used to configure security protection bits, 2015 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)- 2016 * must be skipped. Following configurations are commonly used for both PB config 2017 * and global error reading, since currently they both share the same settings. 2018 * Once it changes, we must remember to use separate configurations for either one. 2019 */ 2020 static int gaudi2_iterator_skip_block_types[] = { 2021 GAUDI2_BLOCK_TYPE_PLL, 2022 GAUDI2_BLOCK_TYPE_EU_BIST, 2023 GAUDI2_BLOCK_TYPE_HBM, 2024 GAUDI2_BLOCK_TYPE_XFT 2025 }; 2026 2027 static struct range gaudi2_iterator_skip_block_ranges[] = { 2028 /* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */ 2029 {mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE}, 2030 {mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE}, 2031 /* Skip all CPU blocks except for CPU_IF */ 2032 {mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE}, 2033 {mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE} 2034 }; 2035 2036 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = { 2037 {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"}, 2038 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"}, 2039 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"}, 2040 {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"}, 2041 {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"}, 2042 }; 2043 2044 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = { 2045 [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even", 2046 [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd", 2047 [HBM_SEI_READ_ERR] = "SEI read data error", 2048 [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error", 2049 [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted", 2050 [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail", 2051 [HBM_SEI_DFI] = "SEI DFI error", 2052 [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read", 2053 [HBM_SEI_BIST_FAIL] = "SEI BIST fail" 2054 }; 2055 2056 struct mmu_spi_sei_cause { 2057 char cause[50]; 2058 int clear_bit; 2059 }; 2060 2061 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = { 2062 {"page fault", 1}, /* INTERRUPT_CLR[1] */ 2063 {"page access", 1}, /* INTERRUPT_CLR[1] */ 2064 {"bypass ddr", 2}, /* INTERRUPT_CLR[2] */ 2065 {"multi hit", 2}, /* INTERRUPT_CLR[2] */ 2066 {"mmu rei0", -1}, /* no clear register bit */ 2067 {"mmu rei1", -1}, /* no clear register bit */ 2068 {"stlb rei0", -1}, /* no clear register bit */ 2069 {"stlb rei1", -1}, /* no clear register bit */ 2070 {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */ 2071 {"rr privileged read hit", 2}, /* INTERRUPT_CLR[2] */ 2072 {"rr secure write hit", 2}, /* INTERRUPT_CLR[2] */ 2073 {"rr secure read hit", 2}, /* INTERRUPT_CLR[2] */ 2074 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2075 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2076 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2077 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2078 {"slave error", 16}, /* INTERRUPT_CLR[16] */ 2079 {"dec error", 17}, /* INTERRUPT_CLR[17] */ 2080 {"burst fifo full", 2} /* INTERRUPT_CLR[2] */ 2081 }; 2082 2083 struct gaudi2_cache_invld_params { 2084 u64 start_va; 2085 u64 end_va; 2086 u32 inv_start_val; 2087 u32 flags; 2088 bool range_invalidation; 2089 }; 2090 2091 struct gaudi2_tpc_idle_data { 2092 struct engines_data *e; 2093 unsigned long *mask; 2094 bool *is_idle; 2095 const char *tpc_fmt; 2096 }; 2097 2098 struct gaudi2_tpc_mmu_data { 2099 u32 rw_asid; 2100 }; 2101 2102 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0}; 2103 2104 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val); 2105 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id); 2106 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id); 2107 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2108 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2109 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val); 2110 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, 2111 bool is_memset); 2112 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2113 struct engines_data *e); 2114 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2115 struct engines_data *e); 2116 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2117 struct engines_data *e); 2118 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr); 2119 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr); 2120 gaudi2_init_scrambler_hbm(struct hl_device * hdev)2121 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) 2122 { 2123 2124 } 2125 gaudi2_get_signal_cb_size(struct hl_device * hdev)2126 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) 2127 { 2128 return sizeof(struct packet_msg_short); 2129 } 2130 gaudi2_get_wait_cb_size(struct hl_device * hdev)2131 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) 2132 { 2133 return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence); 2134 } 2135 gaudi2_iterate_tpcs(struct hl_device * hdev,struct iterate_module_ctx * ctx)2136 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) 2137 { 2138 struct asic_fixed_properties *prop = &hdev->asic_prop; 2139 int dcore, inst, tpc_seq; 2140 u32 offset; 2141 2142 /* init the return code */ 2143 ctx->rc = 0; 2144 2145 for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) { 2146 for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) { 2147 tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 2148 2149 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) 2150 continue; 2151 2152 offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst); 2153 2154 ctx->fn(hdev, dcore, inst, offset, ctx); 2155 if (ctx->rc) { 2156 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", 2157 dcore, inst); 2158 return; 2159 } 2160 } 2161 } 2162 2163 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) 2164 return; 2165 2166 /* special check for PCI TPC (DCORE0_TPC6) */ 2167 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); 2168 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); 2169 if (ctx->rc) 2170 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); 2171 } 2172 gaudi2_host_phys_addr_valid(u64 addr)2173 static bool gaudi2_host_phys_addr_valid(u64 addr) 2174 { 2175 if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1)) 2176 return true; 2177 2178 return false; 2179 } 2180 set_number_of_functional_hbms(struct hl_device * hdev)2181 static int set_number_of_functional_hbms(struct hl_device *hdev) 2182 { 2183 struct asic_fixed_properties *prop = &hdev->asic_prop; 2184 u8 faulty_hbms = hweight64(hdev->dram_binning); 2185 2186 /* check if all HBMs should be used */ 2187 if (!faulty_hbms) { 2188 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); 2189 prop->num_functional_hbms = GAUDI2_HBM_NUM; 2190 return 0; 2191 } 2192 2193 /* 2194 * check for error condition in which number of binning 2195 * candidates is higher than the maximum supported by the 2196 * driver (in which case binning mask shall be ignored and driver will 2197 * set the default) 2198 */ 2199 if (faulty_hbms > MAX_FAULTY_HBMS) { 2200 dev_err(hdev->dev, 2201 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", 2202 MAX_FAULTY_HBMS, hdev->dram_binning); 2203 return -EINVAL; 2204 } 2205 2206 /* 2207 * by default, number of functional HBMs in Gaudi2 is always 2208 * GAUDI2_HBM_NUM - 1. 2209 */ 2210 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; 2211 return 0; 2212 } 2213 gaudi2_set_dram_properties(struct hl_device * hdev)2214 static int gaudi2_set_dram_properties(struct hl_device *hdev) 2215 { 2216 struct asic_fixed_properties *prop = &hdev->asic_prop; 2217 u32 basic_hbm_page_size; 2218 int rc; 2219 2220 rc = set_number_of_functional_hbms(hdev); 2221 if (rc) 2222 return -EINVAL; 2223 2224 /* 2225 * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround 2226 * in which we are using x16 bigger page size to be able to populate the entire 2227 * HBM mappings in the TLB 2228 */ 2229 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; 2230 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; 2231 prop->device_mem_alloc_default_page_size = prop->dram_page_size; 2232 prop->dram_size = prop->num_functional_hbms * SZ_16G; 2233 prop->dram_base_address = DRAM_PHYS_BASE; 2234 prop->dram_end_address = prop->dram_base_address + prop->dram_size; 2235 prop->dram_supports_virtual_memory = true; 2236 2237 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; 2238 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; 2239 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; 2240 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; 2241 2242 /* since DRAM page size differs from DMMU page size we need to allocate 2243 * DRAM memory in units of dram_page size and mapping this memory in 2244 * units of DMMU page size. we overcome this size mismatch using a 2245 * scrambling routine which takes a DRAM page and converts it to a DMMU 2246 * page. 2247 * We therefore: 2248 * 1. partition the virtual address space to DRAM-page (whole) pages. 2249 * (suppose we get n such pages) 2250 * 2. limit the amount of virtual address space we got from 1 above to 2251 * a multiple of 64M as we don't want the scrambled address to cross 2252 * the DRAM virtual address space. 2253 * ( m = (n * DRAM_page_size) / DMMU_page_size). 2254 * 3. determine the and address accordingly 2255 * end_addr = start_addr + m * 48M 2256 * 2257 * the DRAM address MSBs (63:48) are not part of the roundup calculation 2258 */ 2259 prop->dmmu.start_addr = prop->dram_base_address + 2260 (prop->dram_page_size * 2261 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); 2262 2263 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * 2264 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); 2265 2266 return 0; 2267 } 2268 gaudi2_set_fixed_properties(struct hl_device * hdev)2269 static int gaudi2_set_fixed_properties(struct hl_device *hdev) 2270 { 2271 struct asic_fixed_properties *prop = &hdev->asic_prop; 2272 struct hw_queue_properties *q_props; 2273 u32 num_sync_stream_queues = 0; 2274 int i; 2275 2276 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; 2277 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), 2278 GFP_KERNEL); 2279 2280 if (!prop->hw_queues_props) 2281 return -ENOMEM; 2282 2283 q_props = prop->hw_queues_props; 2284 2285 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2286 q_props[i].type = QUEUE_TYPE_HW; 2287 q_props[i].driver_only = 0; 2288 2289 if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) { 2290 q_props[i].supports_sync_stream = 0; 2291 } else { 2292 q_props[i].supports_sync_stream = 1; 2293 num_sync_stream_queues++; 2294 } 2295 2296 q_props[i].cb_alloc_flags = CB_ALLOC_USER; 2297 } 2298 2299 q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU; 2300 q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1; 2301 q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL; 2302 2303 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; 2304 prop->cfg_base_address = CFG_BASE; 2305 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; 2306 prop->host_base_address = HOST_PHYS_BASE_0; 2307 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; 2308 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; 2309 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; 2310 prop->user_dec_intr_count = NUMBER_OF_DEC; 2311 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; 2312 prop->completion_mode = HL_COMPLETION_MODE_CS; 2313 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; 2314 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; 2315 2316 prop->sram_base_address = SRAM_BASE_ADDR; 2317 prop->sram_size = SRAM_SIZE; 2318 prop->sram_end_address = prop->sram_base_address + prop->sram_size; 2319 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; 2320 2321 prop->hints_range_reservation = true; 2322 2323 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; 2324 2325 if (hdev->pldm) 2326 prop->mmu_pgt_size = 0x800000; /* 8MB */ 2327 else 2328 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE; 2329 2330 prop->mmu_pte_size = HL_PTE_SIZE; 2331 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; 2332 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; 2333 2334 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; 2335 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; 2336 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; 2337 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; 2338 prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT; 2339 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; 2340 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; 2341 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; 2342 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; 2343 prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK; 2344 prop->dmmu.page_size = PAGE_SIZE_1GB; 2345 prop->dmmu.num_hops = MMU_ARCH_6_HOPS; 2346 prop->dmmu.last_mask = LAST_MASK; 2347 prop->dmmu.host_resident = 1; 2348 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; 2349 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2350 2351 /* 2352 * this is done in order to be able to validate FW descriptor (i.e. validating that 2353 * the addresses and allocated space for FW image does not cross memory bounds). 2354 * for this reason we set the DRAM size to the minimum possible and later it will 2355 * be modified according to what reported in the cpucp info packet 2356 */ 2357 prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G; 2358 2359 hdev->pmmu_huge_range = true; 2360 prop->pmmu.host_resident = 1; 2361 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; 2362 prop->pmmu.last_mask = LAST_MASK; 2363 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; 2364 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2365 2366 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; 2367 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; 2368 prop->hints_host_hpage_reserved_va_range.start_addr = 2369 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START; 2370 prop->hints_host_hpage_reserved_va_range.end_addr = 2371 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END; 2372 2373 if (PAGE_SIZE == SZ_64K) { 2374 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; 2375 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; 2376 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; 2377 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; 2378 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; 2379 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; 2380 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; 2381 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; 2382 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; 2383 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; 2384 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; 2385 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; 2386 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2387 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2388 prop->pmmu.page_size = PAGE_SIZE_64KB; 2389 2390 /* shifts and masks are the same in PMMU and HPMMU */ 2391 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2392 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; 2393 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2394 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2395 } else { 2396 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; 2397 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; 2398 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; 2399 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; 2400 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; 2401 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; 2402 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; 2403 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; 2404 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; 2405 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; 2406 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; 2407 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; 2408 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2409 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2410 prop->pmmu.page_size = PAGE_SIZE_4KB; 2411 2412 /* shifts and masks are the same in PMMU and HPMMU */ 2413 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2414 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 2415 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2416 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2417 } 2418 2419 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; 2420 prop->num_engine_cores = CPU_ID_MAX; 2421 prop->cfg_size = CFG_SIZE; 2422 prop->max_asid = MAX_ASID; 2423 prop->num_of_events = GAUDI2_EVENT_SIZE; 2424 2425 prop->supports_engine_modes = true; 2426 2427 prop->dc_power_default = DC_POWER_DEFAULT; 2428 2429 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; 2430 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; 2431 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; 2432 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 2433 2434 strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); 2435 2436 prop->mme_master_slave_mode = 1; 2437 2438 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + 2439 (num_sync_stream_queues * HL_RSVD_SOBS); 2440 2441 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + 2442 (num_sync_stream_queues * HL_RSVD_MONS); 2443 2444 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; 2445 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; 2446 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; 2447 2448 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; 2449 2450 prop->fw_cpu_boot_dev_sts0_valid = false; 2451 prop->fw_cpu_boot_dev_sts1_valid = false; 2452 prop->hard_reset_done_by_fw = false; 2453 prop->gic_interrupts_enable = true; 2454 2455 prop->server_type = HL_SERVER_TYPE_UNKNOWN; 2456 2457 prop->max_dec = NUMBER_OF_DEC; 2458 2459 prop->clk_pll_index = HL_GAUDI2_MME_PLL; 2460 2461 prop->dma_mask = 64; 2462 2463 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; 2464 2465 return 0; 2466 } 2467 gaudi2_pci_bars_map(struct hl_device * hdev)2468 static int gaudi2_pci_bars_map(struct hl_device *hdev) 2469 { 2470 static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"}; 2471 bool is_wc[3] = {false, false, true}; 2472 int rc; 2473 2474 rc = hl_pci_bars_map(hdev, name, is_wc); 2475 if (rc) 2476 return rc; 2477 2478 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); 2479 2480 return 0; 2481 } 2482 gaudi2_set_hbm_bar_base(struct hl_device * hdev,u64 addr)2483 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) 2484 { 2485 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2486 struct hl_inbound_pci_region pci_region; 2487 u64 old_addr = addr; 2488 int rc; 2489 2490 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) 2491 return old_addr; 2492 2493 if (hdev->asic_prop.iatu_done_by_fw) 2494 return U64_MAX; 2495 2496 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2497 pci_region.mode = PCI_BAR_MATCH_MODE; 2498 pci_region.bar = DRAM_BAR_ID; 2499 pci_region.addr = addr; 2500 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region); 2501 if (rc) 2502 return U64_MAX; 2503 2504 if (gaudi2) { 2505 old_addr = gaudi2->dram_bar_cur_addr; 2506 gaudi2->dram_bar_cur_addr = addr; 2507 } 2508 2509 return old_addr; 2510 } 2511 gaudi2_init_iatu(struct hl_device * hdev)2512 static int gaudi2_init_iatu(struct hl_device *hdev) 2513 { 2514 struct hl_inbound_pci_region inbound_region; 2515 struct hl_outbound_pci_region outbound_region; 2516 u32 bar_addr_low, bar_addr_high; 2517 int rc; 2518 2519 if (hdev->asic_prop.iatu_done_by_fw) 2520 return 0; 2521 2522 /* Temporary inbound Region 0 - Bar 0 - Point to CFG 2523 * We must map this region in BAR match mode in order to 2524 * fetch BAR physical base address 2525 */ 2526 inbound_region.mode = PCI_BAR_MATCH_MODE; 2527 inbound_region.bar = SRAM_CFG_BAR_ID; 2528 /* Base address must be aligned to Bar size which is 256 MB */ 2529 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; 2530 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2531 if (rc) 2532 return rc; 2533 2534 /* Fetch physical BAR address */ 2535 bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF); 2536 bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF; 2537 2538 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; 2539 2540 /* Inbound Region 0 - Bar 0 - Point to CFG */ 2541 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2542 inbound_region.bar = SRAM_CFG_BAR_ID; 2543 inbound_region.offset_in_bar = 0; 2544 inbound_region.addr = STM_FLASH_BASE_ADDR; 2545 inbound_region.size = CFG_REGION_SIZE; 2546 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2547 if (rc) 2548 return rc; 2549 2550 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ 2551 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2552 inbound_region.bar = SRAM_CFG_BAR_ID; 2553 inbound_region.offset_in_bar = CFG_REGION_SIZE; 2554 inbound_region.addr = BAR0_RSRVD_BASE_ADDR; 2555 inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE; 2556 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 2557 if (rc) 2558 return rc; 2559 2560 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2561 inbound_region.mode = PCI_BAR_MATCH_MODE; 2562 inbound_region.bar = DRAM_BAR_ID; 2563 inbound_region.addr = DRAM_PHYS_BASE; 2564 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region); 2565 if (rc) 2566 return rc; 2567 2568 /* Outbound Region 0 - Point to Host */ 2569 outbound_region.addr = HOST_PHYS_BASE_0; 2570 outbound_region.size = HOST_PHYS_SIZE_0; 2571 rc = hl_pci_set_outbound_region(hdev, &outbound_region); 2572 2573 return rc; 2574 } 2575 gaudi2_get_hw_state(struct hl_device * hdev)2576 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) 2577 { 2578 return RREG32(mmHW_STATE); 2579 } 2580 gaudi2_tpc_binning_init_prop(struct hl_device * hdev)2581 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) 2582 { 2583 struct asic_fixed_properties *prop = &hdev->asic_prop; 2584 2585 /* 2586 * check for error condition in which number of binning candidates 2587 * is higher than the maximum supported by the driver 2588 */ 2589 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { 2590 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", 2591 MAX_CLUSTER_BINNING_FAULTY_TPCS, 2592 hdev->tpc_binning); 2593 return -EINVAL; 2594 } 2595 2596 prop->tpc_binning_mask = hdev->tpc_binning; 2597 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; 2598 2599 return 0; 2600 } 2601 gaudi2_set_tpc_binning_masks(struct hl_device * hdev)2602 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) 2603 { 2604 struct asic_fixed_properties *prop = &hdev->asic_prop; 2605 struct hw_queue_properties *q_props = prop->hw_queues_props; 2606 u64 tpc_binning_mask; 2607 u8 subst_idx = 0; 2608 int i, rc; 2609 2610 rc = gaudi2_tpc_binning_init_prop(hdev); 2611 if (rc) 2612 return rc; 2613 2614 tpc_binning_mask = prop->tpc_binning_mask; 2615 2616 for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) { 2617 u8 subst_seq, binned, qid_base; 2618 2619 if (tpc_binning_mask == 0) 2620 break; 2621 2622 if (subst_idx == 0) { 2623 subst_seq = TPC_ID_DCORE0_TPC6; 2624 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 2625 } else { 2626 subst_seq = TPC_ID_DCORE3_TPC5; 2627 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0; 2628 } 2629 2630 2631 /* clear bit from mask */ 2632 binned = __ffs(tpc_binning_mask); 2633 /* 2634 * Coverity complains about possible out-of-bound access in 2635 * clear_bit 2636 */ 2637 if (binned >= TPC_ID_SIZE) { 2638 dev_err(hdev->dev, 2639 "Invalid binned TPC (binning mask: %llx)\n", 2640 tpc_binning_mask); 2641 return -EINVAL; 2642 } 2643 clear_bit(binned, (unsigned long *)&tpc_binning_mask); 2644 2645 /* also clear replacing TPC bit from enabled mask */ 2646 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); 2647 2648 /* bin substite TPC's Qs */ 2649 q_props[qid_base].binned = 1; 2650 q_props[qid_base + 1].binned = 1; 2651 q_props[qid_base + 2].binned = 1; 2652 q_props[qid_base + 3].binned = 1; 2653 2654 subst_idx++; 2655 } 2656 2657 return 0; 2658 } 2659 gaudi2_set_dec_binning_masks(struct hl_device * hdev)2660 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) 2661 { 2662 struct asic_fixed_properties *prop = &hdev->asic_prop; 2663 u8 num_faulty; 2664 2665 num_faulty = hweight32(hdev->decoder_binning); 2666 2667 /* 2668 * check for error condition in which number of binning candidates 2669 * is higher than the maximum supported by the driver 2670 */ 2671 if (num_faulty > MAX_FAULTY_DECODERS) { 2672 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n", 2673 hdev->decoder_binning); 2674 return -EINVAL; 2675 } 2676 2677 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); 2678 2679 if (prop->decoder_binning_mask) 2680 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); 2681 else 2682 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; 2683 2684 return 0; 2685 } 2686 gaudi2_set_dram_binning_masks(struct hl_device * hdev)2687 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) 2688 { 2689 struct asic_fixed_properties *prop = &hdev->asic_prop; 2690 2691 /* check if we should override default binning */ 2692 if (!hdev->dram_binning) { 2693 prop->dram_binning_mask = 0; 2694 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; 2695 return; 2696 } 2697 2698 /* set DRAM binning constraints */ 2699 prop->faulty_dram_cluster_map |= hdev->dram_binning; 2700 prop->dram_binning_mask = hdev->dram_binning; 2701 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); 2702 } 2703 gaudi2_set_edma_binning_masks(struct hl_device * hdev)2704 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) 2705 { 2706 struct asic_fixed_properties *prop = &hdev->asic_prop; 2707 struct hw_queue_properties *q_props; 2708 u8 seq, num_faulty; 2709 2710 num_faulty = hweight32(hdev->edma_binning); 2711 2712 /* 2713 * check for error condition in which number of binning candidates 2714 * is higher than the maximum supported by the driver 2715 */ 2716 if (num_faulty > MAX_FAULTY_EDMAS) { 2717 dev_err(hdev->dev, 2718 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", 2719 hdev->edma_binning); 2720 return -EINVAL; 2721 } 2722 2723 if (!hdev->edma_binning) { 2724 prop->edma_binning_mask = 0; 2725 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; 2726 return 0; 2727 } 2728 2729 seq = __ffs((unsigned long)hdev->edma_binning); 2730 2731 /* set binning constraints */ 2732 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); 2733 prop->edma_binning_mask = hdev->edma_binning; 2734 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); 2735 2736 /* bin substitute EDMA's queue */ 2737 q_props = prop->hw_queues_props; 2738 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1; 2739 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1; 2740 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1; 2741 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1; 2742 2743 return 0; 2744 } 2745 gaudi2_set_xbar_edge_enable_mask(struct hl_device * hdev,u32 xbar_edge_iso_mask)2746 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) 2747 { 2748 struct asic_fixed_properties *prop = &hdev->asic_prop; 2749 u8 num_faulty, seq; 2750 2751 /* check if we should override default binning */ 2752 if (!xbar_edge_iso_mask) { 2753 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; 2754 return 0; 2755 } 2756 2757 /* 2758 * note that it can be set to value other than 0 only after cpucp packet (i.e. 2759 * only the FW can set a redundancy value). for user it'll always be 0. 2760 */ 2761 num_faulty = hweight32(xbar_edge_iso_mask); 2762 2763 /* 2764 * check for error condition in which number of binning candidates 2765 * is higher than the maximum supported by the driver 2766 */ 2767 if (num_faulty > MAX_FAULTY_XBARS) { 2768 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", 2769 MAX_FAULTY_XBARS); 2770 return -EINVAL; 2771 } 2772 2773 seq = __ffs((unsigned long)xbar_edge_iso_mask); 2774 2775 /* set binning constraints */ 2776 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); 2777 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; 2778 2779 return 0; 2780 } 2781 gaudi2_set_cluster_binning_masks_common(struct hl_device * hdev,u8 xbar_edge_iso_mask)2782 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) 2783 { 2784 int rc; 2785 2786 /* 2787 * mark all clusters as good, each component will "fail" cluster 2788 * based on eFuse/user values. 2789 * If more than single cluster is faulty- the chip is unusable 2790 */ 2791 hdev->asic_prop.faulty_dram_cluster_map = 0; 2792 2793 gaudi2_set_dram_binning_masks(hdev); 2794 2795 rc = gaudi2_set_edma_binning_masks(hdev); 2796 if (rc) 2797 return rc; 2798 2799 rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask); 2800 if (rc) 2801 return rc; 2802 2803 2804 /* always initially set to full mask */ 2805 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; 2806 2807 return 0; 2808 } 2809 gaudi2_set_cluster_binning_masks(struct hl_device * hdev)2810 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) 2811 { 2812 struct asic_fixed_properties *prop = &hdev->asic_prop; 2813 int rc; 2814 2815 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); 2816 if (rc) 2817 return rc; 2818 2819 /* if we have DRAM binning reported by FW we should perform cluster config */ 2820 if (prop->faulty_dram_cluster_map) { 2821 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); 2822 2823 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; 2824 } 2825 2826 return 0; 2827 } 2828 gaudi2_set_binning_masks(struct hl_device * hdev)2829 static int gaudi2_set_binning_masks(struct hl_device *hdev) 2830 { 2831 int rc; 2832 2833 rc = gaudi2_set_cluster_binning_masks(hdev); 2834 if (rc) 2835 return rc; 2836 2837 rc = gaudi2_set_tpc_binning_masks(hdev); 2838 if (rc) 2839 return rc; 2840 2841 rc = gaudi2_set_dec_binning_masks(hdev); 2842 if (rc) 2843 return rc; 2844 2845 return 0; 2846 } 2847 gaudi2_cpucp_info_get(struct hl_device * hdev)2848 static int gaudi2_cpucp_info_get(struct hl_device *hdev) 2849 { 2850 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2851 struct asic_fixed_properties *prop = &hdev->asic_prop; 2852 long max_power; 2853 u64 dram_size; 2854 int rc; 2855 2856 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 2857 return 0; 2858 2859 /* No point of asking this information again when not doing hard reset, as the device 2860 * CPU hasn't been reset 2861 */ 2862 if (hdev->reset_info.in_compute_reset) 2863 return 0; 2864 2865 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 2866 mmCPU_BOOT_ERR1); 2867 if (rc) 2868 return rc; 2869 2870 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); 2871 if (dram_size) { 2872 /* we can have wither 5 or 6 HBMs. other values are invalid */ 2873 2874 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && 2875 (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) { 2876 dev_err(hdev->dev, 2877 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n", 2878 dram_size, prop->dram_size); 2879 dram_size = prop->dram_size; 2880 } 2881 2882 prop->dram_size = dram_size; 2883 prop->dram_end_address = prop->dram_base_address + dram_size; 2884 } 2885 2886 if (!strlen(prop->cpucp_info.card_name)) 2887 strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); 2888 2889 /* Overwrite binning masks with the actual binning values from F/W */ 2890 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; 2891 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; 2892 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); 2893 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); 2894 2895 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", 2896 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, 2897 hdev->decoder_binning); 2898 2899 /* 2900 * at this point the DRAM parameters need to be updated according to data obtained 2901 * from the FW 2902 */ 2903 rc = hdev->asic_funcs->set_dram_properties(hdev); 2904 if (rc) 2905 return rc; 2906 2907 rc = hdev->asic_funcs->set_binning_masks(hdev); 2908 if (rc) 2909 return rc; 2910 2911 max_power = hl_fw_get_max_power(hdev); 2912 if (max_power < 0) 2913 return max_power; 2914 2915 prop->max_power_default = (u64) max_power; 2916 2917 return 0; 2918 } 2919 gaudi2_fetch_psoc_frequency(struct hl_device * hdev)2920 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) 2921 { 2922 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2923 u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS]; 2924 int rc; 2925 2926 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 2927 return 0; 2928 2929 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr); 2930 if (rc) 2931 return rc; 2932 2933 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; 2934 2935 return 0; 2936 } 2937 gaudi2_early_init(struct hl_device * hdev)2938 static int gaudi2_early_init(struct hl_device *hdev) 2939 { 2940 struct asic_fixed_properties *prop = &hdev->asic_prop; 2941 struct pci_dev *pdev = hdev->pdev; 2942 resource_size_t pci_bar_size; 2943 int rc; 2944 2945 rc = gaudi2_set_fixed_properties(hdev); 2946 if (rc) 2947 return rc; 2948 2949 /* Check BAR sizes */ 2950 pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID); 2951 2952 if (pci_bar_size != CFG_BAR_SIZE) { 2953 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 2954 SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 2955 rc = -ENODEV; 2956 goto free_queue_props; 2957 } 2958 2959 pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID); 2960 if (pci_bar_size != MSIX_BAR_SIZE) { 2961 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 2962 MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE); 2963 rc = -ENODEV; 2964 goto free_queue_props; 2965 } 2966 2967 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); 2968 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); 2969 2970 /* 2971 * Only in pldm driver config iATU 2972 */ 2973 if (hdev->pldm) 2974 hdev->asic_prop.iatu_done_by_fw = false; 2975 else 2976 hdev->asic_prop.iatu_done_by_fw = true; 2977 2978 rc = hl_pci_init(hdev); 2979 if (rc) 2980 goto free_queue_props; 2981 2982 /* Before continuing in the initialization, we need to read the preboot 2983 * version to determine whether we run with a security-enabled firmware 2984 */ 2985 rc = hl_fw_read_preboot_status(hdev); 2986 if (rc) { 2987 if (hdev->reset_on_preboot_fail) 2988 /* we are already on failure flow, so don't check if hw_fini fails. */ 2989 hdev->asic_funcs->hw_fini(hdev, true, false); 2990 goto pci_fini; 2991 } 2992 2993 if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 2994 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 2995 rc = hdev->asic_funcs->hw_fini(hdev, true, false); 2996 if (rc) { 2997 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); 2998 goto pci_fini; 2999 } 3000 } 3001 3002 return 0; 3003 3004 pci_fini: 3005 hl_pci_fini(hdev); 3006 free_queue_props: 3007 kfree(hdev->asic_prop.hw_queues_props); 3008 return rc; 3009 } 3010 gaudi2_early_fini(struct hl_device * hdev)3011 static int gaudi2_early_fini(struct hl_device *hdev) 3012 { 3013 kfree(hdev->asic_prop.hw_queues_props); 3014 hl_pci_fini(hdev); 3015 3016 return 0; 3017 } 3018 gaudi2_is_arc_nic_owned(u64 arc_id)3019 static bool gaudi2_is_arc_nic_owned(u64 arc_id) 3020 { 3021 switch (arc_id) { 3022 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 3023 return true; 3024 default: 3025 return false; 3026 } 3027 } 3028 gaudi2_is_arc_tpc_owned(u64 arc_id)3029 static bool gaudi2_is_arc_tpc_owned(u64 arc_id) 3030 { 3031 switch (arc_id) { 3032 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 3033 return true; 3034 default: 3035 return false; 3036 } 3037 } 3038 gaudi2_init_arcs(struct hl_device * hdev)3039 static void gaudi2_init_arcs(struct hl_device *hdev) 3040 { 3041 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3042 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3043 u64 arc_id; 3044 u32 i; 3045 3046 for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) { 3047 if (gaudi2_is_arc_enabled(hdev, i)) 3048 continue; 3049 3050 gaudi2_set_arc_id_cap(hdev, i); 3051 } 3052 3053 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 3054 if (!gaudi2_is_queue_enabled(hdev, i)) 3055 continue; 3056 3057 arc_id = gaudi2_queue_id_to_arc_id[i]; 3058 if (gaudi2_is_arc_enabled(hdev, arc_id)) 3059 continue; 3060 3061 if (gaudi2_is_arc_nic_owned(arc_id) && 3062 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) 3063 continue; 3064 3065 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & 3066 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) 3067 continue; 3068 3069 gaudi2_set_arc_id_cap(hdev, arc_id); 3070 } 3071 3072 /* Fetch ARC scratchpad address */ 3073 hdev->asic_prop.engine_core_interrupt_reg_addr = 3074 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); 3075 } 3076 gaudi2_scrub_arc_dccm(struct hl_device * hdev,u32 cpu_id)3077 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) 3078 { 3079 u32 reg_base, reg_val; 3080 int rc; 3081 3082 switch (cpu_id) { 3083 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3: 3084 /* Each ARC scheduler has 2 consecutive DCCM blocks */ 3085 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3086 ARC_DCCM_BLOCK_SIZE * 2, true); 3087 if (rc) 3088 return rc; 3089 break; 3090 case CPU_ID_SCHED_ARC4: 3091 case CPU_ID_SCHED_ARC5: 3092 case CPU_ID_MME_QMAN_ARC0: 3093 case CPU_ID_MME_QMAN_ARC1: 3094 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 3095 3096 /* Scrub lower DCCM block */ 3097 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3098 ARC_DCCM_BLOCK_SIZE, true); 3099 if (rc) 3100 return rc; 3101 3102 /* Switch to upper DCCM block */ 3103 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1); 3104 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3105 3106 /* Scrub upper DCCM block */ 3107 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3108 ARC_DCCM_BLOCK_SIZE, true); 3109 if (rc) 3110 return rc; 3111 3112 /* Switch to lower DCCM block */ 3113 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0); 3114 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3115 break; 3116 default: 3117 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3118 ARC_DCCM_BLOCK_SIZE, true); 3119 if (rc) 3120 return rc; 3121 } 3122 3123 return 0; 3124 } 3125 gaudi2_scrub_arcs_dccm(struct hl_device * hdev)3126 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) 3127 { 3128 u16 arc_id; 3129 int rc; 3130 3131 for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) { 3132 if (!gaudi2_is_arc_enabled(hdev, arc_id)) 3133 continue; 3134 3135 rc = gaudi2_scrub_arc_dccm(hdev, arc_id); 3136 if (rc) 3137 return rc; 3138 } 3139 3140 return 0; 3141 } 3142 gaudi2_late_init(struct hl_device * hdev)3143 static int gaudi2_late_init(struct hl_device *hdev) 3144 { 3145 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3146 int rc; 3147 3148 hdev->asic_prop.supports_advanced_cpucp_rc = true; 3149 3150 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 3151 gaudi2->virt_msix_db_dma_addr); 3152 if (rc) { 3153 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); 3154 return rc; 3155 } 3156 3157 rc = gaudi2_fetch_psoc_frequency(hdev); 3158 if (rc) { 3159 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); 3160 goto disable_pci_access; 3161 } 3162 3163 gaudi2_init_arcs(hdev); 3164 3165 rc = gaudi2_scrub_arcs_dccm(hdev); 3166 if (rc) { 3167 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 3168 goto disable_pci_access; 3169 } 3170 3171 gaudi2_init_security(hdev); 3172 3173 return 0; 3174 3175 disable_pci_access: 3176 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 3177 3178 return rc; 3179 } 3180 gaudi2_late_fini(struct hl_device * hdev)3181 static void gaudi2_late_fini(struct hl_device *hdev) 3182 { 3183 hl_hwmon_release_resources(hdev); 3184 } 3185 gaudi2_user_mapped_dec_init(struct gaudi2_device * gaudi2,u32 start_idx)3186 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) 3187 { 3188 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3189 3190 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3191 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3192 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3193 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3194 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3195 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3196 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3197 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3198 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3199 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3200 } 3201 gaudi2_user_mapped_blocks_init(struct hl_device * hdev)3202 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) 3203 { 3204 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3205 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3206 u32 block_size, umr_start_idx, num_umr_blocks; 3207 int i; 3208 3209 for (i = 0 ; i < NUM_ARC_CPUS ; i++) { 3210 if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3) 3211 block_size = ARC_DCCM_BLOCK_SIZE * 2; 3212 else 3213 block_size = ARC_DCCM_BLOCK_SIZE; 3214 3215 blocks[i].address = gaudi2_arc_dccm_bases[i]; 3216 blocks[i].size = block_size; 3217 } 3218 3219 blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE; 3220 blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE; 3221 3222 blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE; 3223 blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE; 3224 3225 blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE; 3226 blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE; 3227 3228 blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE; 3229 blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE; 3230 3231 blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE; 3232 blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE; 3233 3234 blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE; 3235 blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE; 3236 3237 blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE; 3238 blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE; 3239 3240 blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE; 3241 blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE; 3242 3243 umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS; 3244 num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS; 3245 for (i = 0 ; i < num_umr_blocks ; i++) { 3246 u8 nic_id, umr_block_id; 3247 3248 nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS; 3249 umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS; 3250 3251 blocks[umr_start_idx + i].address = 3252 mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE + 3253 (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET + 3254 (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET + 3255 umr_block_id * NIC_UMR_OFFSET; 3256 blocks[umr_start_idx + i].size = HL_BLOCK_SIZE; 3257 } 3258 3259 /* Expose decoder HW configuration block to user */ 3260 gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX); 3261 3262 for (i = 1; i < NUM_OF_DCORES; ++i) { 3263 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; 3264 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; 3265 3266 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = 3267 mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET; 3268 3269 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = 3270 mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET; 3271 } 3272 } 3273 gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device * hdev)3274 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) 3275 { 3276 dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr; 3277 void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}; 3278 int i, j, rc = 0; 3279 3280 /* The device ARC works with 32-bits addresses, and because there is a single HW register 3281 * that holds the extension bits (49..28), these bits must be identical in all the allocated 3282 * range. 3283 */ 3284 3285 for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) { 3286 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 3287 &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO); 3288 if (!virt_addr_arr[i]) { 3289 rc = -ENOMEM; 3290 goto free_dma_mem_arr; 3291 } 3292 3293 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; 3294 if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr)) 3295 break; 3296 } 3297 3298 if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) { 3299 dev_err(hdev->dev, 3300 "MSB of ARC accessible DMA memory are not identical in all range\n"); 3301 rc = -EFAULT; 3302 goto free_dma_mem_arr; 3303 } 3304 3305 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; 3306 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; 3307 3308 free_dma_mem_arr: 3309 for (j = 0 ; j < i ; j++) 3310 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j], 3311 dma_addr_arr[j]); 3312 3313 return rc; 3314 } 3315 gaudi2_set_pci_memory_regions(struct hl_device * hdev)3316 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) 3317 { 3318 struct asic_fixed_properties *prop = &hdev->asic_prop; 3319 struct pci_mem_region *region; 3320 3321 /* CFG */ 3322 region = &hdev->pci_mem_region[PCI_REGION_CFG]; 3323 region->region_base = CFG_BASE; 3324 region->region_size = CFG_SIZE; 3325 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; 3326 region->bar_size = CFG_BAR_SIZE; 3327 region->bar_id = SRAM_CFG_BAR_ID; 3328 region->used = 1; 3329 3330 /* SRAM */ 3331 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 3332 region->region_base = SRAM_BASE_ADDR; 3333 region->region_size = SRAM_SIZE; 3334 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; 3335 region->bar_size = CFG_BAR_SIZE; 3336 region->bar_id = SRAM_CFG_BAR_ID; 3337 region->used = 1; 3338 3339 /* DRAM */ 3340 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 3341 region->region_base = DRAM_PHYS_BASE; 3342 region->region_size = hdev->asic_prop.dram_size; 3343 region->offset_in_bar = 0; 3344 region->bar_size = prop->dram_pci_bar_size; 3345 region->bar_id = DRAM_BAR_ID; 3346 region->used = 1; 3347 } 3348 gaudi2_user_interrupt_setup(struct hl_device * hdev)3349 static void gaudi2_user_interrupt_setup(struct hl_device *hdev) 3350 { 3351 struct asic_fixed_properties *prop = &hdev->asic_prop; 3352 int i, j, k; 3353 3354 /* Initialize TPC interrupt */ 3355 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); 3356 3357 /* Initialize unexpected error interrupt */ 3358 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, 3359 HL_USR_INTERRUPT_UNEXPECTED); 3360 3361 /* Initialize common user CQ interrupt */ 3362 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, 3363 HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ); 3364 3365 /* Initialize common decoder interrupt */ 3366 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, 3367 HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER); 3368 3369 /* User interrupts structure holds both decoder and user interrupts from various engines. 3370 * We first initialize the decoder interrupts and then we add the user interrupts. 3371 * The only limitation is that the last decoder interrupt id must be smaller 3372 * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time. 3373 */ 3374 3375 /* Initialize decoder interrupts, expose only normal interrupts, 3376 * error interrupts to be handled by driver 3377 */ 3378 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM; 3379 i += 2, j++) 3380 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, 3381 HL_USR_INTERRUPT_DECODER); 3382 3383 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) 3384 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); 3385 } 3386 gaudi2_get_non_zero_random_int(void)3387 static inline int gaudi2_get_non_zero_random_int(void) 3388 { 3389 int rand = get_random_u32(); 3390 3391 return rand ? rand : 1; 3392 } 3393 gaudi2_special_blocks_free(struct hl_device * hdev)3394 static void gaudi2_special_blocks_free(struct hl_device *hdev) 3395 { 3396 struct asic_fixed_properties *prop = &hdev->asic_prop; 3397 struct hl_skip_blocks_cfg *skip_special_blocks_cfg = 3398 &prop->skip_special_blocks_cfg; 3399 3400 kfree(prop->special_blocks); 3401 kfree(skip_special_blocks_cfg->block_types); 3402 kfree(skip_special_blocks_cfg->block_ranges); 3403 } 3404 gaudi2_special_blocks_iterator_free(struct hl_device * hdev)3405 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) 3406 { 3407 gaudi2_special_blocks_free(hdev); 3408 } 3409 gaudi2_special_block_skip(struct hl_device * hdev,struct hl_special_blocks_cfg * special_blocks_cfg,u32 blk_idx,u32 major,u32 minor,u32 sub_minor)3410 static bool gaudi2_special_block_skip(struct hl_device *hdev, 3411 struct hl_special_blocks_cfg *special_blocks_cfg, 3412 u32 blk_idx, u32 major, u32 minor, u32 sub_minor) 3413 { 3414 return false; 3415 } 3416 gaudi2_special_blocks_config(struct hl_device * hdev)3417 static int gaudi2_special_blocks_config(struct hl_device *hdev) 3418 { 3419 struct asic_fixed_properties *prop = &hdev->asic_prop; 3420 int i, rc; 3421 3422 /* Configure Special blocks */ 3423 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; 3424 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); 3425 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, 3426 sizeof(*prop->special_blocks), GFP_KERNEL); 3427 if (!prop->special_blocks) 3428 return -ENOMEM; 3429 3430 for (i = 0 ; i < prop->num_of_special_blocks ; i++) 3431 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], 3432 sizeof(*prop->special_blocks)); 3433 3434 /* Configure when to skip Special blocks */ 3435 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); 3436 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; 3437 3438 if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) { 3439 prop->skip_special_blocks_cfg.block_types = 3440 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types), 3441 sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL); 3442 if (!prop->skip_special_blocks_cfg.block_types) { 3443 rc = -ENOMEM; 3444 goto free_special_blocks; 3445 } 3446 3447 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, 3448 sizeof(gaudi2_iterator_skip_block_types)); 3449 3450 prop->skip_special_blocks_cfg.block_types_len = 3451 ARRAY_SIZE(gaudi2_iterator_skip_block_types); 3452 } 3453 3454 if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) { 3455 prop->skip_special_blocks_cfg.block_ranges = 3456 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges), 3457 sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL); 3458 if (!prop->skip_special_blocks_cfg.block_ranges) { 3459 rc = -ENOMEM; 3460 goto free_skip_special_blocks_types; 3461 } 3462 3463 for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++) 3464 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], 3465 &gaudi2_iterator_skip_block_ranges[i], 3466 sizeof(struct range)); 3467 3468 prop->skip_special_blocks_cfg.block_ranges_len = 3469 ARRAY_SIZE(gaudi2_iterator_skip_block_ranges); 3470 } 3471 3472 return 0; 3473 3474 free_skip_special_blocks_types: 3475 kfree(prop->skip_special_blocks_cfg.block_types); 3476 free_special_blocks: 3477 kfree(prop->special_blocks); 3478 3479 return rc; 3480 } 3481 gaudi2_special_blocks_iterator_config(struct hl_device * hdev)3482 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) 3483 { 3484 return gaudi2_special_blocks_config(hdev); 3485 } 3486 gaudi2_test_queues_msgs_free(struct hl_device * hdev)3487 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) 3488 { 3489 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3490 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3491 int i; 3492 3493 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3494 /* bail-out if this is an allocation failure point */ 3495 if (!msg_info[i].kern_addr) 3496 break; 3497 3498 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr); 3499 msg_info[i].kern_addr = NULL; 3500 } 3501 } 3502 gaudi2_test_queues_msgs_alloc(struct hl_device * hdev)3503 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) 3504 { 3505 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3506 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3507 int i, rc; 3508 3509 /* allocate a message-short buf for each Q we intend to test */ 3510 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3511 msg_info[i].kern_addr = 3512 (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short), 3513 GFP_KERNEL, &msg_info[i].dma_addr); 3514 if (!msg_info[i].kern_addr) { 3515 dev_err(hdev->dev, 3516 "Failed to allocate dma memory for H/W queue %d testing\n", i); 3517 rc = -ENOMEM; 3518 goto err_exit; 3519 } 3520 } 3521 3522 return 0; 3523 3524 err_exit: 3525 gaudi2_test_queues_msgs_free(hdev); 3526 return rc; 3527 } 3528 gaudi2_sw_init(struct hl_device * hdev)3529 static int gaudi2_sw_init(struct hl_device *hdev) 3530 { 3531 struct asic_fixed_properties *prop = &hdev->asic_prop; 3532 struct gaudi2_device *gaudi2; 3533 int i, rc; 3534 3535 /* Allocate device structure */ 3536 gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL); 3537 if (!gaudi2) 3538 return -ENOMEM; 3539 3540 for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) { 3541 if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid) 3542 continue; 3543 3544 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { 3545 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", 3546 GAUDI2_EVENT_SIZE); 3547 rc = -EINVAL; 3548 goto free_gaudi2_device; 3549 } 3550 3551 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; 3552 } 3553 3554 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) 3555 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); 3556 3557 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; 3558 3559 hdev->asic_specific = gaudi2; 3560 3561 /* Create DMA pool for small allocations. 3562 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped 3563 * PI/CI registers allocated from this pool have this restriction 3564 */ 3565 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, 3566 GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0); 3567 if (!hdev->dma_pool) { 3568 dev_err(hdev->dev, "failed to create DMA pool\n"); 3569 rc = -ENOMEM; 3570 goto free_gaudi2_device; 3571 } 3572 3573 rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev); 3574 if (rc) 3575 goto free_dma_pool; 3576 3577 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 3578 if (!hdev->cpu_accessible_dma_pool) { 3579 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); 3580 rc = -ENOMEM; 3581 goto free_cpu_dma_mem; 3582 } 3583 3584 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, 3585 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 3586 if (rc) { 3587 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); 3588 rc = -EFAULT; 3589 goto free_cpu_accessible_dma_pool; 3590 } 3591 3592 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, 3593 &gaudi2->virt_msix_db_dma_addr); 3594 if (!gaudi2->virt_msix_db_cpu_addr) { 3595 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); 3596 rc = -ENOMEM; 3597 goto free_cpu_accessible_dma_pool; 3598 } 3599 3600 spin_lock_init(&gaudi2->hw_queues_lock); 3601 3602 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, 3603 &gaudi2->scratchpad_bus_address, 3604 GFP_KERNEL | __GFP_ZERO); 3605 if (!gaudi2->scratchpad_kernel_address) { 3606 rc = -ENOMEM; 3607 goto free_virt_msix_db_mem; 3608 } 3609 3610 gaudi2_user_mapped_blocks_init(hdev); 3611 3612 /* Initialize user interrupts */ 3613 gaudi2_user_interrupt_setup(hdev); 3614 3615 hdev->supports_coresight = true; 3616 hdev->supports_sync_stream = true; 3617 hdev->supports_cb_mapping = true; 3618 hdev->supports_wait_for_multi_cs = false; 3619 3620 prop->supports_compute_reset = true; 3621 3622 /* Event queue sanity check added in FW version 1.11 */ 3623 if (hl_is_fw_sw_ver_below(hdev, 1, 11)) 3624 hdev->event_queue.check_eqe_index = false; 3625 else 3626 hdev->event_queue.check_eqe_index = true; 3627 3628 hdev->asic_funcs->set_pci_memory_regions(hdev); 3629 3630 rc = gaudi2_special_blocks_iterator_config(hdev); 3631 if (rc) 3632 goto free_scratchpad_mem; 3633 3634 rc = gaudi2_test_queues_msgs_alloc(hdev); 3635 if (rc) 3636 goto special_blocks_free; 3637 3638 return 0; 3639 3640 special_blocks_free: 3641 gaudi2_special_blocks_iterator_free(hdev); 3642 free_scratchpad_mem: 3643 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, 3644 gaudi2->scratchpad_bus_address); 3645 free_virt_msix_db_mem: 3646 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3647 free_cpu_accessible_dma_pool: 3648 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3649 free_cpu_dma_mem: 3650 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3651 hdev->cpu_accessible_dma_address); 3652 free_dma_pool: 3653 dma_pool_destroy(hdev->dma_pool); 3654 free_gaudi2_device: 3655 kfree(gaudi2); 3656 return rc; 3657 } 3658 gaudi2_sw_fini(struct hl_device * hdev)3659 static int gaudi2_sw_fini(struct hl_device *hdev) 3660 { 3661 struct asic_fixed_properties *prop = &hdev->asic_prop; 3662 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3663 3664 gaudi2_test_queues_msgs_free(hdev); 3665 3666 gaudi2_special_blocks_iterator_free(hdev); 3667 3668 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3669 3670 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3671 3672 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3673 hdev->cpu_accessible_dma_address); 3674 3675 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, 3676 gaudi2->scratchpad_bus_address); 3677 3678 dma_pool_destroy(hdev->dma_pool); 3679 3680 kfree(gaudi2); 3681 3682 return 0; 3683 } 3684 gaudi2_stop_qman_common(struct hl_device * hdev,u32 reg_base)3685 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) 3686 { 3687 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP | 3688 QM_GLBL_CFG1_CQF_STOP | 3689 QM_GLBL_CFG1_CP_STOP); 3690 3691 /* stop also the ARC */ 3692 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP); 3693 } 3694 gaudi2_flush_qman_common(struct hl_device * hdev,u32 reg_base)3695 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) 3696 { 3697 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH | 3698 QM_GLBL_CFG1_CQF_FLUSH | 3699 QM_GLBL_CFG1_CP_FLUSH); 3700 } 3701 gaudi2_flush_qman_arc_common(struct hl_device * hdev,u32 reg_base)3702 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) 3703 { 3704 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH); 3705 } 3706 3707 /** 3708 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters 3709 * 3710 * @hdev: pointer to the habanalabs device structure 3711 * @queue_id: queue to clear fence counters to 3712 * @skip_fence: if true set maximum fence value to all fence counters to avoid 3713 * getting stuck on any fence value. otherwise set all fence 3714 * counters to 0 (standard clear of fence counters) 3715 */ gaudi2_clear_qm_fence_counters_common(struct hl_device * hdev,u32 queue_id,bool skip_fence)3716 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, 3717 bool skip_fence) 3718 { 3719 u32 size, reg_base; 3720 u32 addr, val; 3721 3722 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3723 3724 addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET; 3725 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; 3726 3727 /* 3728 * in case we want to make sure that QM that is stuck on a fence will 3729 * be released we should set the fence counter to a higher value that 3730 * the value the QM waiting for. to comply with any fence counter of 3731 * any value we set maximum fence value to all counters 3732 */ 3733 val = skip_fence ? U32_MAX : 0; 3734 gaudi2_memset_device_lbw(hdev, addr, size, val); 3735 } 3736 gaudi2_qman_manual_flush_common(struct hl_device * hdev,u32 queue_id)3737 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) 3738 { 3739 u32 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3740 3741 gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true); 3742 gaudi2_flush_qman_common(hdev, reg_base); 3743 gaudi2_flush_qman_arc_common(hdev, reg_base); 3744 } 3745 gaudi2_stop_dma_qmans(struct hl_device * hdev)3746 static void gaudi2_stop_dma_qmans(struct hl_device *hdev) 3747 { 3748 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3749 int dcore, inst; 3750 3751 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3752 goto stop_edma_qmans; 3753 3754 /* Stop CPs of PDMA QMANs */ 3755 gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE); 3756 gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE); 3757 3758 stop_edma_qmans: 3759 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3760 return; 3761 3762 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3763 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3764 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3765 u32 qm_base; 3766 3767 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3768 continue; 3769 3770 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 3771 inst * DCORE_EDMA_OFFSET; 3772 3773 /* Stop CPs of EDMA QMANs */ 3774 gaudi2_stop_qman_common(hdev, qm_base); 3775 } 3776 } 3777 } 3778 gaudi2_stop_mme_qmans(struct hl_device * hdev)3779 static void gaudi2_stop_mme_qmans(struct hl_device *hdev) 3780 { 3781 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3782 u32 offset, i; 3783 3784 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 3785 3786 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 3787 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) 3788 continue; 3789 3790 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 3791 } 3792 } 3793 gaudi2_stop_tpc_qmans(struct hl_device * hdev)3794 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) 3795 { 3796 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3797 u32 reg_base; 3798 int i; 3799 3800 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 3801 return; 3802 3803 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 3804 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 3805 continue; 3806 3807 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 3808 gaudi2_stop_qman_common(hdev, reg_base); 3809 } 3810 } 3811 gaudi2_stop_rot_qmans(struct hl_device * hdev)3812 static void gaudi2_stop_rot_qmans(struct hl_device *hdev) 3813 { 3814 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3815 u32 reg_base; 3816 int i; 3817 3818 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 3819 return; 3820 3821 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 3822 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 3823 continue; 3824 3825 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 3826 gaudi2_stop_qman_common(hdev, reg_base); 3827 } 3828 } 3829 gaudi2_stop_nic_qmans(struct hl_device * hdev)3830 static void gaudi2_stop_nic_qmans(struct hl_device *hdev) 3831 { 3832 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3833 u32 reg_base, queue_id; 3834 int i; 3835 3836 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 3837 return; 3838 3839 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 3840 3841 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 3842 if (!(hdev->nic_ports_mask & BIT(i))) 3843 continue; 3844 3845 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3846 gaudi2_stop_qman_common(hdev, reg_base); 3847 } 3848 } 3849 gaudi2_stall_dma_common(struct hl_device * hdev,u32 reg_base)3850 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) 3851 { 3852 u32 reg_val; 3853 3854 reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1); 3855 WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val); 3856 } 3857 gaudi2_dma_stall(struct hl_device * hdev)3858 static void gaudi2_dma_stall(struct hl_device *hdev) 3859 { 3860 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3861 int dcore, inst; 3862 3863 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3864 goto stall_edma; 3865 3866 gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE); 3867 gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE); 3868 3869 stall_edma: 3870 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3871 return; 3872 3873 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3874 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3875 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3876 u32 core_base; 3877 3878 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3879 continue; 3880 3881 core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET + 3882 inst * DCORE_EDMA_OFFSET; 3883 3884 /* Stall CPs of EDMA QMANs */ 3885 gaudi2_stall_dma_common(hdev, core_base); 3886 } 3887 } 3888 } 3889 gaudi2_mme_stall(struct hl_device * hdev)3890 static void gaudi2_mme_stall(struct hl_device *hdev) 3891 { 3892 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3893 u32 offset, i; 3894 3895 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; 3896 3897 for (i = 0 ; i < NUM_OF_DCORES ; i++) 3898 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 3899 WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1); 3900 } 3901 gaudi2_tpc_stall(struct hl_device * hdev)3902 static void gaudi2_tpc_stall(struct hl_device *hdev) 3903 { 3904 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3905 u32 reg_base; 3906 int i; 3907 3908 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 3909 return; 3910 3911 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 3912 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 3913 continue; 3914 3915 reg_base = gaudi2_tpc_cfg_blocks_bases[i]; 3916 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1); 3917 } 3918 } 3919 gaudi2_rotator_stall(struct hl_device * hdev)3920 static void gaudi2_rotator_stall(struct hl_device *hdev) 3921 { 3922 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3923 u32 reg_val; 3924 int i; 3925 3926 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 3927 return; 3928 3929 reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) | 3930 FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) | 3931 FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1); 3932 3933 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 3934 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 3935 continue; 3936 3937 WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val); 3938 } 3939 } 3940 gaudi2_disable_qman_common(struct hl_device * hdev,u32 reg_base)3941 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) 3942 { 3943 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0); 3944 } 3945 gaudi2_disable_dma_qmans(struct hl_device * hdev)3946 static void gaudi2_disable_dma_qmans(struct hl_device *hdev) 3947 { 3948 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3949 int dcore, inst; 3950 3951 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3952 goto stop_edma_qmans; 3953 3954 gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE); 3955 gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE); 3956 3957 stop_edma_qmans: 3958 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3959 return; 3960 3961 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3962 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3963 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3964 u32 qm_base; 3965 3966 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3967 continue; 3968 3969 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 3970 inst * DCORE_EDMA_OFFSET; 3971 3972 /* Disable CPs of EDMA QMANs */ 3973 gaudi2_disable_qman_common(hdev, qm_base); 3974 } 3975 } 3976 } 3977 gaudi2_disable_mme_qmans(struct hl_device * hdev)3978 static void gaudi2_disable_mme_qmans(struct hl_device *hdev) 3979 { 3980 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3981 u32 offset, i; 3982 3983 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 3984 3985 for (i = 0 ; i < NUM_OF_DCORES ; i++) 3986 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 3987 gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 3988 } 3989 gaudi2_disable_tpc_qmans(struct hl_device * hdev)3990 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) 3991 { 3992 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3993 u32 reg_base; 3994 int i; 3995 3996 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 3997 return; 3998 3999 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4000 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4001 continue; 4002 4003 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 4004 gaudi2_disable_qman_common(hdev, reg_base); 4005 } 4006 } 4007 gaudi2_disable_rot_qmans(struct hl_device * hdev)4008 static void gaudi2_disable_rot_qmans(struct hl_device *hdev) 4009 { 4010 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4011 u32 reg_base; 4012 int i; 4013 4014 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4015 return; 4016 4017 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4018 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4019 continue; 4020 4021 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 4022 gaudi2_disable_qman_common(hdev, reg_base); 4023 } 4024 } 4025 gaudi2_disable_nic_qmans(struct hl_device * hdev)4026 static void gaudi2_disable_nic_qmans(struct hl_device *hdev) 4027 { 4028 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4029 u32 reg_base, queue_id; 4030 int i; 4031 4032 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4033 return; 4034 4035 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4036 4037 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4038 if (!(hdev->nic_ports_mask & BIT(i))) 4039 continue; 4040 4041 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4042 gaudi2_disable_qman_common(hdev, reg_base); 4043 } 4044 } 4045 gaudi2_enable_timestamp(struct hl_device * hdev)4046 static void gaudi2_enable_timestamp(struct hl_device *hdev) 4047 { 4048 /* Disable the timestamp counter */ 4049 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4050 4051 /* Zero the lower/upper parts of the 64-bit counter */ 4052 WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0); 4053 WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0); 4054 4055 /* Enable the counter */ 4056 WREG32(mmPSOC_TIMESTAMP_BASE, 1); 4057 } 4058 gaudi2_disable_timestamp(struct hl_device * hdev)4059 static void gaudi2_disable_timestamp(struct hl_device *hdev) 4060 { 4061 /* Disable the timestamp counter */ 4062 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4063 } 4064 gaudi2_irq_name(u16 irq_number)4065 static const char *gaudi2_irq_name(u16 irq_number) 4066 { 4067 switch (irq_number) { 4068 case GAUDI2_IRQ_NUM_EVENT_QUEUE: 4069 return "gaudi2 cpu eq"; 4070 case GAUDI2_IRQ_NUM_COMPLETION: 4071 return "gaudi2 completion"; 4072 case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM: 4073 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; 4074 case GAUDI2_IRQ_NUM_TPC_ASSERT: 4075 return "gaudi2 tpc assert"; 4076 case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR: 4077 return "gaudi2 unexpected error"; 4078 case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST: 4079 return "gaudi2 user completion"; 4080 default: 4081 return "invalid"; 4082 } 4083 } 4084 gaudi2_dec_disable_msix(struct hl_device * hdev,u32 max_irq_num)4085 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) 4086 { 4087 int i, irq, relative_idx; 4088 struct hl_dec *dec; 4089 4090 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) { 4091 irq = pci_irq_vector(hdev->pdev, i); 4092 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4093 4094 dec = hdev->dec + relative_idx / 2; 4095 4096 /* We pass different structures depending on the irq handler. For the abnormal 4097 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4098 * user_interrupt entry 4099 */ 4100 free_irq(irq, ((relative_idx % 2) ? 4101 (void *) dec : 4102 (void *) &hdev->user_interrupt[dec->core_id])); 4103 } 4104 } 4105 gaudi2_dec_enable_msix(struct hl_device * hdev)4106 static int gaudi2_dec_enable_msix(struct hl_device *hdev) 4107 { 4108 int rc, i, irq_init_cnt, irq, relative_idx; 4109 struct hl_dec *dec; 4110 4111 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0; 4112 i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM; 4113 i++, irq_init_cnt++) { 4114 4115 irq = pci_irq_vector(hdev->pdev, i); 4116 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4117 4118 /* We pass different structures depending on the irq handler. For the abnormal 4119 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4120 * user_interrupt entry 4121 * 4122 * TODO: change the dec abnrm to threaded irq 4123 */ 4124 4125 dec = hdev->dec + relative_idx / 2; 4126 if (relative_idx % 2) { 4127 rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0, 4128 gaudi2_irq_name(i), (void *) dec); 4129 } else { 4130 rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt, 4131 hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4132 gaudi2_irq_name(i), 4133 (void *) &hdev->user_interrupt[dec->core_id]); 4134 } 4135 4136 if (rc) { 4137 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4138 goto free_dec_irqs; 4139 } 4140 } 4141 4142 return 0; 4143 4144 free_dec_irqs: 4145 gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt)); 4146 return rc; 4147 } 4148 gaudi2_enable_msix(struct hl_device * hdev)4149 static int gaudi2_enable_msix(struct hl_device *hdev) 4150 { 4151 struct asic_fixed_properties *prop = &hdev->asic_prop; 4152 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4153 int rc, irq, i, j, user_irq_init_cnt; 4154 struct hl_cq *cq; 4155 4156 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) 4157 return 0; 4158 4159 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, 4160 PCI_IRQ_MSIX); 4161 if (rc < 0) { 4162 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", 4163 GAUDI2_MSIX_ENTRIES, rc); 4164 return rc; 4165 } 4166 4167 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4168 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4169 rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq); 4170 if (rc) { 4171 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4172 goto free_irq_vectors; 4173 } 4174 4175 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4176 rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE), 4177 &hdev->event_queue); 4178 if (rc) { 4179 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4180 goto free_completion_irq; 4181 } 4182 4183 rc = gaudi2_dec_enable_msix(hdev); 4184 if (rc) { 4185 dev_err(hdev->dev, "Failed to enable decoder IRQ"); 4186 goto free_event_irq; 4187 } 4188 4189 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4190 rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt, 4191 hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4192 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt); 4193 if (rc) { 4194 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4195 goto free_dec_irq; 4196 } 4197 4198 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4199 rc = request_irq(irq, hl_irq_handler_user_interrupt, 0, 4200 gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR), 4201 &hdev->unexpected_error_interrupt); 4202 if (rc) { 4203 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4204 goto free_tpc_irq; 4205 } 4206 4207 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; 4208 user_irq_init_cnt < prop->user_interrupt_count; 4209 i++, j++, user_irq_init_cnt++) { 4210 4211 irq = pci_irq_vector(hdev->pdev, i); 4212 rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt, 4213 hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4214 gaudi2_irq_name(i), &hdev->user_interrupt[j]); 4215 4216 if (rc) { 4217 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4218 goto free_user_irq; 4219 } 4220 } 4221 4222 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; 4223 4224 return 0; 4225 4226 free_user_irq: 4227 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; 4228 i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) { 4229 4230 irq = pci_irq_vector(hdev->pdev, i); 4231 free_irq(irq, &hdev->user_interrupt[j]); 4232 } 4233 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4234 free_irq(irq, &hdev->unexpected_error_interrupt); 4235 free_tpc_irq: 4236 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4237 free_irq(irq, &hdev->tpc_interrupt); 4238 free_dec_irq: 4239 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1); 4240 free_event_irq: 4241 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4242 free_irq(irq, cq); 4243 4244 free_completion_irq: 4245 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4246 free_irq(irq, cq); 4247 4248 free_irq_vectors: 4249 pci_free_irq_vectors(hdev->pdev); 4250 4251 return rc; 4252 } 4253 gaudi2_sync_irqs(struct hl_device * hdev)4254 static void gaudi2_sync_irqs(struct hl_device *hdev) 4255 { 4256 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4257 int i, j; 4258 int irq; 4259 4260 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4261 return; 4262 4263 /* Wait for all pending IRQs to be finished */ 4264 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); 4265 4266 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) { 4267 irq = pci_irq_vector(hdev->pdev, i); 4268 synchronize_irq(irq); 4269 } 4270 4271 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); 4272 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); 4273 4274 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; 4275 i++, j++) { 4276 irq = pci_irq_vector(hdev->pdev, i); 4277 synchronize_irq(irq); 4278 } 4279 4280 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); 4281 } 4282 gaudi2_disable_msix(struct hl_device * hdev)4283 static void gaudi2_disable_msix(struct hl_device *hdev) 4284 { 4285 struct asic_fixed_properties *prop = &hdev->asic_prop; 4286 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4287 struct hl_cq *cq; 4288 int irq, i, j, k; 4289 4290 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4291 return; 4292 4293 gaudi2_sync_irqs(hdev); 4294 4295 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4296 free_irq(irq, &hdev->event_queue); 4297 4298 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); 4299 4300 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4301 free_irq(irq, &hdev->tpc_interrupt); 4302 4303 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4304 free_irq(irq, &hdev->unexpected_error_interrupt); 4305 4306 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; 4307 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { 4308 4309 irq = pci_irq_vector(hdev->pdev, i); 4310 free_irq(irq, &hdev->user_interrupt[j]); 4311 } 4312 4313 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4314 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4315 free_irq(irq, cq); 4316 4317 pci_free_irq_vectors(hdev->pdev); 4318 4319 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; 4320 } 4321 gaudi2_stop_dcore_dec(struct hl_device * hdev,int dcore_id)4322 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) 4323 { 4324 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4325 u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4326 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4327 int rc; 4328 4329 if (hdev->pldm) 4330 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4331 else 4332 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4333 4334 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4335 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 4336 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4337 continue; 4338 4339 offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET; 4340 4341 WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0); 4342 4343 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4344 4345 /* Wait till all traffic from decoder stops 4346 * before apply core reset. 4347 */ 4348 rc = hl_poll_timeout( 4349 hdev, 4350 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4351 graceful, 4352 (graceful & graceful_pend_mask), 4353 100, 4354 timeout_usec); 4355 if (rc) 4356 dev_err(hdev->dev, 4357 "Failed to stop traffic from DCORE%d Decoder %d\n", 4358 dcore_id, dec_id); 4359 } 4360 } 4361 gaudi2_stop_pcie_dec(struct hl_device * hdev)4362 static void gaudi2_stop_pcie_dec(struct hl_device *hdev) 4363 { 4364 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4365 u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4366 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4367 int rc; 4368 4369 if (hdev->pldm) 4370 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4371 else 4372 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4373 4374 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4375 dec_bit = PCIE_DEC_SHIFT + dec_id; 4376 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4377 continue; 4378 4379 offset = dec_id * PCIE_VDEC_OFFSET; 4380 4381 WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0); 4382 4383 WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4384 4385 /* Wait till all traffic from decoder stops 4386 * before apply core reset. 4387 */ 4388 rc = hl_poll_timeout( 4389 hdev, 4390 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4391 graceful, 4392 (graceful & graceful_pend_mask), 4393 100, 4394 timeout_usec); 4395 if (rc) 4396 dev_err(hdev->dev, 4397 "Failed to stop traffic from PCIe Decoder %d\n", 4398 dec_id); 4399 } 4400 } 4401 gaudi2_stop_dec(struct hl_device * hdev)4402 static void gaudi2_stop_dec(struct hl_device *hdev) 4403 { 4404 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4405 int dcore_id; 4406 4407 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) 4408 return; 4409 4410 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 4411 gaudi2_stop_dcore_dec(hdev, dcore_id); 4412 4413 gaudi2_stop_pcie_dec(hdev); 4414 } 4415 gaudi2_set_arc_running_mode(struct hl_device * hdev,u32 cpu_id,u32 run_mode)4416 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4417 { 4418 u32 reg_base, reg_val; 4419 4420 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4421 if (run_mode == HL_ENGINE_CORE_RUN) 4422 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); 4423 else 4424 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); 4425 4426 WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val); 4427 } 4428 gaudi2_halt_arcs(struct hl_device * hdev)4429 static void gaudi2_halt_arcs(struct hl_device *hdev) 4430 { 4431 u16 arc_id; 4432 4433 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) { 4434 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4435 gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT); 4436 } 4437 } 4438 gaudi2_verify_arc_running_mode(struct hl_device * hdev,u32 cpu_id,u32 run_mode)4439 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4440 { 4441 int rc; 4442 u32 reg_base, val, ack_mask, timeout_usec = 100000; 4443 4444 if (hdev->pldm) 4445 timeout_usec *= 100; 4446 4447 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4448 if (run_mode == HL_ENGINE_CORE_RUN) 4449 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK; 4450 else 4451 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK; 4452 4453 rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET, 4454 val, ((val & ack_mask) == ack_mask), 4455 1000, timeout_usec); 4456 4457 if (!rc) { 4458 /* Clear */ 4459 val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0); 4460 WREG32(reg_base + ARC_HALT_REQ_OFFSET, val); 4461 } 4462 4463 return rc; 4464 } 4465 gaudi2_reset_arcs(struct hl_device * hdev)4466 static void gaudi2_reset_arcs(struct hl_device *hdev) 4467 { 4468 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4469 u16 arc_id; 4470 4471 if (!gaudi2) 4472 return; 4473 4474 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) 4475 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4476 gaudi2_clr_arc_id_cap(hdev, arc_id); 4477 } 4478 gaudi2_nic_qmans_manual_flush(struct hl_device * hdev)4479 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) 4480 { 4481 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4482 u32 queue_id; 4483 int i; 4484 4485 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4486 return; 4487 4488 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4489 4490 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4491 if (!(hdev->nic_ports_mask & BIT(i))) 4492 continue; 4493 4494 gaudi2_qman_manual_flush_common(hdev, queue_id); 4495 } 4496 } 4497 gaudi2_set_engine_cores(struct hl_device * hdev,u32 * core_ids,u32 num_cores,u32 core_command)4498 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, 4499 u32 num_cores, u32 core_command) 4500 { 4501 int i, rc; 4502 4503 for (i = 0 ; i < num_cores ; i++) { 4504 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) 4505 gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command); 4506 } 4507 4508 for (i = 0 ; i < num_cores ; i++) { 4509 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) { 4510 rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command); 4511 4512 if (rc) { 4513 dev_err(hdev->dev, "failed to %s arc: %d\n", 4514 (core_command == HL_ENGINE_CORE_HALT) ? 4515 "HALT" : "RUN", core_ids[i]); 4516 return -1; 4517 } 4518 } 4519 } 4520 4521 return 0; 4522 } 4523 gaudi2_set_tpc_engine_mode(struct hl_device * hdev,u32 engine_id,u32 engine_command)4524 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4525 { 4526 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4527 u32 reg_base, reg_addr, reg_val, tpc_id; 4528 4529 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4530 return 0; 4531 4532 tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id]; 4533 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) 4534 return 0; 4535 4536 reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id]; 4537 reg_addr = reg_base + TPC_CFG_STALL_OFFSET; 4538 reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK, 4539 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4540 WREG32(reg_addr, reg_val); 4541 4542 if (engine_command == HL_ENGINE_RESUME) { 4543 reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id]; 4544 reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET; 4545 RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK); 4546 } 4547 4548 return 0; 4549 } 4550 gaudi2_set_mme_engine_mode(struct hl_device * hdev,u32 engine_id,u32 engine_command)4551 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4552 { 4553 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4554 u32 reg_base, reg_addr, reg_val, mme_id; 4555 4556 mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id]; 4557 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) 4558 return 0; 4559 4560 reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id]; 4561 reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET; 4562 reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK, 4563 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4564 WREG32(reg_addr, reg_val); 4565 4566 return 0; 4567 } 4568 gaudi2_set_edma_engine_mode(struct hl_device * hdev,u32 engine_id,u32 engine_command)4569 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4570 { 4571 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4572 u32 reg_base, reg_addr, reg_val, edma_id; 4573 4574 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4575 return 0; 4576 4577 edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id]; 4578 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) 4579 return 0; 4580 4581 reg_base = gaudi2_dma_core_blocks_bases[edma_id]; 4582 reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET; 4583 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 4584 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4585 WREG32(reg_addr, reg_val); 4586 4587 if (engine_command == HL_ENGINE_STALL) { 4588 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) | 4589 FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1); 4590 WREG32(reg_addr, reg_val); 4591 } 4592 4593 return 0; 4594 } 4595 gaudi2_set_engine_modes(struct hl_device * hdev,u32 * engine_ids,u32 num_engines,u32 engine_command)4596 static int gaudi2_set_engine_modes(struct hl_device *hdev, 4597 u32 *engine_ids, u32 num_engines, u32 engine_command) 4598 { 4599 int i, rc; 4600 4601 for (i = 0 ; i < num_engines ; ++i) { 4602 switch (engine_ids[i]) { 4603 case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5: 4604 case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5: 4605 case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5: 4606 case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5: 4607 rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command); 4608 if (rc) 4609 return rc; 4610 4611 break; 4612 case GAUDI2_DCORE0_ENGINE_ID_MME: 4613 case GAUDI2_DCORE1_ENGINE_ID_MME: 4614 case GAUDI2_DCORE2_ENGINE_ID_MME: 4615 case GAUDI2_DCORE3_ENGINE_ID_MME: 4616 rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command); 4617 if (rc) 4618 return rc; 4619 4620 break; 4621 case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1: 4622 case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1: 4623 case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1: 4624 case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1: 4625 rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command); 4626 if (rc) 4627 return rc; 4628 4629 break; 4630 default: 4631 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); 4632 return -EINVAL; 4633 } 4634 } 4635 4636 return 0; 4637 } 4638 gaudi2_set_engines(struct hl_device * hdev,u32 * engine_ids,u32 num_engines,u32 engine_command)4639 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, 4640 u32 num_engines, u32 engine_command) 4641 { 4642 switch (engine_command) { 4643 case HL_ENGINE_CORE_HALT: 4644 case HL_ENGINE_CORE_RUN: 4645 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command); 4646 4647 case HL_ENGINE_STALL: 4648 case HL_ENGINE_RESUME: 4649 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command); 4650 4651 default: 4652 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); 4653 return -EINVAL; 4654 } 4655 } 4656 gaudi2_halt_engines(struct hl_device * hdev,bool hard_reset,bool fw_reset)4657 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 4658 { 4659 u32 wait_timeout_ms; 4660 4661 if (hdev->pldm) 4662 wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC; 4663 else 4664 wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC; 4665 4666 if (fw_reset) 4667 goto skip_engines; 4668 4669 gaudi2_stop_dma_qmans(hdev); 4670 gaudi2_stop_mme_qmans(hdev); 4671 gaudi2_stop_tpc_qmans(hdev); 4672 gaudi2_stop_rot_qmans(hdev); 4673 gaudi2_stop_nic_qmans(hdev); 4674 msleep(wait_timeout_ms); 4675 4676 gaudi2_halt_arcs(hdev); 4677 gaudi2_dma_stall(hdev); 4678 gaudi2_mme_stall(hdev); 4679 gaudi2_tpc_stall(hdev); 4680 gaudi2_rotator_stall(hdev); 4681 4682 msleep(wait_timeout_ms); 4683 4684 gaudi2_stop_dec(hdev); 4685 4686 /* 4687 * in case of soft reset do a manual flush for QMANs (currently called 4688 * only for NIC QMANs 4689 */ 4690 if (!hard_reset) 4691 gaudi2_nic_qmans_manual_flush(hdev); 4692 4693 gaudi2_disable_dma_qmans(hdev); 4694 gaudi2_disable_mme_qmans(hdev); 4695 gaudi2_disable_tpc_qmans(hdev); 4696 gaudi2_disable_rot_qmans(hdev); 4697 gaudi2_disable_nic_qmans(hdev); 4698 gaudi2_disable_timestamp(hdev); 4699 4700 skip_engines: 4701 if (hard_reset) { 4702 gaudi2_disable_msix(hdev); 4703 return; 4704 } 4705 4706 gaudi2_sync_irqs(hdev); 4707 } 4708 gaudi2_init_firmware_preload_params(struct hl_device * hdev)4709 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) 4710 { 4711 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 4712 4713 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 4714 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 4715 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 4716 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 4717 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 4718 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; 4719 } 4720 gaudi2_init_firmware_loader(struct hl_device * hdev)4721 static void gaudi2_init_firmware_loader(struct hl_device *hdev) 4722 { 4723 struct fw_load_mgr *fw_loader = &hdev->fw_loader; 4724 struct dynamic_fw_load_mgr *dynamic_loader; 4725 struct cpu_dyn_regs *dyn_regs; 4726 4727 /* fill common fields */ 4728 fw_loader->fw_comp_loaded = FW_TYPE_NONE; 4729 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; 4730 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; 4731 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; 4732 fw_loader->skip_bmc = false; 4733 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; 4734 fw_loader->dram_bar_id = DRAM_BAR_ID; 4735 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; 4736 4737 /* here we update initial values for few specific dynamic regs (as 4738 * before reading the first descriptor from FW those value has to be 4739 * hard-coded). in later stages of the protocol those values will be 4740 * updated automatically by reading the FW descriptor so data there 4741 * will always be up-to-date 4742 */ 4743 dynamic_loader = &hdev->fw_loader.dynamic_loader; 4744 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 4745 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 4746 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 4747 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; 4748 } 4749 gaudi2_init_cpu(struct hl_device * hdev)4750 static int gaudi2_init_cpu(struct hl_device *hdev) 4751 { 4752 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4753 int rc; 4754 4755 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 4756 return 0; 4757 4758 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) 4759 return 0; 4760 4761 rc = hl_fw_init_cpu(hdev); 4762 if (rc) 4763 return rc; 4764 4765 gaudi2->hw_cap_initialized |= HW_CAP_CPU; 4766 4767 return 0; 4768 } 4769 gaudi2_init_cpu_queues(struct hl_device * hdev,u32 cpu_timeout)4770 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) 4771 { 4772 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 4773 struct asic_fixed_properties *prop = &hdev->asic_prop; 4774 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4775 struct cpu_dyn_regs *dyn_regs; 4776 struct hl_eq *eq; 4777 u32 status; 4778 int err; 4779 4780 if (!hdev->cpu_queues_enable) 4781 return 0; 4782 4783 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 4784 return 0; 4785 4786 eq = &hdev->event_queue; 4787 4788 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4789 4790 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 4791 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 4792 4793 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 4794 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 4795 4796 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); 4797 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); 4798 4799 WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 4800 WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 4801 WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 4802 4803 /* Used for EQ CI */ 4804 WREG32(mmCPU_IF_EQ_RD_OFFS, 0); 4805 4806 WREG32(mmCPU_IF_PF_PQ_PI, 0); 4807 4808 WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP); 4809 4810 /* Let the ARC know we are ready as it is now handling those queues */ 4811 4812 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 4813 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 4814 4815 err = hl_poll_timeout( 4816 hdev, 4817 mmCPU_IF_QUEUE_INIT, 4818 status, 4819 (status == PQ_INIT_STATUS_READY_FOR_HOST), 4820 1000, 4821 cpu_timeout); 4822 4823 if (err) { 4824 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); 4825 return -EIO; 4826 } 4827 4828 /* update FW application security bits */ 4829 if (prop->fw_cpu_boot_dev_sts0_valid) 4830 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 4831 4832 if (prop->fw_cpu_boot_dev_sts1_valid) 4833 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 4834 4835 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; 4836 return 0; 4837 } 4838 gaudi2_init_qman_pq(struct hl_device * hdev,u32 reg_base,u32 queue_id_base)4839 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, 4840 u32 queue_id_base) 4841 { 4842 struct hl_hw_queue *q; 4843 u32 pq_id, pq_offset; 4844 4845 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 4846 q = &hdev->kernel_queues[queue_id_base + pq_id]; 4847 pq_offset = pq_id * 4; 4848 4849 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 4850 lower_32_bits(q->bus_address)); 4851 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 4852 upper_32_bits(q->bus_address)); 4853 WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH)); 4854 WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0); 4855 WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0); 4856 } 4857 } 4858 gaudi2_init_qman_cp(struct hl_device * hdev,u32 reg_base)4859 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) 4860 { 4861 u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi; 4862 4863 mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 4864 mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 4865 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4866 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4867 4868 for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) { 4869 cp_offset = cp_id * 4; 4870 4871 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo); 4872 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi); 4873 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo); 4874 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi); 4875 } 4876 4877 /* allow QMANs to accept work from ARC CQF */ 4878 WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1)); 4879 } 4880 gaudi2_init_qman_pqc(struct hl_device * hdev,u32 reg_base,u32 queue_id_base)4881 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, 4882 u32 queue_id_base) 4883 { 4884 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4885 u32 pq_id, pq_offset, so_base_lo, so_base_hi; 4886 4887 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4888 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4889 4890 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 4891 pq_offset = pq_id * 4; 4892 4893 /* Configure QMAN HBW to scratchpad as it is not needed */ 4894 WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset, 4895 lower_32_bits(gaudi2->scratchpad_bus_address)); 4896 WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset, 4897 upper_32_bits(gaudi2->scratchpad_bus_address)); 4898 WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset, 4899 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry))); 4900 4901 WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0); 4902 WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA); 4903 WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo); 4904 WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi); 4905 } 4906 4907 /* Enable QMAN H/W completion */ 4908 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 4909 } 4910 gaudi2_get_dyn_sp_reg(struct hl_device * hdev,u32 queue_id_base)4911 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) 4912 { 4913 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4914 u32 sp_reg_addr; 4915 4916 switch (queue_id_base) { 4917 case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3: 4918 fallthrough; 4919 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 4920 fallthrough; 4921 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 4922 fallthrough; 4923 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 4924 fallthrough; 4925 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 4926 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 4927 break; 4928 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 4929 fallthrough; 4930 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 4931 fallthrough; 4932 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 4933 fallthrough; 4934 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 4935 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); 4936 break; 4937 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 4938 fallthrough; 4939 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 4940 fallthrough; 4941 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 4942 fallthrough; 4943 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 4944 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); 4945 break; 4946 case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3: 4947 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); 4948 break; 4949 case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3: 4950 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); 4951 break; 4952 default: 4953 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); 4954 return 0; 4955 } 4956 4957 return sp_reg_addr; 4958 } 4959 gaudi2_init_qman_common(struct hl_device * hdev,u32 reg_base,u32 queue_id_base)4960 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, 4961 u32 queue_id_base) 4962 { 4963 u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset; 4964 int map_table_entry; 4965 4966 WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot); 4967 4968 irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base); 4969 WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset)); 4970 WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset)); 4971 4972 map_table_entry = gaudi2_qman_async_event_id[queue_id_base]; 4973 WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET, 4974 gaudi2_irq_map_table[map_table_entry].cpu_id); 4975 4976 WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK); 4977 4978 WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT); 4979 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0); 4980 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0); 4981 4982 /* Enable the QMAN channel. 4983 * PDMA QMAN configuration is different, as we do not allow user to 4984 * access some of the CPs. 4985 * PDMA0: CP2/3 are reserved for the ARC usage. 4986 * PDMA1: CP1/2/3 are reserved for the ARC usage. 4987 */ 4988 if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]) 4989 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE); 4990 else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]) 4991 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE); 4992 else 4993 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE); 4994 } 4995 gaudi2_init_qman(struct hl_device * hdev,u32 reg_base,u32 queue_id_base)4996 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, 4997 u32 queue_id_base) 4998 { 4999 u32 pq_id; 5000 5001 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) 5002 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; 5003 5004 gaudi2_init_qman_pq(hdev, reg_base, queue_id_base); 5005 gaudi2_init_qman_cp(hdev, reg_base); 5006 gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base); 5007 gaudi2_init_qman_common(hdev, reg_base, queue_id_base); 5008 } 5009 gaudi2_init_dma_core(struct hl_device * hdev,u32 reg_base,u32 dma_core_id,bool is_secure)5010 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, 5011 u32 dma_core_id, bool is_secure) 5012 { 5013 u32 prot, irq_handler_offset; 5014 struct cpu_dyn_regs *dyn_regs; 5015 int map_table_entry; 5016 5017 prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT; 5018 if (is_secure) 5019 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT; 5020 5021 WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot); 5022 5023 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5024 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); 5025 5026 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET, 5027 lower_32_bits(CFG_BASE + irq_handler_offset)); 5028 5029 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET, 5030 upper_32_bits(CFG_BASE + irq_handler_offset)); 5031 5032 map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id]; 5033 WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET, 5034 gaudi2_irq_map_table[map_table_entry].cpu_id); 5035 5036 /* Enable the DMA channel */ 5037 WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT); 5038 } 5039 gaudi2_init_kdma(struct hl_device * hdev)5040 static void gaudi2_init_kdma(struct hl_device *hdev) 5041 { 5042 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5043 u32 reg_base; 5044 5045 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) 5046 return; 5047 5048 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA]; 5049 5050 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true); 5051 5052 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; 5053 } 5054 gaudi2_init_pdma(struct hl_device * hdev)5055 static void gaudi2_init_pdma(struct hl_device *hdev) 5056 { 5057 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5058 u32 reg_base; 5059 5060 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) 5061 return; 5062 5063 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0]; 5064 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false); 5065 5066 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]; 5067 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0); 5068 5069 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1]; 5070 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false); 5071 5072 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]; 5073 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0); 5074 5075 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; 5076 } 5077 gaudi2_init_edma_instance(struct hl_device * hdev,u8 seq)5078 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) 5079 { 5080 u32 reg_base, base_edma_core_id, base_edma_qman_id; 5081 5082 base_edma_core_id = DMA_CORE_ID_EDMA0 + seq; 5083 base_edma_qman_id = edma_stream_base[seq]; 5084 5085 reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id]; 5086 gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false); 5087 5088 reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id]; 5089 gaudi2_init_qman(hdev, reg_base, base_edma_qman_id); 5090 } 5091 gaudi2_init_edma(struct hl_device * hdev)5092 static void gaudi2_init_edma(struct hl_device *hdev) 5093 { 5094 struct asic_fixed_properties *prop = &hdev->asic_prop; 5095 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5096 int dcore, inst; 5097 5098 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) 5099 return; 5100 5101 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 5102 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 5103 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 5104 5105 if (!(prop->edma_enabled_mask & BIT(seq))) 5106 continue; 5107 5108 gaudi2_init_edma_instance(hdev, seq); 5109 5110 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); 5111 } 5112 } 5113 } 5114 5115 /* 5116 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell. 5117 * @hdev: pointer to habanalabs device structure. 5118 * @sob_id: sync object ID. 5119 * @first_mon_id: ID of first monitor out of 3 consecutive monitors. 5120 * @interrupt_id: interrupt ID. 5121 * 5122 * Some initiators cannot have HBW address in their completion address registers, and thus cannot 5123 * write directly to the HBW host memory of the virtual MSI-X doorbell. 5124 * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write. 5125 * 5126 * The mechanism in the sync manager block is composed of a master monitor with 3 messages. 5127 * In addition to the HBW write, the other 2 messages are for preparing the monitor to next 5128 * completion, by decrementing the sync object value and re-arming the monitor. 5129 */ gaudi2_arm_monitors_for_virt_msix_db(struct hl_device * hdev,u32 sob_id,u32 first_mon_id,u32 interrupt_id)5130 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, 5131 u32 first_mon_id, u32 interrupt_id) 5132 { 5133 u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config; 5134 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5135 u64 addr; 5136 u8 mask; 5137 5138 /* Reset the SOB value */ 5139 sob_offset = sob_id * sizeof(u32); 5140 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 5141 5142 /* Configure 3 monitors: 5143 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) 5144 * 2. Decrement SOB value by 1. 5145 * 3. Re-arm the master monitor. 5146 */ 5147 5148 first_mon_offset = first_mon_id * sizeof(u32); 5149 5150 /* 2nd monitor: Decrement SOB value by 1 */ 5151 mon_offset = first_mon_offset + sizeof(u32); 5152 5153 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 5154 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5155 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5156 5157 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ 5158 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) | 5159 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1); 5160 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5161 5162 /* 3rd monitor: Re-arm the master monitor */ 5163 mon_offset = first_mon_offset + 2 * sizeof(u32); 5164 5165 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset; 5166 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5167 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5168 5169 sob_group = sob_id / 8; 5170 mask = ~BIT(sob_id & 0x7); 5171 mode = 0; /* comparison mode is "greater than or equal to" */ 5172 arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) | 5173 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) | 5174 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) | 5175 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1); 5176 5177 payload = arm; 5178 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5179 5180 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ 5181 mon_offset = first_mon_offset; 5182 5183 config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */ 5184 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config); 5185 5186 addr = gaudi2->virt_msix_db_dma_addr; 5187 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5188 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5189 5190 payload = interrupt_id; 5191 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5192 5193 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm); 5194 } 5195 gaudi2_prepare_sm_for_virt_msix_db(struct hl_device * hdev)5196 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) 5197 { 5198 u32 decoder_id, sob_id, first_mon_id, interrupt_id; 5199 struct asic_fixed_properties *prop = &hdev->asic_prop; 5200 5201 /* Decoder normal/abnormal interrupts */ 5202 for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) { 5203 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) 5204 continue; 5205 5206 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5207 first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id; 5208 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id; 5209 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5210 5211 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5212 first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id; 5213 interrupt_id += 1; 5214 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5215 } 5216 } 5217 gaudi2_init_sm(struct hl_device * hdev)5218 static void gaudi2_init_sm(struct hl_device *hdev) 5219 { 5220 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5221 u64 cq_address; 5222 u32 reg_val; 5223 int i; 5224 5225 /* Enable HBW/LBW CQ for completion monitors */ 5226 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5227 reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1); 5228 5229 for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++) 5230 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5231 5232 /* Enable only HBW CQ for KDMA completion monitor */ 5233 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5234 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5235 5236 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ 5237 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); 5238 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); 5239 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION); 5240 5241 for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) { 5242 cq_address = 5243 hdev->completion_queue[i].bus_address; 5244 5245 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i), 5246 lower_32_bits(cq_address)); 5247 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i), 5248 upper_32_bits(cq_address)); 5249 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i), 5250 ilog2(HL_CQ_SIZE_IN_BYTES)); 5251 } 5252 5253 /* Configure kernel ASID and MMU BP*/ 5254 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000); 5255 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0); 5256 5257 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ 5258 gaudi2_prepare_sm_for_virt_msix_db(hdev); 5259 } 5260 gaudi2_init_mme_acc(struct hl_device * hdev,u32 reg_base)5261 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) 5262 { 5263 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5264 u32 reg_val; 5265 int i; 5266 5267 reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0); 5268 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1); 5269 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1); 5270 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1); 5271 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1); 5272 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1); 5273 5274 WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val); 5275 WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF); 5276 5277 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) { 5278 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i); 5279 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); 5280 } 5281 } 5282 gaudi2_init_dcore_mme(struct hl_device * hdev,int dcore_id,bool config_qman_only)5283 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, 5284 bool config_qman_only) 5285 { 5286 u32 queue_id_base, reg_base; 5287 5288 switch (dcore_id) { 5289 case 0: 5290 queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 5291 break; 5292 case 1: 5293 queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 5294 break; 5295 case 2: 5296 queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 5297 break; 5298 case 3: 5299 queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 5300 break; 5301 default: 5302 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); 5303 return; 5304 } 5305 5306 if (!config_qman_only) { 5307 reg_base = gaudi2_mme_acc_blocks_bases[dcore_id]; 5308 gaudi2_init_mme_acc(hdev, reg_base); 5309 } 5310 5311 reg_base = gaudi2_qm_blocks_bases[queue_id_base]; 5312 gaudi2_init_qman(hdev, reg_base, queue_id_base); 5313 } 5314 gaudi2_init_mme(struct hl_device * hdev)5315 static void gaudi2_init_mme(struct hl_device *hdev) 5316 { 5317 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5318 int i; 5319 5320 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) 5321 return; 5322 5323 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 5324 gaudi2_init_dcore_mme(hdev, i, false); 5325 5326 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); 5327 } 5328 } 5329 gaudi2_init_tpc_cfg(struct hl_device * hdev,u32 reg_base)5330 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) 5331 { 5332 /* Mask arithmetic and QM interrupts in TPC */ 5333 WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE); 5334 5335 /* Set 16 cache lines */ 5336 WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET, 5337 2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT); 5338 } 5339 5340 struct gaudi2_tpc_init_cfg_data { 5341 enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES]; 5342 }; 5343 gaudi2_init_tpc_config(struct hl_device * hdev,int dcore,int inst,u32 offset,struct iterate_module_ctx * ctx)5344 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, 5345 u32 offset, struct iterate_module_ctx *ctx) 5346 { 5347 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5348 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; 5349 u32 queue_id_base; 5350 u8 seq; 5351 5352 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); 5353 5354 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) 5355 /* gets last sequence number */ 5356 seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE; 5357 else 5358 seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 5359 5360 gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset); 5361 gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base); 5362 5363 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); 5364 } 5365 gaudi2_init_tpc(struct hl_device * hdev)5366 static void gaudi2_init_tpc(struct hl_device *hdev) 5367 { 5368 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5369 struct gaudi2_tpc_init_cfg_data init_cfg_data; 5370 struct iterate_module_ctx tpc_iter; 5371 5372 if (!hdev->asic_prop.tpc_enabled_mask) 5373 return; 5374 5375 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) 5376 return; 5377 5378 init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0; 5379 init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0; 5380 init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0; 5381 init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0; 5382 tpc_iter.fn = &gaudi2_init_tpc_config; 5383 tpc_iter.data = &init_cfg_data; 5384 gaudi2_iterate_tpcs(hdev, &tpc_iter); 5385 } 5386 gaudi2_init_rotator(struct hl_device * hdev)5387 static void gaudi2_init_rotator(struct hl_device *hdev) 5388 { 5389 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5390 u32 i, reg_base, queue_id; 5391 5392 queue_id = GAUDI2_QUEUE_ID_ROT_0_0; 5393 5394 for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 5395 reg_base = gaudi2_qm_blocks_bases[queue_id]; 5396 gaudi2_init_qman(hdev, reg_base, queue_id); 5397 5398 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); 5399 } 5400 } 5401 gaudi2_init_vdec_brdg_ctrl(struct hl_device * hdev,u64 base_addr,u32 decoder_id)5402 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) 5403 { 5404 u32 sob_id; 5405 5406 /* VCMD normal interrupt */ 5407 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5408 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, 5409 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5410 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5411 5412 /* VCMD abnormal interrupt */ 5413 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5414 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, 5415 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5416 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5417 } 5418 gaudi2_init_dec(struct hl_device * hdev)5419 static void gaudi2_init_dec(struct hl_device *hdev) 5420 { 5421 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5422 u32 dcore_id, dec_id, dec_bit; 5423 u64 base_addr; 5424 5425 if (!hdev->asic_prop.decoder_enabled_mask) 5426 return; 5427 5428 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) 5429 return; 5430 5431 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 5432 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 5433 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 5434 5435 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5436 continue; 5437 5438 base_addr = mmDCORE0_DEC0_CMD_BASE + 5439 BRDG_CTRL_BLOCK_OFFSET + 5440 dcore_id * DCORE_OFFSET + 5441 dec_id * DCORE_VDEC_OFFSET; 5442 5443 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5444 5445 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5446 } 5447 5448 for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) { 5449 dec_bit = PCIE_DEC_SHIFT + dec_id; 5450 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5451 continue; 5452 5453 base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET + 5454 dec_id * DCORE_VDEC_OFFSET; 5455 5456 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5457 5458 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5459 } 5460 } 5461 gaudi2_mmu_update_asid_hop0_addr(struct hl_device * hdev,u32 stlb_base,u32 asid,u64 phys_addr)5462 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, 5463 u32 stlb_base, u32 asid, u64 phys_addr) 5464 { 5465 u32 status, timeout_usec; 5466 int rc; 5467 5468 if (hdev->pldm || !hdev->pdev) 5469 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5470 else 5471 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 5472 5473 WREG32(stlb_base + STLB_ASID_OFFSET, asid); 5474 WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 5475 WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT); 5476 WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000); 5477 5478 rc = hl_poll_timeout( 5479 hdev, 5480 stlb_base + STLB_BUSY_OFFSET, 5481 status, 5482 !(status & 0x80000000), 5483 1000, 5484 timeout_usec); 5485 5486 if (rc) { 5487 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); 5488 return rc; 5489 } 5490 5491 return 0; 5492 } 5493 gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device * hdev,u32 stlb_base,u32 start_offset,u32 inv_start_val,u32 flags)5494 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, 5495 u32 start_offset, u32 inv_start_val, 5496 u32 flags) 5497 { 5498 /* clear PMMU mem line cache (only needed in mmu range invalidation) */ 5499 if (flags & MMU_OP_CLEAR_MEMCACHE) 5500 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1); 5501 5502 if (flags & MMU_OP_SKIP_LOW_CACHE_INV) 5503 return; 5504 5505 WREG32(stlb_base + start_offset, inv_start_val); 5506 } 5507 gaudi2_mmu_invalidate_cache_status_poll(struct hl_device * hdev,u32 stlb_base,struct gaudi2_cache_invld_params * inv_params)5508 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, 5509 struct gaudi2_cache_invld_params *inv_params) 5510 { 5511 u32 status, timeout_usec, start_offset; 5512 int rc; 5513 5514 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : 5515 GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5516 5517 /* poll PMMU mem line cache (only needed in mmu range invalidation) */ 5518 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { 5519 rc = hl_poll_timeout( 5520 hdev, 5521 mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 5522 status, 5523 status & 0x1, 5524 1000, 5525 timeout_usec); 5526 5527 if (rc) 5528 return rc; 5529 5530 /* Need to manually reset the status to 0 */ 5531 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0); 5532 } 5533 5534 /* Lower cache does not work with cache lines, hence we can skip its 5535 * invalidation upon map and invalidate only upon unmap 5536 */ 5537 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) 5538 return 0; 5539 5540 start_offset = inv_params->range_invalidation ? 5541 STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET; 5542 5543 rc = hl_poll_timeout( 5544 hdev, 5545 stlb_base + start_offset, 5546 status, 5547 !(status & 0x1), 5548 1000, 5549 timeout_usec); 5550 5551 return rc; 5552 } 5553 gaudi2_is_hmmu_enabled(struct hl_device * hdev,int dcore_id,int hmmu_id)5554 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) 5555 { 5556 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5557 u32 hw_cap; 5558 5559 hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id); 5560 5561 if (gaudi2->hw_cap_initialized & hw_cap) 5562 return true; 5563 5564 return false; 5565 } 5566 5567 /* this function shall be called only for HMMUs for which capability bit is set */ get_hmmu_stlb_base(int dcore_id,int hmmu_id)5568 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id) 5569 { 5570 u32 offset; 5571 5572 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5573 return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset); 5574 } 5575 gaudi2_mmu_invalidate_cache_trigger(struct hl_device * hdev,u32 stlb_base,struct gaudi2_cache_invld_params * inv_params)5576 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, 5577 struct gaudi2_cache_invld_params *inv_params) 5578 { 5579 u32 start_offset; 5580 5581 if (inv_params->range_invalidation) { 5582 /* Set the addresses range 5583 * Note: that the start address we set in register, is not included in 5584 * the range of the invalidation, by design. 5585 * that's why we need to set lower address than the one we actually 5586 * want to be included in the range invalidation. 5587 */ 5588 u64 start = inv_params->start_va - 1; 5589 5590 start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET; 5591 5592 WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET, 5593 start >> MMU_RANGE_INV_VA_LSB_SHIFT); 5594 5595 WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET, 5596 start >> MMU_RANGE_INV_VA_MSB_SHIFT); 5597 5598 WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET, 5599 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); 5600 5601 WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET, 5602 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); 5603 } else { 5604 start_offset = STLB_INV_ALL_START_OFFSET; 5605 } 5606 5607 gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset, 5608 inv_params->inv_start_val, inv_params->flags); 5609 } 5610 gaudi2_hmmu_invalidate_cache_trigger(struct hl_device * hdev,int dcore_id,int hmmu_id,struct gaudi2_cache_invld_params * inv_params)5611 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, 5612 int dcore_id, int hmmu_id, 5613 struct gaudi2_cache_invld_params *inv_params) 5614 { 5615 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5616 5617 gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params); 5618 } 5619 gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device * hdev,int dcore_id,int hmmu_id,struct gaudi2_cache_invld_params * inv_params)5620 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, 5621 int dcore_id, int hmmu_id, 5622 struct gaudi2_cache_invld_params *inv_params) 5623 { 5624 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5625 5626 return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params); 5627 } 5628 gaudi2_hmmus_invalidate_cache(struct hl_device * hdev,struct gaudi2_cache_invld_params * inv_params)5629 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, 5630 struct gaudi2_cache_invld_params *inv_params) 5631 { 5632 int dcore_id, hmmu_id; 5633 5634 /* first send all invalidation commands */ 5635 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5636 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5637 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5638 continue; 5639 5640 gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params); 5641 } 5642 } 5643 5644 /* next, poll all invalidations status */ 5645 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5646 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5647 int rc; 5648 5649 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5650 continue; 5651 5652 rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id, 5653 inv_params); 5654 if (rc) 5655 return rc; 5656 } 5657 } 5658 5659 return 0; 5660 } 5661 gaudi2_mmu_invalidate_cache(struct hl_device * hdev,bool is_hard,u32 flags)5662 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) 5663 { 5664 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5665 struct gaudi2_cache_invld_params invld_params; 5666 int rc = 0; 5667 5668 if (hdev->reset_info.hard_reset_pending) 5669 return rc; 5670 5671 invld_params.range_invalidation = false; 5672 invld_params.inv_start_val = 1; 5673 5674 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5675 invld_params.flags = flags; 5676 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5677 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5678 &invld_params); 5679 } else if (flags & MMU_OP_PHYS_PACK) { 5680 invld_params.flags = 0; 5681 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5682 } 5683 5684 return rc; 5685 } 5686 gaudi2_mmu_invalidate_cache_range(struct hl_device * hdev,bool is_hard,u32 flags,u32 asid,u64 va,u64 size)5687 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, 5688 u32 flags, u32 asid, u64 va, u64 size) 5689 { 5690 struct gaudi2_cache_invld_params invld_params = {0}; 5691 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5692 u64 start_va, end_va; 5693 u32 inv_start_val; 5694 int rc = 0; 5695 5696 if (hdev->reset_info.hard_reset_pending) 5697 return 0; 5698 5699 inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT | 5700 1 << MMU_RANGE_INV_ASID_EN_SHIFT | 5701 asid << MMU_RANGE_INV_ASID_SHIFT); 5702 start_va = va; 5703 end_va = start_va + size; 5704 5705 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5706 /* As range invalidation does not support zero address we will 5707 * do full invalidation in this case 5708 */ 5709 if (start_va) { 5710 invld_params.range_invalidation = true; 5711 invld_params.start_va = start_va; 5712 invld_params.end_va = end_va; 5713 invld_params.inv_start_val = inv_start_val; 5714 invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE; 5715 } else { 5716 invld_params.range_invalidation = false; 5717 invld_params.inv_start_val = 1; 5718 invld_params.flags = flags; 5719 } 5720 5721 5722 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5723 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5724 &invld_params); 5725 if (rc) 5726 return rc; 5727 5728 } else if (flags & MMU_OP_PHYS_PACK) { 5729 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va); 5730 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va); 5731 invld_params.inv_start_val = inv_start_val; 5732 invld_params.flags = flags; 5733 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5734 } 5735 5736 return rc; 5737 } 5738 gaudi2_mmu_update_hop0_addr(struct hl_device * hdev,u32 stlb_base)5739 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base) 5740 { 5741 struct asic_fixed_properties *prop = &hdev->asic_prop; 5742 u64 hop0_addr; 5743 u32 asid, max_asid = prop->max_asid; 5744 int rc; 5745 5746 /* it takes too much time to init all of the ASIDs on palladium */ 5747 if (hdev->pldm) 5748 max_asid = min((u32) 8, max_asid); 5749 5750 for (asid = 0 ; asid < max_asid ; asid++) { 5751 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; 5752 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr); 5753 if (rc) { 5754 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); 5755 return rc; 5756 } 5757 } 5758 5759 return 0; 5760 } 5761 gaudi2_mmu_init_common(struct hl_device * hdev,u32 mmu_base,u32 stlb_base)5762 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base) 5763 { 5764 u32 status, timeout_usec; 5765 int rc; 5766 5767 if (hdev->pldm || !hdev->pdev) 5768 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5769 else 5770 timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5771 5772 WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1); 5773 5774 rc = hl_poll_timeout( 5775 hdev, 5776 stlb_base + STLB_SRAM_INIT_OFFSET, 5777 status, 5778 !status, 5779 1000, 5780 timeout_usec); 5781 5782 if (rc) 5783 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); 5784 5785 rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base); 5786 if (rc) 5787 return rc; 5788 5789 WREG32(mmu_base + MMU_BYPASS_OFFSET, 0); 5790 5791 rc = hl_poll_timeout( 5792 hdev, 5793 stlb_base + STLB_INV_ALL_START_OFFSET, 5794 status, 5795 !status, 5796 1000, 5797 timeout_usec); 5798 5799 if (rc) 5800 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); 5801 5802 WREG32(mmu_base + MMU_ENABLE_OFFSET, 1); 5803 5804 return rc; 5805 } 5806 gaudi2_pci_mmu_init(struct hl_device * hdev)5807 static int gaudi2_pci_mmu_init(struct hl_device *hdev) 5808 { 5809 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5810 u32 mmu_base, stlb_base; 5811 int rc; 5812 5813 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) 5814 return 0; 5815 5816 mmu_base = mmPMMU_HBW_MMU_BASE; 5817 stlb_base = mmPMMU_HBW_STLB_BASE; 5818 5819 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 5820 (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) | 5821 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) | 5822 (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) | 5823 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) | 5824 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT), 5825 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 5826 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 5827 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 5828 PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 5829 PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 5830 5831 WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0); 5832 5833 if (PAGE_SIZE == SZ_64K) { 5834 /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */ 5835 RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5836 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) | 5837 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) | 5838 FIELD_PREP( 5839 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK, 5840 1), 5841 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK | 5842 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK | 5843 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK); 5844 } 5845 5846 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK); 5847 5848 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); 5849 if (rc) 5850 return rc; 5851 5852 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; 5853 5854 return 0; 5855 } 5856 gaudi2_dcore_hmmu_init(struct hl_device * hdev,int dcore_id,int hmmu_id)5857 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, 5858 int hmmu_id) 5859 { 5860 struct asic_fixed_properties *prop = &hdev->asic_prop; 5861 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5862 u32 offset, mmu_base, stlb_base, hw_cap; 5863 u8 dmmu_seq; 5864 int rc; 5865 5866 dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id; 5867 hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq; 5868 5869 /* 5870 * return if DMMU is already initialized or if it's not out of 5871 * isolation (due to cluster binning) 5872 */ 5873 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) 5874 return 0; 5875 5876 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5877 mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset; 5878 stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset; 5879 5880 RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */, 5881 MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK); 5882 5883 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 5884 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) | 5885 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) | 5886 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) | 5887 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) | 5888 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3), 5889 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 5890 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 5891 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 5892 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 5893 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 5894 5895 RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, 5896 STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK); 5897 5898 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK); 5899 5900 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); 5901 if (rc) 5902 return rc; 5903 5904 gaudi2->hw_cap_initialized |= hw_cap; 5905 5906 return 0; 5907 } 5908 gaudi2_hbm_mmu_init(struct hl_device * hdev)5909 static int gaudi2_hbm_mmu_init(struct hl_device *hdev) 5910 { 5911 int rc, dcore_id, hmmu_id; 5912 5913 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 5914 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) { 5915 rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id); 5916 if (rc) 5917 return rc; 5918 } 5919 5920 return 0; 5921 } 5922 gaudi2_mmu_init(struct hl_device * hdev)5923 static int gaudi2_mmu_init(struct hl_device *hdev) 5924 { 5925 int rc; 5926 5927 rc = gaudi2_pci_mmu_init(hdev); 5928 if (rc) 5929 return rc; 5930 5931 rc = gaudi2_hbm_mmu_init(hdev); 5932 if (rc) 5933 return rc; 5934 5935 return 0; 5936 } 5937 gaudi2_hw_init(struct hl_device * hdev)5938 static int gaudi2_hw_init(struct hl_device *hdev) 5939 { 5940 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5941 int rc; 5942 5943 /* Let's mark in the H/W that we have reached this point. We check 5944 * this value in the reset_before_init function to understand whether 5945 * we need to reset the chip before doing H/W init. This register is 5946 * cleared by the H/W upon H/W reset 5947 */ 5948 WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 5949 5950 /* Perform read from the device to make sure device is up */ 5951 RREG32(mmHW_STATE); 5952 5953 /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. 5954 * So we set it here and if anyone tries to move it later to 5955 * a different address, there will be an error 5956 */ 5957 if (hdev->asic_prop.iatu_done_by_fw) 5958 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; 5959 5960 /* 5961 * Before pushing u-boot/linux to device, need to set the hbm bar to 5962 * base address of dram 5963 */ 5964 if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 5965 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); 5966 return -EIO; 5967 } 5968 5969 rc = gaudi2_init_cpu(hdev); 5970 if (rc) { 5971 dev_err(hdev->dev, "failed to initialize CPU\n"); 5972 return rc; 5973 } 5974 5975 gaudi2_init_scrambler_hbm(hdev); 5976 gaudi2_init_kdma(hdev); 5977 5978 rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC); 5979 if (rc) { 5980 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); 5981 return rc; 5982 } 5983 5984 rc = gaudi2->cpucp_info_get(hdev); 5985 if (rc) { 5986 dev_err(hdev->dev, "Failed to get cpucp info\n"); 5987 return rc; 5988 } 5989 5990 rc = gaudi2_mmu_init(hdev); 5991 if (rc) 5992 return rc; 5993 5994 gaudi2_init_pdma(hdev); 5995 gaudi2_init_edma(hdev); 5996 gaudi2_init_sm(hdev); 5997 gaudi2_init_tpc(hdev); 5998 gaudi2_init_mme(hdev); 5999 gaudi2_init_rotator(hdev); 6000 gaudi2_init_dec(hdev); 6001 gaudi2_enable_timestamp(hdev); 6002 6003 rc = gaudi2_coresight_init(hdev); 6004 if (rc) 6005 goto disable_queues; 6006 6007 rc = gaudi2_enable_msix(hdev); 6008 if (rc) 6009 goto disable_queues; 6010 6011 /* Perform read from the device to flush all configuration */ 6012 RREG32(mmHW_STATE); 6013 6014 return 0; 6015 6016 disable_queues: 6017 gaudi2_disable_dma_qmans(hdev); 6018 gaudi2_disable_mme_qmans(hdev); 6019 gaudi2_disable_tpc_qmans(hdev); 6020 gaudi2_disable_rot_qmans(hdev); 6021 gaudi2_disable_nic_qmans(hdev); 6022 6023 gaudi2_disable_timestamp(hdev); 6024 6025 return rc; 6026 } 6027 6028 /** 6029 * gaudi2_send_hard_reset_cmd - common function to handle reset 6030 * 6031 * @hdev: pointer to the habanalabs device structure 6032 * 6033 * This function handles the various possible scenarios for reset. 6034 * It considers if reset is handled by driver\FW and what FW components are loaded 6035 */ gaudi2_send_hard_reset_cmd(struct hl_device * hdev)6036 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) 6037 { 6038 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6039 bool heartbeat_reset, preboot_only, cpu_initialized = false; 6040 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6041 u32 cpu_boot_status; 6042 6043 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); 6044 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); 6045 6046 /* 6047 * Handle corner case where failure was at cpu management app load, 6048 * and driver didn't detect any failure while loading the FW, 6049 * then at such scenario driver will send only HALT_MACHINE 6050 * and no one will respond to this request since FW already back to preboot 6051 * and it cannot handle such cmd. 6052 * In this case next time the management app loads it'll check on events register 6053 * which will still have the halt indication, and will reboot the device. 6054 * The solution is to let preboot clear all relevant registers before next boot 6055 * once driver send COMMS_RST_DEV. 6056 */ 6057 cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS); 6058 6059 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && 6060 (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL)) 6061 cpu_initialized = true; 6062 6063 /* 6064 * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways: 6065 * 1. FW reset: FW initiate the reset sequence 6066 * 2. driver reset: FW will start HALT sequence (the preparations for the 6067 * reset but not the reset itself as it is not implemented 6068 * on their part) and LKD will wait to let FW complete the 6069 * sequence before issuing the reset 6070 */ 6071 if (!preboot_only && cpu_initialized) { 6072 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), 6073 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id); 6074 6075 msleep(GAUDI2_CPU_RESET_WAIT_MSEC); 6076 } 6077 6078 /* 6079 * When working with preboot (without Linux/Boot fit) we can 6080 * communicate only using the COMMS commands to issue halt/reset. 6081 * 6082 * For the case in which we are working with Linux/Bootfit this is a hail-mary 6083 * attempt to revive the card in the small chance that the f/w has 6084 * experienced a watchdog event, which caused it to return back to preboot. 6085 * In that case, triggering reset through GIC won't help. We need to 6086 * trigger the reset as if Linux wasn't loaded. 6087 * 6088 * We do it only if the reset cause was HB, because that would be the 6089 * indication of such an event. 6090 * 6091 * In case watchdog hasn't expired but we still got HB, then this won't 6092 * do any damage. 6093 */ 6094 6095 if (heartbeat_reset || preboot_only || !cpu_initialized) { 6096 if (hdev->asic_prop.hard_reset_done_by_fw) 6097 hl_fw_ask_hard_reset_without_linux(hdev); 6098 else 6099 hl_fw_ask_halt_machine_without_linux(hdev); 6100 } 6101 } 6102 6103 /** 6104 * gaudi2_execute_hard_reset - execute hard reset by driver/FW 6105 * 6106 * @hdev: pointer to the habanalabs device structure 6107 * 6108 * This function executes hard reset based on if driver/FW should do the reset 6109 */ gaudi2_execute_hard_reset(struct hl_device * hdev)6110 static void gaudi2_execute_hard_reset(struct hl_device *hdev) 6111 { 6112 if (hdev->asic_prop.hard_reset_done_by_fw) { 6113 gaudi2_send_hard_reset_cmd(hdev); 6114 return; 6115 } 6116 6117 /* Set device to handle FLR by H/W as we will put the device 6118 * CPU to halt mode 6119 */ 6120 WREG32(mmPCIE_AUX_FLR_CTRL, 6121 (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); 6122 6123 gaudi2_send_hard_reset_cmd(hdev); 6124 6125 WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1); 6126 } 6127 gaudi2_get_soft_rst_done_indication(struct hl_device * hdev,u32 poll_timeout_us)6128 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us) 6129 { 6130 int i, rc = 0; 6131 u32 reg_val; 6132 6133 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6134 rc = hl_poll_timeout( 6135 hdev, 6136 mmCPU_RST_STATUS_TO_HOST, 6137 reg_val, 6138 reg_val == CPU_RST_STATUS_SOFT_RST_DONE, 6139 1000, 6140 poll_timeout_us); 6141 6142 if (rc) 6143 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", 6144 reg_val); 6145 return rc; 6146 } 6147 6148 /** 6149 * gaudi2_execute_soft_reset - execute soft reset by driver/FW 6150 * 6151 * @hdev: pointer to the habanalabs device structure 6152 * @driver_performs_reset: true if driver should perform reset instead of f/w. 6153 * @poll_timeout_us: time to wait for response from f/w. 6154 * 6155 * This function executes soft reset based on if driver/FW should do the reset 6156 */ gaudi2_execute_soft_reset(struct hl_device * hdev,bool driver_performs_reset,u32 poll_timeout_us)6157 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, 6158 u32 poll_timeout_us) 6159 { 6160 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6161 int rc = 0; 6162 6163 if (!driver_performs_reset) { 6164 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) { 6165 /* set SP to indicate reset request sent to FW */ 6166 if (dyn_regs->cpu_rst_status) 6167 WREG32(le32_to_cpu(dyn_regs->cpu_rst_status), CPU_RST_STATUS_NA); 6168 else 6169 WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA); 6170 WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq), 6171 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id); 6172 6173 /* wait for f/w response */ 6174 rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us); 6175 } else { 6176 rc = hl_fw_send_soft_reset(hdev); 6177 } 6178 return rc; 6179 } 6180 6181 /* Block access to engines, QMANs and SM during reset, these 6182 * RRs will be reconfigured after soft reset. 6183 * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset. 6184 */ 6185 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, 6186 mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE); 6187 6188 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, 6189 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE, 6190 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE); 6191 6192 WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1); 6193 return 0; 6194 } 6195 gaudi2_poll_btm_indication(struct hl_device * hdev,u32 poll_timeout_us)6196 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) 6197 { 6198 int i, rc = 0; 6199 u32 reg_val; 6200 6201 /* We poll the BTM done indication multiple times after reset due to 6202 * a HW errata 'GAUDI2_0300' 6203 */ 6204 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6205 rc = hl_poll_timeout( 6206 hdev, 6207 mmPSOC_GLOBAL_CONF_BTM_FSM, 6208 reg_val, 6209 reg_val == 0, 6210 1000, 6211 poll_timeout_us); 6212 6213 if (rc) 6214 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); 6215 } 6216 gaudi2_hw_fini(struct hl_device * hdev,bool hard_reset,bool fw_reset)6217 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 6218 { 6219 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6220 u32 poll_timeout_us, reset_sleep_ms; 6221 bool driver_performs_reset = false; 6222 int rc; 6223 6224 if (hdev->pldm) { 6225 reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC : 6226 GAUDI2_PLDM_SRESET_TIMEOUT_MSEC; 6227 poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC; 6228 } else { 6229 reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC; 6230 poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC; 6231 } 6232 6233 if (fw_reset) 6234 goto skip_reset; 6235 6236 gaudi2_reset_arcs(hdev); 6237 6238 if (hard_reset) { 6239 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; 6240 gaudi2_execute_hard_reset(hdev); 6241 } else { 6242 /* 6243 * As we have to support also work with preboot only (which does not supports 6244 * soft reset) we have to make sure that security is disabled before letting driver 6245 * do the reset. user shall control the BFE flags to avoid asking soft reset in 6246 * secured device with preboot only. 6247 */ 6248 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && 6249 !hdev->asic_prop.fw_security_enabled); 6250 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us); 6251 if (rc) 6252 return rc; 6253 } 6254 6255 skip_reset: 6256 if (driver_performs_reset || hard_reset) { 6257 /* 6258 * Instead of waiting for BTM indication we should wait for preboot ready: 6259 * Consider the below scenario: 6260 * 1. FW update is being triggered 6261 * - setting the dirty bit 6262 * 2. hard reset will be triggered due to the dirty bit 6263 * 3. FW initiates the reset: 6264 * - dirty bit cleared 6265 * - BTM indication cleared 6266 * - preboot ready indication cleared 6267 * 4. during hard reset: 6268 * - BTM indication will be set 6269 * - BIST test performed and another reset triggered 6270 * 5. only after this reset the preboot will set the preboot ready 6271 * 6272 * when polling on BTM indication alone we can lose sync with FW while trying to 6273 * communicate with FW that is during reset. 6274 * to overcome this we will always wait to preboot ready indication 6275 */ 6276 6277 /* without this sleep reset will not work */ 6278 msleep(reset_sleep_ms); 6279 6280 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) 6281 hl_fw_wait_preboot_ready(hdev); 6282 else 6283 gaudi2_poll_btm_indication(hdev, poll_timeout_us); 6284 } 6285 6286 if (!gaudi2) 6287 return 0; 6288 6289 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); 6290 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); 6291 6292 /* 6293 * Clear NIC capability mask in order for driver to re-configure 6294 * NIC QMANs. NIC ports will not be re-configured during soft 6295 * reset as we call gaudi2_nic_init only during hard reset 6296 */ 6297 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); 6298 6299 if (hard_reset) { 6300 gaudi2->hw_cap_initialized &= 6301 ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK | 6302 HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q | 6303 HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK | 6304 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA | 6305 HW_CAP_MME_MASK | HW_CAP_ROT_MASK); 6306 6307 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); 6308 } else { 6309 gaudi2->hw_cap_initialized &= 6310 ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET | 6311 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK | 6312 HW_CAP_ROT_MASK); 6313 } 6314 return 0; 6315 } 6316 gaudi2_suspend(struct hl_device * hdev)6317 static int gaudi2_suspend(struct hl_device *hdev) 6318 { 6319 int rc; 6320 6321 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 6322 if (rc) 6323 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); 6324 6325 return rc; 6326 } 6327 gaudi2_resume(struct hl_device * hdev)6328 static int gaudi2_resume(struct hl_device *hdev) 6329 { 6330 return gaudi2_init_iatu(hdev); 6331 } 6332 gaudi2_mmap(struct hl_device * hdev,struct vm_area_struct * vma,void * cpu_addr,dma_addr_t dma_addr,size_t size)6333 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 6334 void *cpu_addr, dma_addr_t dma_addr, size_t size) 6335 { 6336 int rc; 6337 6338 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 6339 VM_DONTCOPY | VM_NORESERVE); 6340 6341 #ifdef _HAS_DMA_MMAP_COHERENT 6342 6343 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); 6344 if (rc) 6345 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 6346 6347 #else 6348 6349 rc = remap_pfn_range(vma, vma->vm_start, 6350 virt_to_phys(cpu_addr) >> PAGE_SHIFT, 6351 size, vma->vm_page_prot); 6352 if (rc) 6353 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 6354 6355 #endif 6356 6357 return rc; 6358 } 6359 gaudi2_is_queue_enabled(struct hl_device * hdev,u32 hw_queue_id)6360 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) 6361 { 6362 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6363 u64 hw_cap_mask = 0; 6364 u64 hw_tpc_cap_bit = 0; 6365 u64 hw_nic_cap_bit = 0; 6366 u64 hw_test_cap_bit = 0; 6367 6368 switch (hw_queue_id) { 6369 case GAUDI2_QUEUE_ID_PDMA_0_0: 6370 case GAUDI2_QUEUE_ID_PDMA_0_1: 6371 case GAUDI2_QUEUE_ID_PDMA_1_0: 6372 hw_cap_mask = HW_CAP_PDMA_MASK; 6373 break; 6374 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 6375 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 6376 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); 6377 break; 6378 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 6379 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE + 6380 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); 6381 break; 6382 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 6383 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE + 6384 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); 6385 break; 6386 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 6387 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE + 6388 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); 6389 break; 6390 6391 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 6392 hw_test_cap_bit = HW_CAP_MME_SHIFT; 6393 break; 6394 6395 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 6396 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1; 6397 break; 6398 6399 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 6400 hw_test_cap_bit = HW_CAP_MME_SHIFT + 2; 6401 break; 6402 6403 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 6404 hw_test_cap_bit = HW_CAP_MME_SHIFT + 3; 6405 break; 6406 6407 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3: 6408 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + 6409 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); 6410 6411 /* special case where cap bit refers to the first queue id */ 6412 if (!hw_tpc_cap_bit) 6413 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); 6414 break; 6415 6416 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 6417 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE + 6418 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); 6419 break; 6420 6421 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 6422 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) + 6423 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); 6424 break; 6425 6426 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 6427 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) + 6428 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); 6429 break; 6430 6431 case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 6432 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE); 6433 break; 6434 6435 case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3: 6436 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); 6437 break; 6438 6439 case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3: 6440 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); 6441 6442 /* special case where cap bit refers to the first queue id */ 6443 if (!hw_nic_cap_bit) 6444 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); 6445 break; 6446 6447 case GAUDI2_QUEUE_ID_CPU_PQ: 6448 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); 6449 6450 default: 6451 return false; 6452 } 6453 6454 if (hw_tpc_cap_bit) 6455 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); 6456 6457 if (hw_nic_cap_bit) 6458 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); 6459 6460 if (hw_test_cap_bit) 6461 hw_cap_mask = BIT_ULL(hw_test_cap_bit); 6462 6463 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); 6464 } 6465 gaudi2_is_arc_enabled(struct hl_device * hdev,u64 arc_id)6466 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) 6467 { 6468 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6469 6470 switch (arc_id) { 6471 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6472 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6473 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); 6474 6475 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6476 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6477 6478 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6479 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6480 6481 default: 6482 return false; 6483 } 6484 } 6485 gaudi2_clr_arc_id_cap(struct hl_device * hdev,u64 arc_id)6486 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6487 { 6488 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6489 6490 switch (arc_id) { 6491 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6492 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6493 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); 6494 break; 6495 6496 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6497 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6498 break; 6499 6500 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6501 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6502 break; 6503 6504 default: 6505 return; 6506 } 6507 } 6508 gaudi2_set_arc_id_cap(struct hl_device * hdev,u64 arc_id)6509 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6510 { 6511 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6512 6513 switch (arc_id) { 6514 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6515 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6516 gaudi2->active_hw_arc |= BIT_ULL(arc_id); 6517 break; 6518 6519 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6520 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); 6521 break; 6522 6523 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6524 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); 6525 break; 6526 6527 default: 6528 return; 6529 } 6530 } 6531 gaudi2_ring_doorbell(struct hl_device * hdev,u32 hw_queue_id,u32 pi)6532 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 6533 { 6534 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6535 u32 pq_offset, reg_base, db_reg_offset, db_value; 6536 6537 if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) { 6538 /* 6539 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs. 6540 * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ 6541 * number. 6542 */ 6543 pq_offset = (hw_queue_id & 0x3) * 4; 6544 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6545 db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset; 6546 } else { 6547 db_reg_offset = mmCPU_IF_PF_PQ_PI; 6548 } 6549 6550 db_value = pi; 6551 6552 /* ring the doorbell */ 6553 WREG32(db_reg_offset, db_value); 6554 6555 if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) { 6556 /* make sure device CPU will read latest data from host */ 6557 mb(); 6558 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 6559 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 6560 } 6561 } 6562 gaudi2_pqe_write(struct hl_device * hdev,__le64 * pqe,struct hl_bd * bd)6563 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) 6564 { 6565 __le64 *pbd = (__le64 *) bd; 6566 6567 /* The QMANs are on the host memory so a simple copy suffice */ 6568 pqe[0] = pbd[0]; 6569 pqe[1] = pbd[1]; 6570 } 6571 gaudi2_dma_alloc_coherent(struct hl_device * hdev,size_t size,dma_addr_t * dma_handle,gfp_t flags)6572 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, 6573 dma_addr_t *dma_handle, gfp_t flags) 6574 { 6575 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); 6576 } 6577 gaudi2_dma_free_coherent(struct hl_device * hdev,size_t size,void * cpu_addr,dma_addr_t dma_handle)6578 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, 6579 void *cpu_addr, dma_addr_t dma_handle) 6580 { 6581 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); 6582 } 6583 gaudi2_send_cpu_message(struct hl_device * hdev,u32 * msg,u16 len,u32 timeout,u64 * result)6584 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, 6585 u32 timeout, u64 *result) 6586 { 6587 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6588 6589 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { 6590 if (result) 6591 *result = 0; 6592 return 0; 6593 } 6594 6595 if (!timeout) 6596 timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC; 6597 6598 return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result); 6599 } 6600 gaudi2_dma_pool_zalloc(struct hl_device * hdev,size_t size,gfp_t mem_flags,dma_addr_t * dma_handle)6601 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, 6602 gfp_t mem_flags, dma_addr_t *dma_handle) 6603 { 6604 if (size > GAUDI2_DMA_POOL_BLK_SIZE) 6605 return NULL; 6606 6607 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 6608 } 6609 gaudi2_dma_pool_free(struct hl_device * hdev,void * vaddr,dma_addr_t dma_addr)6610 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) 6611 { 6612 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); 6613 } 6614 gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device * hdev,size_t size,dma_addr_t * dma_handle)6615 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 6616 dma_addr_t *dma_handle) 6617 { 6618 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 6619 } 6620 gaudi2_cpu_accessible_dma_pool_free(struct hl_device * hdev,size_t size,void * vaddr)6621 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) 6622 { 6623 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 6624 } 6625 gaudi2_dma_map_single(struct hl_device * hdev,void * addr,int len,enum dma_data_direction dir)6626 static dma_addr_t gaudi2_dma_map_single(struct hl_device *hdev, void *addr, int len, 6627 enum dma_data_direction dir) 6628 { 6629 dma_addr_t dma_addr; 6630 6631 dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir); 6632 if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr))) 6633 return 0; 6634 6635 return dma_addr; 6636 } 6637 gaudi2_dma_unmap_single(struct hl_device * hdev,dma_addr_t addr,int len,enum dma_data_direction dir)6638 static void gaudi2_dma_unmap_single(struct hl_device *hdev, dma_addr_t addr, int len, 6639 enum dma_data_direction dir) 6640 { 6641 dma_unmap_single(&hdev->pdev->dev, addr, len, dir); 6642 } 6643 gaudi2_validate_cb_address(struct hl_device * hdev,struct hl_cs_parser * parser)6644 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) 6645 { 6646 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 6647 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6648 6649 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { 6650 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); 6651 return -EINVAL; 6652 } 6653 6654 /* Just check if CB address is valid */ 6655 6656 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6657 parser->user_cb_size, 6658 asic_prop->sram_user_base_address, 6659 asic_prop->sram_end_address)) 6660 return 0; 6661 6662 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6663 parser->user_cb_size, 6664 asic_prop->dram_user_base_address, 6665 asic_prop->dram_end_address)) 6666 return 0; 6667 6668 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && 6669 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6670 parser->user_cb_size, 6671 asic_prop->dmmu.start_addr, 6672 asic_prop->dmmu.end_addr)) 6673 return 0; 6674 6675 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { 6676 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6677 parser->user_cb_size, 6678 asic_prop->pmmu.start_addr, 6679 asic_prop->pmmu.end_addr) || 6680 hl_mem_area_inside_range( 6681 (u64) (uintptr_t) parser->user_cb, 6682 parser->user_cb_size, 6683 asic_prop->pmmu_huge.start_addr, 6684 asic_prop->pmmu_huge.end_addr)) 6685 return 0; 6686 6687 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { 6688 if (!hdev->pdev) 6689 return 0; 6690 6691 if (!device_iommu_mapped(&hdev->pdev->dev)) 6692 return 0; 6693 } 6694 6695 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", 6696 parser->user_cb, parser->user_cb_size); 6697 6698 return -EFAULT; 6699 } 6700 gaudi2_cs_parser(struct hl_device * hdev,struct hl_cs_parser * parser)6701 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 6702 { 6703 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6704 6705 if (!parser->is_kernel_allocated_cb) 6706 return gaudi2_validate_cb_address(hdev, parser); 6707 6708 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 6709 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); 6710 return -EINVAL; 6711 } 6712 6713 return 0; 6714 } 6715 gaudi2_send_heartbeat(struct hl_device * hdev)6716 static int gaudi2_send_heartbeat(struct hl_device *hdev) 6717 { 6718 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6719 6720 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 6721 return 0; 6722 6723 return hl_fw_send_heartbeat(hdev); 6724 } 6725 6726 /* This is an internal helper function, used to update the KDMA mmu props. 6727 * Should be called with a proper kdma lock. 6728 */ gaudi2_kdma_set_mmbp_asid(struct hl_device * hdev,bool mmu_bypass,u32 asid)6729 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, 6730 bool mmu_bypass, u32 asid) 6731 { 6732 u32 rw_asid, rw_mmu_bp; 6733 6734 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 6735 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 6736 6737 rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) | 6738 (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT); 6739 6740 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid); 6741 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp); 6742 } 6743 gaudi2_arm_cq_monitor(struct hl_device * hdev,u32 sob_id,u32 mon_id,u32 cq_id,u32 mon_payload,u32 sync_value)6744 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, 6745 u32 mon_payload, u32 sync_value) 6746 { 6747 u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm; 6748 u8 mask; 6749 6750 sob_offset = sob_id * 4; 6751 mon_offset = mon_id * 4; 6752 6753 /* Reset the SOB value */ 6754 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 6755 6756 /* Configure this address with CQ_ID 0 because CQ_EN is set */ 6757 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id); 6758 6759 /* Configure this address with CS index because CQ_EN is set */ 6760 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload); 6761 6762 sync_group_id = sob_id / 8; 6763 mask = ~(1 << (sob_id & 0x7)); 6764 mode = 1; /* comparison mode is "equal to" */ 6765 6766 mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value); 6767 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode); 6768 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask); 6769 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id); 6770 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm); 6771 } 6772 6773 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */ gaudi2_send_job_to_kdma(struct hl_device * hdev,u64 src_addr,u64 dst_addr,u32 size,bool is_memset)6774 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, 6775 u64 src_addr, u64 dst_addr, 6776 u32 size, bool is_memset) 6777 { 6778 u32 comp_val, commit_mask, *polling_addr, timeout, status = 0; 6779 struct hl_cq_entry *cq_base; 6780 struct hl_cq *cq; 6781 u64 comp_addr; 6782 int rc; 6783 6784 gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION, 6785 GAUDI2_RESERVED_MON_KDMA_COMPLETION, 6786 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1); 6787 6788 comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + 6789 (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32)); 6790 6791 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 6792 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 6793 6794 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr)); 6795 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr)); 6796 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr)); 6797 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr)); 6798 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr)); 6799 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr)); 6800 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val); 6801 WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size); 6802 6803 commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) | 6804 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1); 6805 6806 if (is_memset) 6807 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1); 6808 6809 WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask); 6810 6811 /* Wait for completion */ 6812 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; 6813 cq_base = cq->kernel_address; 6814 polling_addr = (u32 *)&cq_base[cq->ci]; 6815 6816 if (hdev->pldm) 6817 /* for each 1MB 20 second of timeout */ 6818 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20; 6819 else 6820 timeout = KDMA_TIMEOUT_USEC; 6821 6822 /* Polling */ 6823 rc = hl_poll_timeout_memory( 6824 hdev, 6825 polling_addr, 6826 status, 6827 (status == 1), 6828 1000, 6829 timeout, 6830 true); 6831 6832 *polling_addr = 0; 6833 6834 if (rc) { 6835 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); 6836 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT); 6837 return rc; 6838 } 6839 6840 cq->ci = hl_cq_inc_ptr(cq->ci); 6841 6842 return 0; 6843 } 6844 gaudi2_memset_device_lbw(struct hl_device * hdev,u32 addr,u32 size,u32 val)6845 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) 6846 { 6847 u32 i; 6848 6849 for (i = 0 ; i < size ; i += sizeof(u32)) 6850 WREG32(addr + i, val); 6851 } 6852 gaudi2_qman_set_test_mode(struct hl_device * hdev,u32 hw_queue_id,bool enable)6853 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) 6854 { 6855 u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6856 6857 if (enable) { 6858 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE); 6859 WREG32(reg_base + QM_PQC_CFG_OFFSET, 0); 6860 } else { 6861 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED); 6862 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 6863 } 6864 } 6865 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device * hdev,u32 hw_queue_id)6866 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) 6867 { 6868 return hdev->asic_prop.first_available_user_sob[0] + 6869 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; 6870 } 6871 gaudi2_test_queue_clear(struct hl_device * hdev,u32 hw_queue_id)6872 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) 6873 { 6874 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6875 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 6876 6877 /* Reset the SOB value */ 6878 WREG32(sob_addr, 0); 6879 } 6880 gaudi2_test_queue_send_msg_short(struct hl_device * hdev,u32 hw_queue_id,u32 sob_val,struct gaudi2_queues_test_info * msg_info)6881 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, 6882 struct gaudi2_queues_test_info *msg_info) 6883 { 6884 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6885 u32 tmp, sob_base = 1; 6886 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; 6887 size_t pkt_size = sizeof(struct packet_msg_short); 6888 int rc; 6889 6890 tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) | 6891 (1 << GAUDI2_PKT_CTL_EB_SHIFT) | 6892 (1 << GAUDI2_PKT_CTL_MB_SHIFT) | 6893 (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) | 6894 (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT); 6895 6896 msg_short_pkt->value = cpu_to_le32(sob_val); 6897 msg_short_pkt->ctl = cpu_to_le32(tmp); 6898 6899 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); 6900 if (rc) 6901 dev_err(hdev->dev, 6902 "Failed to send msg_short packet to H/W queue %d\n", hw_queue_id); 6903 6904 return rc; 6905 } 6906 gaudi2_test_queue_wait_completion(struct hl_device * hdev,u32 hw_queue_id,u32 sob_val)6907 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) 6908 { 6909 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6910 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 6911 u32 timeout_usec, tmp; 6912 int rc; 6913 6914 if (hdev->pldm) 6915 timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC; 6916 else 6917 timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC; 6918 6919 rc = hl_poll_timeout( 6920 hdev, 6921 sob_addr, 6922 tmp, 6923 (tmp == sob_val), 6924 1000, 6925 timeout_usec); 6926 6927 if (rc == -ETIMEDOUT) { 6928 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", 6929 hw_queue_id, tmp); 6930 rc = -EIO; 6931 } 6932 6933 return rc; 6934 } 6935 gaudi2_test_cpu_queue(struct hl_device * hdev)6936 static int gaudi2_test_cpu_queue(struct hl_device *hdev) 6937 { 6938 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6939 6940 /* 6941 * check capability here as send_cpu_message() won't update the result 6942 * value if no capability 6943 */ 6944 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 6945 return 0; 6946 6947 return hl_fw_test_cpu_queue(hdev); 6948 } 6949 gaudi2_test_queues(struct hl_device * hdev)6950 static int gaudi2_test_queues(struct hl_device *hdev) 6951 { 6952 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6953 struct gaudi2_queues_test_info *msg_info; 6954 u32 sob_val = 0x5a5a; 6955 int i, rc; 6956 6957 /* send test message on all enabled Qs */ 6958 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 6959 if (!gaudi2_is_queue_enabled(hdev, i)) 6960 continue; 6961 6962 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; 6963 gaudi2_qman_set_test_mode(hdev, i, true); 6964 gaudi2_test_queue_clear(hdev, i); 6965 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info); 6966 if (rc) 6967 goto done; 6968 } 6969 6970 rc = gaudi2_test_cpu_queue(hdev); 6971 if (rc) 6972 goto done; 6973 6974 /* verify that all messages were processed */ 6975 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 6976 if (!gaudi2_is_queue_enabled(hdev, i)) 6977 continue; 6978 6979 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val); 6980 if (rc) 6981 /* chip is not usable, no need for cleanups, just bail-out with error */ 6982 goto done; 6983 6984 gaudi2_test_queue_clear(hdev, i); 6985 gaudi2_qman_set_test_mode(hdev, i, false); 6986 } 6987 6988 done: 6989 return rc; 6990 } 6991 gaudi2_compute_reset_late_init(struct hl_device * hdev)6992 static int gaudi2_compute_reset_late_init(struct hl_device *hdev) 6993 { 6994 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6995 size_t irq_arr_size; 6996 int rc; 6997 6998 gaudi2_init_arcs(hdev); 6999 7000 rc = gaudi2_scrub_arcs_dccm(hdev); 7001 if (rc) { 7002 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 7003 return rc; 7004 } 7005 7006 gaudi2_init_security(hdev); 7007 7008 /* Unmask all IRQs since some could have been received during the soft reset */ 7009 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); 7010 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); 7011 } 7012 gaudi2_get_edma_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7013 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7014 struct engines_data *e) 7015 { 7016 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7017 struct asic_fixed_properties *prop = &hdev->asic_prop; 7018 unsigned long *mask = (unsigned long *) mask_arr; 7019 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; 7020 bool is_idle = true, is_eng_idle; 7021 int engine_idx, i, j; 7022 u64 offset; 7023 7024 if (e) 7025 hl_engine_data_sprintf(e, 7026 "\nCORE EDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7027 "---- ---- ------- ------------ ------------- -------------\n"); 7028 7029 for (i = 0; i < NUM_OF_DCORES; i++) { 7030 for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) { 7031 int seq = i * NUM_OF_EDMA_PER_DCORE + j; 7032 7033 if (!(prop->edma_enabled_mask & BIT(seq))) 7034 continue; 7035 7036 engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 + 7037 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7038 offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET; 7039 7040 dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset); 7041 dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset); 7042 7043 qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset); 7044 qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset); 7045 qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset); 7046 7047 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7048 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7049 is_idle &= is_eng_idle; 7050 7051 if (mask && !is_eng_idle) 7052 set_bit(engine_idx, mask); 7053 7054 if (e) 7055 hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N", 7056 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7057 } 7058 } 7059 7060 return is_idle; 7061 } 7062 gaudi2_get_pdma_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7063 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7064 struct engines_data *e) 7065 { 7066 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7067 unsigned long *mask = (unsigned long *) mask_arr; 7068 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; 7069 bool is_idle = true, is_eng_idle; 7070 int engine_idx, i; 7071 u64 offset; 7072 7073 if (e) 7074 hl_engine_data_sprintf(e, 7075 "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7076 "---- ------- ------------ ------------- -------------\n"); 7077 7078 for (i = 0 ; i < NUM_OF_PDMA ; i++) { 7079 engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i; 7080 offset = i * PDMA_OFFSET; 7081 dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset); 7082 dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset); 7083 7084 qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset); 7085 qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset); 7086 qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset); 7087 7088 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7089 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7090 is_idle &= is_eng_idle; 7091 7092 if (mask && !is_eng_idle) 7093 set_bit(engine_idx, mask); 7094 7095 if (e) 7096 hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N", 7097 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7098 } 7099 7100 return is_idle; 7101 } 7102 gaudi2_get_nic_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7103 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7104 struct engines_data *e) 7105 { 7106 unsigned long *mask = (unsigned long *) mask_arr; 7107 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; 7108 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7109 bool is_idle = true, is_eng_idle; 7110 int engine_idx, i; 7111 u64 offset = 0; 7112 7113 /* NIC, twelve macros in Full chip */ 7114 if (e && hdev->nic_ports_mask) 7115 hl_engine_data_sprintf(e, 7116 "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" 7117 "--- ------- ------------ ----------\n"); 7118 7119 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 7120 if (!(i & 1)) 7121 offset = i / 2 * NIC_OFFSET; 7122 else 7123 offset += NIC_QM_OFFSET; 7124 7125 if (!(hdev->nic_ports_mask & BIT(i))) 7126 continue; 7127 7128 engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i; 7129 7130 7131 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); 7132 qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset); 7133 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); 7134 7135 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7136 is_idle &= is_eng_idle; 7137 7138 if (mask && !is_eng_idle) 7139 set_bit(engine_idx, mask); 7140 7141 if (e) 7142 hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N", 7143 qm_glbl_sts0, qm_cgm_sts); 7144 } 7145 7146 return is_idle; 7147 } 7148 gaudi2_get_mme_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7149 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7150 struct engines_data *e) 7151 { 7152 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts; 7153 unsigned long *mask = (unsigned long *) mask_arr; 7154 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; 7155 bool is_idle = true, is_eng_idle; 7156 int engine_idx, i; 7157 u64 offset; 7158 7159 if (e) 7160 hl_engine_data_sprintf(e, 7161 "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" 7162 "--- ---- ------- ------------ ---------------\n"); 7163 /* MME, one per Dcore */ 7164 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7165 engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET; 7166 offset = i * DCORE_OFFSET; 7167 7168 qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset); 7169 qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset); 7170 qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset); 7171 7172 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7173 is_idle &= is_eng_idle; 7174 7175 mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset); 7176 is_eng_idle &= IS_MME_IDLE(mme_arch_sts); 7177 is_idle &= is_eng_idle; 7178 7179 if (e) 7180 hl_engine_data_sprintf(e, mme_fmt, i, "N", 7181 is_eng_idle ? "Y" : "N", 7182 qm_glbl_sts0, 7183 mme_arch_sts); 7184 7185 if (mask && !is_eng_idle) 7186 set_bit(engine_idx, mask); 7187 } 7188 7189 return is_idle; 7190 } 7191 gaudi2_is_tpc_engine_idle(struct hl_device * hdev,int dcore,int inst,u32 offset,struct iterate_module_ctx * ctx)7192 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, 7193 struct iterate_module_ctx *ctx) 7194 { 7195 struct gaudi2_tpc_idle_data *idle_data = ctx->data; 7196 u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7197 bool is_eng_idle; 7198 int engine_idx; 7199 7200 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) 7201 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6; 7202 else 7203 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 + 7204 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst; 7205 7206 tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset); 7207 qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset); 7208 qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset); 7209 qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset); 7210 7211 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7212 IS_TPC_IDLE(tpc_cfg_sts); 7213 *(idle_data->is_idle) &= is_eng_idle; 7214 7215 if (idle_data->mask && !is_eng_idle) 7216 set_bit(engine_idx, idle_data->mask); 7217 7218 if (idle_data->e) 7219 hl_engine_data_sprintf(idle_data->e, 7220 idle_data->tpc_fmt, dcore, inst, 7221 is_eng_idle ? "Y" : "N", 7222 qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); 7223 } 7224 gaudi2_get_tpc_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7225 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7226 struct engines_data *e) 7227 { 7228 struct asic_fixed_properties *prop = &hdev->asic_prop; 7229 unsigned long *mask = (unsigned long *) mask_arr; 7230 bool is_idle = true; 7231 7232 struct gaudi2_tpc_idle_data tpc_idle_data = { 7233 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", 7234 .e = e, 7235 .mask = mask, 7236 .is_idle = &is_idle, 7237 }; 7238 struct iterate_module_ctx tpc_iter = { 7239 .fn = &gaudi2_is_tpc_engine_idle, 7240 .data = &tpc_idle_data, 7241 }; 7242 7243 if (e && prop->tpc_enabled_mask) 7244 hl_engine_data_sprintf(e, 7245 "\nCORE TPC is_idle QM_GLBL_STS0 QM_CGM_STS STATUS\n" 7246 "---- --- ------- ------------ ---------- ------\n"); 7247 7248 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7249 7250 return *tpc_idle_data.is_idle; 7251 } 7252 gaudi2_get_decoder_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7253 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7254 struct engines_data *e) 7255 { 7256 struct asic_fixed_properties *prop = &hdev->asic_prop; 7257 unsigned long *mask = (unsigned long *) mask_arr; 7258 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; 7259 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; 7260 bool is_idle = true, is_eng_idle; 7261 u32 dec_swreg15, dec_enabled_bit; 7262 int engine_idx, i, j; 7263 u64 offset; 7264 7265 /* Decoders, two each Dcore and two shared PCIe decoders */ 7266 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) 7267 hl_engine_data_sprintf(e, 7268 "\nCORE DEC is_idle VSI_CMD_SWREG15\n" 7269 "---- --- ------- ---------------\n"); 7270 7271 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7272 for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) { 7273 dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j); 7274 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) 7275 continue; 7276 7277 engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 + 7278 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7279 offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET; 7280 7281 dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset); 7282 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7283 is_idle &= is_eng_idle; 7284 7285 if (mask && !is_eng_idle) 7286 set_bit(engine_idx, mask); 7287 7288 if (e) 7289 hl_engine_data_sprintf(e, dec_fmt, i, j, 7290 is_eng_idle ? "Y" : "N", dec_swreg15); 7291 } 7292 } 7293 7294 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) 7295 hl_engine_data_sprintf(e, 7296 "\nPCIe DEC is_idle VSI_CMD_SWREG15\n" 7297 "-------- ------- ---------------\n"); 7298 7299 /* Check shared(PCIe) decoders */ 7300 for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) { 7301 dec_enabled_bit = PCIE_DEC_SHIFT + i; 7302 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) 7303 continue; 7304 7305 engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i; 7306 offset = i * DCORE_DEC_OFFSET; 7307 dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset); 7308 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7309 is_idle &= is_eng_idle; 7310 7311 if (mask && !is_eng_idle) 7312 set_bit(engine_idx, mask); 7313 7314 if (e) 7315 hl_engine_data_sprintf(e, pcie_dec_fmt, i, 7316 is_eng_idle ? "Y" : "N", dec_swreg15); 7317 } 7318 7319 return is_idle; 7320 } 7321 gaudi2_get_rotator_idle_status(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7322 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7323 struct engines_data *e) 7324 { 7325 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; 7326 unsigned long *mask = (unsigned long *) mask_arr; 7327 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7328 bool is_idle = true, is_eng_idle; 7329 int engine_idx, i; 7330 u64 offset; 7331 7332 if (e) 7333 hl_engine_data_sprintf(e, 7334 "\nCORE ROT is_idle QM_GLBL_STS0 QM_GLBL_STS1 QM_CGM_STS\n" 7335 "---- --- ------- ------------ ------------ ----------\n"); 7336 7337 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7338 engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i; 7339 7340 offset = i * ROT_OFFSET; 7341 7342 qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset); 7343 qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset); 7344 qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset); 7345 7346 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7347 is_idle &= is_eng_idle; 7348 7349 if (mask && !is_eng_idle) 7350 set_bit(engine_idx, mask); 7351 7352 if (e) 7353 hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", 7354 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7355 } 7356 7357 return is_idle; 7358 } 7359 gaudi2_is_device_idle(struct hl_device * hdev,u64 * mask_arr,u8 mask_len,struct engines_data * e)7360 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7361 struct engines_data *e) 7362 { 7363 bool is_idle = true; 7364 7365 is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e); 7366 is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e); 7367 is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e); 7368 is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e); 7369 is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e); 7370 is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e); 7371 is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e); 7372 7373 return is_idle; 7374 } 7375 gaudi2_hw_queues_lock(struct hl_device * hdev)7376 static void gaudi2_hw_queues_lock(struct hl_device *hdev) 7377 __acquires(&gaudi2->hw_queues_lock) 7378 { 7379 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7380 7381 spin_lock(&gaudi2->hw_queues_lock); 7382 } 7383 gaudi2_hw_queues_unlock(struct hl_device * hdev)7384 static void gaudi2_hw_queues_unlock(struct hl_device *hdev) 7385 __releases(&gaudi2->hw_queues_lock) 7386 { 7387 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7388 7389 spin_unlock(&gaudi2->hw_queues_lock); 7390 } 7391 gaudi2_get_pci_id(struct hl_device * hdev)7392 static u32 gaudi2_get_pci_id(struct hl_device *hdev) 7393 { 7394 return hdev->pdev->device; 7395 } 7396 gaudi2_get_eeprom_data(struct hl_device * hdev,void * data,size_t max_size)7397 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) 7398 { 7399 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7400 7401 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7402 return 0; 7403 7404 return hl_fw_get_eeprom_data(hdev, data, max_size); 7405 } 7406 gaudi2_update_eq_ci(struct hl_device * hdev,u32 val)7407 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) 7408 { 7409 WREG32(mmCPU_IF_EQ_RD_OFFS, val); 7410 } 7411 gaudi2_get_events_stat(struct hl_device * hdev,bool aggregate,u32 * size)7412 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 7413 { 7414 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7415 7416 if (aggregate) { 7417 *size = (u32) sizeof(gaudi2->events_stat_aggregate); 7418 return gaudi2->events_stat_aggregate; 7419 } 7420 7421 *size = (u32) sizeof(gaudi2->events_stat); 7422 return gaudi2->events_stat; 7423 } 7424 gaudi2_mmu_vdec_dcore_prepare(struct hl_device * hdev,int dcore_id,int dcore_vdec_id,u32 rw_asid,u32 rw_mmu_bp)7425 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, 7426 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7427 { 7428 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * 7429 dcore_vdec_id + DCORE_OFFSET * dcore_id; 7430 7431 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7432 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7433 7434 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7435 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7436 7437 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7438 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7439 7440 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7441 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7442 7443 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7444 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7445 } 7446 gaudi2_mmu_dcore_prepare(struct hl_device * hdev,int dcore_id,u32 asid)7447 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) 7448 { 7449 u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7450 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7451 struct asic_fixed_properties *prop = &hdev->asic_prop; 7452 u32 dcore_offset = dcore_id * DCORE_OFFSET; 7453 u32 vdec_id, i, ports_offset, reg_val; 7454 u8 edma_seq_base; 7455 7456 /* EDMA */ 7457 edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE; 7458 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { 7459 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7460 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7461 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7462 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7463 } 7464 7465 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { 7466 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7467 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7468 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7469 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7470 } 7471 7472 /* Sync Mngr */ 7473 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid); 7474 /* 7475 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID 7476 * for any access type 7477 */ 7478 if (dcore_id > 0) { 7479 reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) | 7480 (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT); 7481 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val); 7482 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0); 7483 } 7484 7485 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0); 7486 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid); 7487 7488 for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) { 7489 ports_offset = i * DCORE_MME_SBTE_OFFSET; 7490 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP + 7491 dcore_offset + ports_offset, 0); 7492 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID + 7493 dcore_offset + ports_offset, rw_asid); 7494 } 7495 7496 for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) { 7497 ports_offset = i * DCORE_MME_WB_OFFSET; 7498 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP + 7499 dcore_offset + ports_offset, 0); 7500 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID + 7501 dcore_offset + ports_offset, rw_asid); 7502 } 7503 7504 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7505 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7506 7507 /* 7508 * Decoders 7509 */ 7510 for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) { 7511 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) 7512 gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0); 7513 } 7514 } 7515 gudi2_mmu_vdec_shared_prepare(struct hl_device * hdev,int shared_vdec_id,u32 rw_asid,u32 rw_mmu_bp)7516 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, 7517 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7518 { 7519 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; 7520 7521 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7522 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7523 7524 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7525 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7526 7527 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7528 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7529 7530 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7531 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7532 7533 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7534 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7535 } 7536 gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device * hdev,int arc_farm_id,u32 rw_asid,u32 rw_mmu_bp)7537 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, 7538 u32 rw_asid, u32 rw_mmu_bp) 7539 { 7540 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; 7541 7542 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp); 7543 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid); 7544 } 7545 gaudi2_arc_mmu_prepare(struct hl_device * hdev,u32 cpu_id,u32 asid)7546 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) 7547 { 7548 u32 reg_base, reg_offset, reg_val = 0; 7549 7550 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 7551 7552 /* Enable MMU and configure asid for all relevant ARC regions */ 7553 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0); 7554 reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid); 7555 7556 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL); 7557 WREG32(reg_base + reg_offset, reg_val); 7558 7559 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW); 7560 WREG32(reg_base + reg_offset, reg_val); 7561 7562 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA); 7563 WREG32(reg_base + reg_offset, reg_val); 7564 7565 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA); 7566 WREG32(reg_base + reg_offset, reg_val); 7567 7568 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA); 7569 WREG32(reg_base + reg_offset, reg_val); 7570 7571 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE); 7572 WREG32(reg_base + reg_offset, reg_val); 7573 7574 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL); 7575 WREG32(reg_base + reg_offset, reg_val); 7576 7577 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL); 7578 WREG32(reg_base + reg_offset, reg_val); 7579 7580 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL); 7581 WREG32(reg_base + reg_offset, reg_val); 7582 7583 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL); 7584 WREG32(reg_base + reg_offset, reg_val); 7585 7586 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL); 7587 WREG32(reg_base + reg_offset, reg_val); 7588 } 7589 gaudi2_arc_mmu_prepare_all(struct hl_device * hdev,u32 asid)7590 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) 7591 { 7592 int i; 7593 7594 if (hdev->fw_components & FW_TYPE_BOOT_CPU) 7595 return hl_fw_cpucp_engine_core_asid_set(hdev, asid); 7596 7597 for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7598 gaudi2_arc_mmu_prepare(hdev, i, asid); 7599 7600 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 7601 if (!gaudi2_is_queue_enabled(hdev, i)) 7602 continue; 7603 7604 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid); 7605 } 7606 7607 return 0; 7608 } 7609 gaudi2_mmu_shared_prepare(struct hl_device * hdev,u32 asid)7610 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) 7611 { 7612 struct asic_fixed_properties *prop = &hdev->asic_prop; 7613 u32 rw_asid, offset; 7614 int rc, i; 7615 7616 rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) | 7617 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid); 7618 7619 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7620 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7621 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7622 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7623 7624 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7625 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7626 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7627 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7628 7629 /* ROT */ 7630 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7631 offset = i * ROT_OFFSET; 7632 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid); 7633 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7634 RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK); 7635 RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK); 7636 RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK); 7637 } 7638 7639 /* Shared Decoders are the last bits in the decoders mask */ 7640 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) 7641 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0); 7642 7643 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) 7644 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0); 7645 7646 /* arc farm arc dup eng */ 7647 for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7648 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0); 7649 7650 rc = gaudi2_arc_mmu_prepare_all(hdev, asid); 7651 if (rc) 7652 return rc; 7653 7654 return 0; 7655 } 7656 gaudi2_tpc_mmu_prepare(struct hl_device * hdev,int dcore,int inst,u32 offset,struct iterate_module_ctx * ctx)7657 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, 7658 struct iterate_module_ctx *ctx) 7659 { 7660 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; 7661 7662 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0); 7663 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); 7664 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7665 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); 7666 } 7667 7668 /* zero the MMUBP and set the ASID */ gaudi2_mmu_prepare(struct hl_device * hdev,u32 asid)7669 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) 7670 { 7671 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7672 struct gaudi2_tpc_mmu_data tpc_mmu_data; 7673 struct iterate_module_ctx tpc_iter = { 7674 .fn = &gaudi2_tpc_mmu_prepare, 7675 .data = &tpc_mmu_data, 7676 }; 7677 int rc, i; 7678 7679 if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) { 7680 dev_crit(hdev->dev, "asid %u is too big\n", asid); 7681 return -EINVAL; 7682 } 7683 7684 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 7685 return 0; 7686 7687 rc = gaudi2_mmu_shared_prepare(hdev, asid); 7688 if (rc) 7689 return rc; 7690 7691 /* configure DCORE MMUs */ 7692 tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7693 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7694 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7695 for (i = 0 ; i < NUM_OF_DCORES ; i++) 7696 gaudi2_mmu_dcore_prepare(hdev, i, asid); 7697 7698 return 0; 7699 } 7700 is_info_event(u32 event)7701 static inline bool is_info_event(u32 event) 7702 { 7703 switch (event) { 7704 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 7705 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 7706 7707 /* return in case of NIC status event - these events are received periodically and not as 7708 * an indication to an error. 7709 */ 7710 case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1: 7711 return true; 7712 default: 7713 return false; 7714 } 7715 } 7716 gaudi2_print_event(struct hl_device * hdev,u16 event_type,bool ratelimited,const char * fmt,...)7717 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, 7718 bool ratelimited, const char *fmt, ...) 7719 { 7720 struct va_format vaf; 7721 va_list args; 7722 7723 va_start(args, fmt); 7724 vaf.fmt = fmt; 7725 vaf.va = &args; 7726 7727 if (ratelimited) 7728 dev_err_ratelimited(hdev->dev, "%s: %pV\n", 7729 gaudi2_irq_map_table[event_type].valid ? 7730 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7731 else 7732 dev_err(hdev->dev, "%s: %pV\n", 7733 gaudi2_irq_map_table[event_type].valid ? 7734 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7735 7736 va_end(args); 7737 } 7738 gaudi2_handle_ecc_event(struct hl_device * hdev,u16 event_type,struct hl_eq_ecc_data * ecc_data)7739 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, 7740 struct hl_eq_ecc_data *ecc_data) 7741 { 7742 u64 ecc_address = 0, ecc_syndrom = 0; 7743 u8 memory_wrapper_idx = 0; 7744 7745 ecc_address = le64_to_cpu(ecc_data->ecc_address); 7746 ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom); 7747 memory_wrapper_idx = ecc_data->memory_wrapper_idx; 7748 7749 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7750 "ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.", 7751 ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical); 7752 7753 return !!ecc_data->is_critical; 7754 } 7755 print_lower_qman_data_on_err(struct hl_device * hdev,u64 qman_base)7756 static void print_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base) 7757 { 7758 u32 lo, hi, cq_ptr_size, arc_cq_ptr_size; 7759 u64 cq_ptr, arc_cq_ptr, cp_current_inst; 7760 7761 lo = RREG32(qman_base + QM_CQ_PTR_LO_4_OFFSET); 7762 hi = RREG32(qman_base + QM_CQ_PTR_HI_4_OFFSET); 7763 cq_ptr = ((u64) hi) << 32 | lo; 7764 cq_ptr_size = RREG32(qman_base + QM_CQ_TSIZE_4_OFFSET); 7765 7766 lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_OFFSET); 7767 hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_OFFSET); 7768 arc_cq_ptr = ((u64) hi) << 32 | lo; 7769 arc_cq_ptr_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_OFFSET); 7770 7771 lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET); 7772 hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET); 7773 cp_current_inst = ((u64) hi) << 32 | lo; 7774 7775 dev_info(hdev->dev, 7776 "LowerQM. CQ: {ptr %#llx, size %u}, ARC_CQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n", 7777 cq_ptr, cq_ptr_size, arc_cq_ptr, arc_cq_ptr_size, cp_current_inst); 7778 } 7779 gaudi2_handle_qman_err_generic(struct hl_device * hdev,u16 event_type,u64 qman_base,u32 qid_base)7780 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, 7781 u64 qman_base, u32 qid_base) 7782 { 7783 u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0; 7784 u64 glbl_sts_addr, arb_err_addr; 7785 char reg_desc[32]; 7786 7787 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); 7788 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); 7789 7790 /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */ 7791 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { 7792 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i); 7793 7794 if (!glbl_sts_val) 7795 continue; 7796 7797 if (i == QMAN_STREAMS) { 7798 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM"); 7799 num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE; 7800 } else { 7801 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i); 7802 num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE; 7803 } 7804 7805 for (j = 0 ; j < num_error_causes ; j++) 7806 if (glbl_sts_val & BIT(j)) { 7807 gaudi2_print_event(hdev, event_type, true, 7808 "%s. err cause: %s", reg_desc, 7809 i == QMAN_STREAMS ? 7810 gaudi2_lower_qman_error_cause[j] : 7811 gaudi2_qman_error_cause[j]); 7812 error_count++; 7813 } 7814 7815 if (i == QMAN_STREAMS) 7816 print_lower_qman_data_on_err(hdev, qman_base); 7817 } 7818 7819 arb_err_val = RREG32(arb_err_addr); 7820 7821 if (!arb_err_val) 7822 goto out; 7823 7824 for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) { 7825 if (arb_err_val & BIT(j)) { 7826 gaudi2_print_event(hdev, event_type, true, 7827 "ARB_ERR. err cause: %s", 7828 gaudi2_qman_arb_error_cause[j]); 7829 error_count++; 7830 } 7831 } 7832 7833 out: 7834 return error_count; 7835 } 7836 gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device * hdev,u64 rtr_mstr_if_base_addr,bool is_write,char * name,enum gaudi2_engine_id id,u64 * event_mask)7837 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, 7838 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 7839 enum gaudi2_engine_id id, u64 *event_mask) 7840 { 7841 u32 razwi_hi, razwi_lo, razwi_xy; 7842 u16 eng_id = id; 7843 u8 rd_wr_flag; 7844 7845 if (is_write) { 7846 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI); 7847 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO); 7848 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY); 7849 rd_wr_flag = HL_RAZWI_WRITE; 7850 } else { 7851 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI); 7852 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO); 7853 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY); 7854 rd_wr_flag = HL_RAZWI_READ; 7855 } 7856 7857 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1, 7858 rd_wr_flag | HL_RAZWI_HBW, event_mask); 7859 7860 dev_err_ratelimited(hdev->dev, 7861 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", 7862 name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy); 7863 } 7864 gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device * hdev,u64 rtr_mstr_if_base_addr,bool is_write,char * name,enum gaudi2_engine_id id,u64 * event_mask)7865 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, 7866 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 7867 enum gaudi2_engine_id id, u64 *event_mask) 7868 { 7869 u64 razwi_addr = CFG_BASE; 7870 u32 razwi_xy; 7871 u16 eng_id = id; 7872 u8 rd_wr_flag; 7873 7874 if (is_write) { 7875 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI); 7876 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY); 7877 rd_wr_flag = HL_RAZWI_WRITE; 7878 } else { 7879 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI); 7880 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY); 7881 rd_wr_flag = HL_RAZWI_READ; 7882 } 7883 7884 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask); 7885 dev_err_ratelimited(hdev->dev, 7886 "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n", 7887 name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr, 7888 razwi_xy); 7889 } 7890 gaudi2_razwi_calc_engine_id(struct hl_device * hdev,enum razwi_event_sources module,u8 module_idx)7891 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, 7892 enum razwi_event_sources module, u8 module_idx) 7893 { 7894 switch (module) { 7895 case RAZWI_TPC: 7896 if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES)) 7897 return GAUDI2_DCORE0_ENGINE_ID_TPC_6; 7898 return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 7899 (module_idx % NUM_OF_TPC_PER_DCORE) + 7900 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 7901 7902 case RAZWI_MME: 7903 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + 7904 (module_idx * ENGINE_ID_DCORE_OFFSET)); 7905 7906 case RAZWI_EDMA: 7907 return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 7908 (module_idx % NUM_OF_EDMA_PER_DCORE)); 7909 7910 case RAZWI_PDMA: 7911 return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx); 7912 7913 case RAZWI_NIC: 7914 return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx)); 7915 7916 case RAZWI_DEC: 7917 if (module_idx == 8) 7918 return GAUDI2_PCIE_ENGINE_ID_DEC_0; 7919 7920 if (module_idx == 9) 7921 return GAUDI2_PCIE_ENGINE_ID_DEC_1; 7922 ; 7923 return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 7924 (module_idx % NUM_OF_DEC_PER_DCORE) + 7925 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 7926 7927 case RAZWI_ROT: 7928 return GAUDI2_ENGINE_ID_ROT_0 + module_idx; 7929 7930 default: 7931 return GAUDI2_ENGINE_ID_SIZE; 7932 } 7933 } 7934 7935 /* 7936 * This function handles RR(Range register) hit events. 7937 * raised be initiators not PSOC RAZWI. 7938 */ gaudi2_ack_module_razwi_event_handler(struct hl_device * hdev,enum razwi_event_sources module,u8 module_idx,u8 module_sub_idx,u64 * event_mask)7939 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, 7940 enum razwi_event_sources module, u8 module_idx, 7941 u8 module_sub_idx, u64 *event_mask) 7942 { 7943 bool via_sft = false; 7944 u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx; 7945 u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr; 7946 u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0; 7947 u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0; 7948 char initiator_name[64]; 7949 7950 switch (module) { 7951 case RAZWI_TPC: 7952 sprintf(initiator_name, "TPC_%u", module_idx); 7953 if (hdev->tpc_binning) { 7954 binned_idx = __ffs(hdev->tpc_binning); 7955 if (binned_idx == module_idx) 7956 module_idx = TPC_ID_DCORE0_TPC6; 7957 } 7958 7959 hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx]; 7960 7961 if (hl_is_fw_sw_ver_below(hdev, 1, 9) && 7962 !hdev->asic_prop.fw_security_enabled && 7963 ((module_idx == 0) || (module_idx == 1))) 7964 lbw_rtr_id = DCORE0_RTR0; 7965 else 7966 lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx]; 7967 break; 7968 case RAZWI_MME: 7969 sprintf(initiator_name, "MME_%u", module_idx); 7970 switch (module_sub_idx) { 7971 case MME_WAP0: 7972 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0; 7973 break; 7974 case MME_WAP1: 7975 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1; 7976 break; 7977 case MME_WRITE: 7978 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write; 7979 break; 7980 case MME_READ: 7981 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read; 7982 break; 7983 case MME_SBTE0: 7984 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0; 7985 break; 7986 case MME_SBTE1: 7987 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1; 7988 break; 7989 case MME_SBTE2: 7990 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2; 7991 break; 7992 case MME_SBTE3: 7993 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3; 7994 break; 7995 case MME_SBTE4: 7996 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4; 7997 break; 7998 default: 7999 return; 8000 } 8001 lbw_rtr_id = hbw_rtr_id; 8002 break; 8003 case RAZWI_EDMA: 8004 hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx]; 8005 dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE; 8006 /* SFT has separate MSTR_IF for LBW, only there we can 8007 * read the LBW razwi related registers 8008 */ 8009 lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE + 8010 dcore_id * SFT_DCORE_OFFSET; 8011 via_sft = true; 8012 sprintf(initiator_name, "EDMA_%u", module_idx); 8013 break; 8014 case RAZWI_PDMA: 8015 hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx]; 8016 lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx]; 8017 sprintf(initiator_name, "PDMA_%u", module_idx); 8018 break; 8019 case RAZWI_NIC: 8020 hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx]; 8021 lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx]; 8022 sprintf(initiator_name, "NIC_%u", module_idx); 8023 break; 8024 case RAZWI_DEC: 8025 sprintf(initiator_name, "DEC_%u", module_idx); 8026 if (hdev->decoder_binning) { 8027 binned_idx = __ffs(hdev->decoder_binning); 8028 if (binned_idx == module_idx) 8029 module_idx = DEC_ID_PCIE_VDEC1; 8030 } 8031 hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx]; 8032 lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx]; 8033 break; 8034 case RAZWI_ROT: 8035 hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx]; 8036 lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx]; 8037 sprintf(initiator_name, "ROT_%u", module_idx); 8038 break; 8039 default: 8040 return; 8041 } 8042 8043 /* Find router mstr_if register base */ 8044 if (!via_sft) { 8045 dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE; 8046 dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE; 8047 hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE + 8048 dcore_id * DCORE_OFFSET + 8049 dcore_rtr_id * DCORE_RTR_OFFSET + 8050 RTR_MSTR_IF_OFFSET; 8051 lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr + 8052 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); 8053 } 8054 8055 /* Find out event cause by reading "RAZWI_HAPPENED" registers */ 8056 hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED); 8057 hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED); 8058 lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED); 8059 lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED); 8060 8061 eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx); 8062 if (hbw_shrd_aw) { 8063 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true, 8064 initiator_name, eng_id, event_mask); 8065 8066 /* Clear event indication */ 8067 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw); 8068 } 8069 8070 if (hbw_shrd_ar) { 8071 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false, 8072 initiator_name, eng_id, event_mask); 8073 8074 /* Clear event indication */ 8075 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar); 8076 } 8077 8078 if (lbw_shrd_aw) { 8079 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true, 8080 initiator_name, eng_id, event_mask); 8081 8082 /* Clear event indication */ 8083 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw); 8084 } 8085 8086 if (lbw_shrd_ar) { 8087 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false, 8088 initiator_name, eng_id, event_mask); 8089 8090 /* Clear event indication */ 8091 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar); 8092 } 8093 } 8094 gaudi2_check_if_razwi_happened(struct hl_device * hdev)8095 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) 8096 { 8097 struct asic_fixed_properties *prop = &hdev->asic_prop; 8098 u8 mod_idx, sub_mod; 8099 8100 /* check all TPCs */ 8101 for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) { 8102 if (prop->tpc_enabled_mask & BIT(mod_idx)) 8103 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL); 8104 } 8105 8106 /* check all MMEs */ 8107 for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8108 for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++) 8109 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx, 8110 sub_mod, NULL); 8111 8112 /* check all EDMAs */ 8113 for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8114 if (prop->edma_enabled_mask & BIT(mod_idx)) 8115 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL); 8116 8117 /* check all PDMAs */ 8118 for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++) 8119 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL); 8120 8121 /* check all NICs */ 8122 for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++) 8123 if (hdev->nic_ports_mask & BIT(mod_idx)) 8124 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0, 8125 NULL); 8126 8127 /* check all DECs */ 8128 for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++) 8129 if (prop->decoder_enabled_mask & BIT(mod_idx)) 8130 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL); 8131 8132 /* check all ROTs */ 8133 for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++) 8134 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL); 8135 } 8136 gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info * razwi_info,u32 array_size,u32 axuser_xy,u32 * base,u16 * eng_id,char * eng_name)8137 static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size, 8138 u32 axuser_xy, u32 *base, u16 *eng_id, 8139 char *eng_name) 8140 { 8141 8142 int i, num_of_eng = 0; 8143 u16 str_size = 0; 8144 8145 for (i = 0 ; i < array_size ; i++) { 8146 if (axuser_xy != razwi_info[i].axuser_xy) 8147 continue; 8148 8149 eng_id[num_of_eng] = razwi_info[i].eng_id; 8150 base[num_of_eng] = razwi_info[i].rtr_ctrl; 8151 if (!num_of_eng) 8152 str_size += scnprintf(eng_name + str_size, 8153 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", 8154 razwi_info[i].eng_name); 8155 else 8156 str_size += scnprintf(eng_name + str_size, 8157 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", 8158 razwi_info[i].eng_name); 8159 num_of_eng++; 8160 } 8161 8162 return num_of_eng; 8163 } 8164 gaudi2_handle_psoc_razwi_happened(struct hl_device * hdev,u32 razwi_reg,u64 * event_mask)8165 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, 8166 u64 *event_mask) 8167 { 8168 u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0; 8169 u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8170 u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8171 char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE]; 8172 bool razwi_happened = false; 8173 u64 addr; 8174 int i; 8175 8176 num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info), 8177 axuser_xy, base, eng_id, eng_name_str); 8178 8179 /* If no match for XY coordinates, try to find it in MME razwi table */ 8180 if (!num_of_eng) { 8181 axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg); 8182 num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info, 8183 ARRAY_SIZE(mme_razwi_info), 8184 axuser_xy, base, eng_id, 8185 eng_name_str); 8186 } 8187 8188 for (i = 0 ; i < num_of_eng ; i++) { 8189 if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) { 8190 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI); 8191 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO); 8192 addr = ((u64)addr_hi << 32) + addr_lo; 8193 if (addr) { 8194 dev_err(hdev->dev, 8195 "PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8196 eng_name_str, addr); 8197 hl_handle_razwi(hdev, addr, &eng_id[0], 8198 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask); 8199 razwi_happened = true; 8200 } 8201 } 8202 8203 if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) { 8204 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI); 8205 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO); 8206 addr = ((u64)addr_hi << 32) + addr_lo; 8207 if (addr) { 8208 dev_err(hdev->dev, 8209 "PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8210 eng_name_str, addr); 8211 hl_handle_razwi(hdev, addr, &eng_id[0], 8212 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask); 8213 razwi_happened = true; 8214 } 8215 } 8216 8217 if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) { 8218 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR); 8219 if (addr_lo) { 8220 dev_err(hdev->dev, 8221 "PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8222 eng_name_str, addr_lo); 8223 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8224 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask); 8225 razwi_happened = true; 8226 } 8227 } 8228 8229 if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) { 8230 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR); 8231 if (addr_lo) { 8232 dev_err(hdev->dev, 8233 "PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8234 eng_name_str, addr_lo); 8235 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8236 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask); 8237 razwi_happened = true; 8238 } 8239 } 8240 /* In common case the loop will break, when there is only one engine id, or 8241 * several engines with the same router. The exceptional case is with psoc razwi 8242 * from EDMA, where it's possible to get axuser id which fits 2 routers (2 8243 * interfaces of sft router). In this case, maybe the first router won't hold info 8244 * and we will need to iterate on the other router. 8245 */ 8246 if (razwi_happened) 8247 break; 8248 } 8249 8250 return razwi_happened; 8251 } 8252 8253 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */ gaudi2_ack_psoc_razwi_event_handler(struct hl_device * hdev,u64 * event_mask)8254 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) 8255 { 8256 u32 razwi_mask_info, razwi_intr = 0, error_count = 0; 8257 8258 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { 8259 razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT); 8260 if (!razwi_intr) 8261 return 0; 8262 } 8263 8264 razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO); 8265 8266 dev_err_ratelimited(hdev->dev, 8267 "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", 8268 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info), 8269 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info), 8270 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info), 8271 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info), 8272 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info)); 8273 8274 if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask)) 8275 error_count++; 8276 else 8277 dev_err_ratelimited(hdev->dev, 8278 "PSOC RAZWI interrupt: invalid razwi info (0x%x)\n", 8279 razwi_mask_info); 8280 8281 /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */ 8282 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) 8283 WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr); 8284 8285 return error_count; 8286 } 8287 _gaudi2_handle_qm_sei_err(struct hl_device * hdev,u64 qman_base,u16 event_type)8288 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) 8289 { 8290 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8291 8292 sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET); 8293 8294 for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) { 8295 if (sts_val & BIT(i)) { 8296 gaudi2_print_event(hdev, event_type, true, 8297 "err cause: %s", gaudi2_qm_sei_error_cause[i]); 8298 sts_clr_val |= BIT(i); 8299 error_count++; 8300 } 8301 } 8302 8303 WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val); 8304 8305 return error_count; 8306 } 8307 gaudi2_handle_qm_sei_err(struct hl_device * hdev,u16 event_type,bool extended_err_check,u64 * event_mask)8308 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, 8309 bool extended_err_check, u64 *event_mask) 8310 { 8311 enum razwi_event_sources module; 8312 u32 error_count = 0; 8313 u64 qman_base; 8314 u8 index; 8315 8316 switch (event_type) { 8317 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP: 8318 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 8319 qman_base = mmDCORE0_TPC0_QM_BASE + 8320 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET + 8321 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET; 8322 module = RAZWI_TPC; 8323 break; 8324 case GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 8325 qman_base = mmDCORE0_TPC6_QM_BASE; 8326 module = RAZWI_TPC; 8327 break; 8328 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 8329 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 8330 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 8331 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 8332 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 8333 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 8334 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 8335 qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET; 8336 module = RAZWI_MME; 8337 break; 8338 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 8339 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 8340 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; 8341 qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET; 8342 module = RAZWI_PDMA; 8343 break; 8344 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 8345 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 8346 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 8347 qman_base = mmROT0_QM_BASE + index * ROT_OFFSET; 8348 module = RAZWI_ROT; 8349 break; 8350 default: 8351 return 0; 8352 } 8353 8354 error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8355 8356 /* There is a single event per NIC macro, so should check its both QMAN blocks */ 8357 if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE && 8358 event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE) 8359 error_count += _gaudi2_handle_qm_sei_err(hdev, 8360 qman_base + NIC_QM_OFFSET, event_type); 8361 8362 if (extended_err_check) { 8363 /* check if RAZWI happened */ 8364 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask); 8365 hl_check_for_glbl_errors(hdev); 8366 } 8367 8368 return error_count; 8369 } 8370 gaudi2_handle_qman_err(struct hl_device * hdev,u16 event_type,u64 * event_mask)8371 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8372 { 8373 u32 qid_base, error_count = 0; 8374 u64 qman_base; 8375 u8 index = 0; 8376 8377 switch (event_type) { 8378 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM: 8379 index = event_type - GAUDI2_EVENT_TPC0_QM; 8380 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS; 8381 qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8382 break; 8383 case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM: 8384 index = event_type - GAUDI2_EVENT_TPC6_QM; 8385 qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS; 8386 qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8387 break; 8388 case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM: 8389 index = event_type - GAUDI2_EVENT_TPC12_QM; 8390 qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS; 8391 qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8392 break; 8393 case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM: 8394 index = event_type - GAUDI2_EVENT_TPC18_QM; 8395 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS; 8396 qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8397 break; 8398 case GAUDI2_EVENT_TPC24_QM: 8399 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 8400 qman_base = mmDCORE0_TPC6_QM_BASE; 8401 break; 8402 case GAUDI2_EVENT_MME0_QM: 8403 qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 8404 qman_base = mmDCORE0_MME_QM_BASE; 8405 break; 8406 case GAUDI2_EVENT_MME1_QM: 8407 qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 8408 qman_base = mmDCORE1_MME_QM_BASE; 8409 break; 8410 case GAUDI2_EVENT_MME2_QM: 8411 qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 8412 qman_base = mmDCORE2_MME_QM_BASE; 8413 break; 8414 case GAUDI2_EVENT_MME3_QM: 8415 qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 8416 qman_base = mmDCORE3_MME_QM_BASE; 8417 break; 8418 case GAUDI2_EVENT_HDMA0_QM: 8419 index = 0; 8420 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0; 8421 qman_base = mmDCORE0_EDMA0_QM_BASE; 8422 break; 8423 case GAUDI2_EVENT_HDMA1_QM: 8424 index = 1; 8425 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0; 8426 qman_base = mmDCORE0_EDMA1_QM_BASE; 8427 break; 8428 case GAUDI2_EVENT_HDMA2_QM: 8429 index = 2; 8430 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0; 8431 qman_base = mmDCORE1_EDMA0_QM_BASE; 8432 break; 8433 case GAUDI2_EVENT_HDMA3_QM: 8434 index = 3; 8435 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0; 8436 qman_base = mmDCORE1_EDMA1_QM_BASE; 8437 break; 8438 case GAUDI2_EVENT_HDMA4_QM: 8439 index = 4; 8440 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0; 8441 qman_base = mmDCORE2_EDMA0_QM_BASE; 8442 break; 8443 case GAUDI2_EVENT_HDMA5_QM: 8444 index = 5; 8445 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0; 8446 qman_base = mmDCORE2_EDMA1_QM_BASE; 8447 break; 8448 case GAUDI2_EVENT_HDMA6_QM: 8449 index = 6; 8450 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0; 8451 qman_base = mmDCORE3_EDMA0_QM_BASE; 8452 break; 8453 case GAUDI2_EVENT_HDMA7_QM: 8454 index = 7; 8455 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0; 8456 qman_base = mmDCORE3_EDMA1_QM_BASE; 8457 break; 8458 case GAUDI2_EVENT_PDMA0_QM: 8459 qid_base = GAUDI2_QUEUE_ID_PDMA_0_0; 8460 qman_base = mmPDMA0_QM_BASE; 8461 break; 8462 case GAUDI2_EVENT_PDMA1_QM: 8463 qid_base = GAUDI2_QUEUE_ID_PDMA_1_0; 8464 qman_base = mmPDMA1_QM_BASE; 8465 break; 8466 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 8467 qid_base = GAUDI2_QUEUE_ID_ROT_0_0; 8468 qman_base = mmROT0_QM_BASE; 8469 break; 8470 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 8471 qid_base = GAUDI2_QUEUE_ID_ROT_1_0; 8472 qman_base = mmROT1_QM_BASE; 8473 break; 8474 default: 8475 return 0; 8476 } 8477 8478 error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, qid_base); 8479 8480 /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */ 8481 if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) { 8482 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8483 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask); 8484 } 8485 8486 hl_check_for_glbl_errors(hdev); 8487 8488 return error_count; 8489 } 8490 gaudi2_handle_arc_farm_sei_err(struct hl_device * hdev,u16 event_type)8491 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type) 8492 { 8493 u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm; 8494 8495 for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) { 8496 sts_clr_val = 0; 8497 sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS + 8498 (arc_farm * ARC_FARM_OFFSET)); 8499 8500 for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) { 8501 if (sts_val & BIT(i)) { 8502 gaudi2_print_event(hdev, event_type, true, 8503 "ARC FARM ARC %u err cause: %s", 8504 arc_farm, gaudi2_arc_sei_error_cause[i]); 8505 sts_clr_val |= BIT(i); 8506 error_count++; 8507 } 8508 } 8509 WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET), 8510 sts_clr_val); 8511 } 8512 8513 hl_check_for_glbl_errors(hdev); 8514 8515 return error_count; 8516 } 8517 gaudi2_handle_cpu_sei_err(struct hl_device * hdev,u16 event_type)8518 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) 8519 { 8520 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8521 8522 sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS); 8523 8524 for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) { 8525 if (sts_val & BIT(i)) { 8526 gaudi2_print_event(hdev, event_type, true, 8527 "err cause: %s", gaudi2_cpu_sei_error_cause[i]); 8528 sts_clr_val |= BIT(i); 8529 error_count++; 8530 } 8531 } 8532 8533 hl_check_for_glbl_errors(hdev); 8534 8535 WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val); 8536 8537 return error_count; 8538 } 8539 gaudi2_handle_rot_err(struct hl_device * hdev,u8 rot_index,u16 event_type,struct hl_eq_razwi_with_intr_cause * razwi_with_intr_cause,u64 * event_mask)8540 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, 8541 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8542 u64 *event_mask) 8543 { 8544 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8545 u32 error_count = 0; 8546 int i; 8547 8548 for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++) 8549 if (intr_cause_data & BIT(i)) { 8550 gaudi2_print_event(hdev, event_type, true, 8551 "err cause: %s", guadi2_rot_error_cause[i]); 8552 error_count++; 8553 } 8554 8555 /* check if RAZWI happened */ 8556 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask); 8557 hl_check_for_glbl_errors(hdev); 8558 8559 return error_count; 8560 } 8561 gaudi2_tpc_ack_interrupts(struct hl_device * hdev,u8 tpc_index,u16 event_type,struct hl_eq_razwi_with_intr_cause * razwi_with_intr_cause,u64 * event_mask)8562 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, 8563 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8564 u64 *event_mask) 8565 { 8566 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8567 u32 error_count = 0; 8568 int i; 8569 8570 for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++) 8571 if (intr_cause_data & BIT(i)) { 8572 gaudi2_print_event(hdev, event_type, true, 8573 "interrupt cause: %s", gaudi2_tpc_interrupts_cause[i]); 8574 error_count++; 8575 } 8576 8577 /* check if RAZWI happened */ 8578 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask); 8579 hl_check_for_glbl_errors(hdev); 8580 8581 return error_count; 8582 } 8583 gaudi2_handle_dec_err(struct hl_device * hdev,u8 dec_index,u16 event_type,u64 * event_mask)8584 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, 8585 u64 *event_mask) 8586 { 8587 u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0; 8588 int i; 8589 8590 if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES) 8591 /* DCORE DEC */ 8592 sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR + 8593 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) + 8594 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE); 8595 else 8596 /* PCIE DEC */ 8597 sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET * 8598 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); 8599 8600 sts_val = RREG32(sts_addr); 8601 8602 for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) { 8603 if (sts_val & BIT(i)) { 8604 gaudi2_print_event(hdev, event_type, true, 8605 "err cause: %s", gaudi2_dec_error_cause[i]); 8606 sts_clr_val |= BIT(i); 8607 error_count++; 8608 } 8609 } 8610 8611 /* check if RAZWI happened */ 8612 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask); 8613 hl_check_for_glbl_errors(hdev); 8614 8615 /* Write 1 clear errors */ 8616 WREG32(sts_addr, sts_clr_val); 8617 8618 return error_count; 8619 } 8620 gaudi2_handle_mme_err(struct hl_device * hdev,u8 mme_index,u16 event_type,u64 * event_mask)8621 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8622 u64 *event_mask) 8623 { 8624 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8625 int i; 8626 8627 sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index; 8628 sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index; 8629 8630 sts_val = RREG32(sts_addr); 8631 8632 for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) { 8633 if (sts_val & BIT(i)) { 8634 gaudi2_print_event(hdev, event_type, true, 8635 "err cause: %s", guadi2_mme_error_cause[i]); 8636 sts_clr_val |= BIT(i); 8637 error_count++; 8638 } 8639 } 8640 8641 /* check if RAZWI happened */ 8642 for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++) 8643 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask); 8644 8645 hl_check_for_glbl_errors(hdev); 8646 8647 WREG32(sts_clr_addr, sts_clr_val); 8648 8649 return error_count; 8650 } 8651 gaudi2_handle_mme_sbte_err(struct hl_device * hdev,u16 event_type,u64 intr_cause_data)8652 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type, 8653 u64 intr_cause_data) 8654 { 8655 int i, error_count = 0; 8656 8657 for (i = 0 ; i < GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE ; i++) 8658 if (intr_cause_data & BIT(i)) { 8659 gaudi2_print_event(hdev, event_type, true, 8660 "err cause: %s", guadi2_mme_sbte_error_cause[i]); 8661 error_count++; 8662 } 8663 8664 hl_check_for_glbl_errors(hdev); 8665 8666 return error_count; 8667 } 8668 gaudi2_handle_mme_wap_err(struct hl_device * hdev,u8 mme_index,u16 event_type,u64 * event_mask)8669 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8670 u64 *event_mask) 8671 { 8672 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8673 int i; 8674 8675 sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index; 8676 sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index; 8677 8678 sts_val = RREG32(sts_addr); 8679 8680 for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) { 8681 if (sts_val & BIT(i)) { 8682 gaudi2_print_event(hdev, event_type, true, 8683 "err cause: %s", guadi2_mme_wap_error_cause[i]); 8684 sts_clr_val |= BIT(i); 8685 error_count++; 8686 } 8687 } 8688 8689 /* check if RAZWI happened on WAP0/1 */ 8690 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask); 8691 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask); 8692 hl_check_for_glbl_errors(hdev); 8693 8694 WREG32(sts_clr_addr, sts_clr_val); 8695 8696 return error_count; 8697 } 8698 gaudi2_handle_kdma_core_event(struct hl_device * hdev,u16 event_type,u64 intr_cause_data)8699 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, 8700 u64 intr_cause_data) 8701 { 8702 u32 error_count = 0; 8703 int i; 8704 8705 /* If an AXI read or write error is received, an error is reported and 8706 * interrupt message is sent. Due to an HW errata, when reading the cause 8707 * register of the KDMA engine, the reported error is always HBW even if 8708 * the actual error caused by a LBW KDMA transaction. 8709 */ 8710 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8711 if (intr_cause_data & BIT(i)) { 8712 gaudi2_print_event(hdev, event_type, true, 8713 "err cause: %s", gaudi2_kdma_core_interrupts_cause[i]); 8714 error_count++; 8715 } 8716 8717 hl_check_for_glbl_errors(hdev); 8718 8719 return error_count; 8720 } 8721 gaudi2_handle_dma_core_event(struct hl_device * hdev,u16 event_type,u64 intr_cause)8722 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) 8723 { 8724 u32 error_count = 0; 8725 int i; 8726 8727 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8728 if (intr_cause & BIT(i)) { 8729 gaudi2_print_event(hdev, event_type, true, 8730 "err cause: %s", gaudi2_dma_core_interrupts_cause[i]); 8731 error_count++; 8732 } 8733 8734 hl_check_for_glbl_errors(hdev); 8735 8736 return error_count; 8737 } 8738 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device * hdev,u64 * event_mask)8739 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) 8740 { 8741 u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr; 8742 8743 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED; 8744 if (RREG32(razwi_happened_addr)) { 8745 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8746 GAUDI2_ENGINE_ID_PCIE, event_mask); 8747 WREG32(razwi_happened_addr, 0x1); 8748 } 8749 8750 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED; 8751 if (RREG32(razwi_happened_addr)) { 8752 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8753 GAUDI2_ENGINE_ID_PCIE, event_mask); 8754 WREG32(razwi_happened_addr, 0x1); 8755 } 8756 8757 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED; 8758 if (RREG32(razwi_happened_addr)) { 8759 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8760 GAUDI2_ENGINE_ID_PCIE, event_mask); 8761 WREG32(razwi_happened_addr, 0x1); 8762 } 8763 8764 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED; 8765 if (RREG32(razwi_happened_addr)) { 8766 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8767 GAUDI2_ENGINE_ID_PCIE, event_mask); 8768 WREG32(razwi_happened_addr, 0x1); 8769 } 8770 } 8771 gaudi2_print_pcie_addr_dec_info(struct hl_device * hdev,u16 event_type,u64 intr_cause_data,u64 * event_mask)8772 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, 8773 u64 intr_cause_data, u64 *event_mask) 8774 { 8775 u32 error_count = 0; 8776 int i; 8777 8778 gaudi2_print_event(hdev, event_type, true, 8779 "intr_cause_data: %#llx", intr_cause_data); 8780 8781 for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) { 8782 if (!(intr_cause_data & BIT_ULL(i))) 8783 continue; 8784 8785 gaudi2_print_event(hdev, event_type, true, 8786 "err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]); 8787 error_count++; 8788 8789 /* 8790 * Always check for LBW and HBW additional info as the indication itself is 8791 * sometimes missing 8792 */ 8793 } 8794 8795 hl_check_for_glbl_errors(hdev); 8796 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask); 8797 8798 return error_count; 8799 } 8800 gaudi2_handle_pif_fatal(struct hl_device * hdev,u16 event_type,u64 intr_cause_data)8801 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, 8802 u64 intr_cause_data) 8803 8804 { 8805 u32 error_count = 0; 8806 int i; 8807 8808 for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) { 8809 if (intr_cause_data & BIT_ULL(i)) { 8810 gaudi2_print_event(hdev, event_type, true, 8811 "err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]); 8812 error_count++; 8813 } 8814 } 8815 8816 return error_count; 8817 } 8818 gaudi2_handle_hif_fatal(struct hl_device * hdev,u16 event_type,u64 intr_cause_data)8819 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) 8820 { 8821 u32 error_count = 0; 8822 int i; 8823 8824 for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) { 8825 if (intr_cause_data & BIT_ULL(i)) { 8826 gaudi2_print_event(hdev, event_type, true, 8827 "err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]); 8828 error_count++; 8829 } 8830 } 8831 8832 return error_count; 8833 } 8834 gaudi2_handle_page_error(struct hl_device * hdev,u64 mmu_base,bool is_pmmu,u64 * event_mask)8835 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, 8836 u64 *event_mask) 8837 { 8838 u32 valid, val; 8839 u64 addr; 8840 8841 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 8842 8843 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK)) 8844 return; 8845 8846 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE)); 8847 addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK; 8848 addr <<= 32; 8849 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA)); 8850 8851 if (is_pmmu) { 8852 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); 8853 } else { 8854 8855 addr = gaudi2_mmu_descramble_addr(hdev, addr); 8856 addr &= HW_UNSCRAMBLED_BITS_MASK; 8857 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", 8858 addr, addr + ~HW_UNSCRAMBLED_BITS_MASK); 8859 } 8860 8861 hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask); 8862 8863 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 8864 } 8865 gaudi2_handle_access_error(struct hl_device * hdev,u64 mmu_base,bool is_pmmu)8866 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) 8867 { 8868 u32 valid, val; 8869 u64 addr; 8870 8871 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 8872 8873 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK)) 8874 return; 8875 8876 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE)); 8877 addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK; 8878 addr <<= 32; 8879 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA)); 8880 8881 if (!is_pmmu) 8882 addr = gaudi2_mmu_descramble_addr(hdev, addr); 8883 8884 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", 8885 is_pmmu ? "PMMU" : "HMMU", addr); 8886 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 8887 } 8888 gaudi2_handle_mmu_spi_sei_generic(struct hl_device * hdev,u16 event_type,u64 mmu_base,bool is_pmmu,u64 * event_mask)8889 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, 8890 u64 mmu_base, bool is_pmmu, u64 *event_mask) 8891 { 8892 u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0; 8893 int i; 8894 8895 spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET); 8896 8897 for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) { 8898 if (spi_sei_cause & BIT(i)) { 8899 gaudi2_print_event(hdev, event_type, true, 8900 "err cause: %s", gaudi2_mmu_spi_sei[i].cause); 8901 8902 if (i == 0) 8903 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask); 8904 else if (i == 1) 8905 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 8906 8907 if (gaudi2_mmu_spi_sei[i].clear_bit >= 0) 8908 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit); 8909 8910 error_count++; 8911 } 8912 } 8913 8914 /* Clear cause */ 8915 WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause); 8916 8917 /* Clear interrupt */ 8918 WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr); 8919 8920 return error_count; 8921 } 8922 gaudi2_handle_sm_err(struct hl_device * hdev,u16 event_type,u8 sm_index)8923 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) 8924 { 8925 u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log, 8926 cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0; 8927 int i; 8928 8929 sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index; 8930 cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index; 8931 8932 sei_cause_val = RREG32(sei_cause_addr); 8933 sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val); 8934 cq_intr_val = RREG32(cq_intr_addr); 8935 8936 /* SEI interrupt */ 8937 if (sei_cause_cause) { 8938 /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */ 8939 sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK, 8940 sei_cause_val); 8941 8942 for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) { 8943 if (!(sei_cause_cause & BIT(i))) 8944 continue; 8945 8946 gaudi2_print_event(hdev, event_type, true, 8947 "err cause: %s. %s: 0x%X", 8948 gaudi2_sm_sei_cause[i].cause_name, 8949 gaudi2_sm_sei_cause[i].log_name, 8950 sei_cause_log); 8951 error_count++; 8952 break; 8953 } 8954 8955 /* Clear SM_SEI_CAUSE */ 8956 WREG32(sei_cause_addr, 0); 8957 } 8958 8959 /* CQ interrupt */ 8960 if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) { 8961 cq_intr_queue_index = 8962 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK, 8963 cq_intr_val); 8964 8965 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", 8966 sm_index, cq_intr_queue_index); 8967 error_count++; 8968 8969 /* Clear CQ_INTR */ 8970 WREG32(cq_intr_addr, 0); 8971 } 8972 8973 hl_check_for_glbl_errors(hdev); 8974 8975 return error_count; 8976 } 8977 get_hmmu_base(u16 event_type)8978 static u64 get_hmmu_base(u16 event_type) 8979 { 8980 u8 dcore, index_in_dcore; 8981 8982 switch (event_type) { 8983 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP: 8984 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR: 8985 dcore = 0; 8986 index_in_dcore = 0; 8987 break; 8988 case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP: 8989 case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR: 8990 dcore = 1; 8991 index_in_dcore = 0; 8992 break; 8993 case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP: 8994 case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR: 8995 dcore = 0; 8996 index_in_dcore = 1; 8997 break; 8998 case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP: 8999 case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR: 9000 dcore = 1; 9001 index_in_dcore = 1; 9002 break; 9003 case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP: 9004 case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR: 9005 dcore = 3; 9006 index_in_dcore = 2; 9007 break; 9008 case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP: 9009 case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR: 9010 dcore = 2; 9011 index_in_dcore = 2; 9012 break; 9013 case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP: 9014 case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR: 9015 dcore = 3; 9016 index_in_dcore = 3; 9017 break; 9018 case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP: 9019 case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR: 9020 dcore = 2; 9021 index_in_dcore = 3; 9022 break; 9023 case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP: 9024 case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR: 9025 dcore = 0; 9026 index_in_dcore = 2; 9027 break; 9028 case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP: 9029 case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR: 9030 dcore = 1; 9031 index_in_dcore = 2; 9032 break; 9033 case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP: 9034 case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR: 9035 dcore = 0; 9036 index_in_dcore = 3; 9037 break; 9038 case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP: 9039 case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR: 9040 dcore = 1; 9041 index_in_dcore = 3; 9042 break; 9043 case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9044 case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9045 dcore = 3; 9046 index_in_dcore = 0; 9047 break; 9048 case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP: 9049 case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR: 9050 dcore = 2; 9051 index_in_dcore = 0; 9052 break; 9053 case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP: 9054 case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR: 9055 dcore = 3; 9056 index_in_dcore = 1; 9057 break; 9058 case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP: 9059 case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR: 9060 dcore = 2; 9061 index_in_dcore = 1; 9062 break; 9063 default: 9064 return ULONG_MAX; 9065 } 9066 9067 return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET; 9068 } 9069 gaudi2_handle_mmu_spi_sei_err(struct hl_device * hdev,u16 event_type,u64 * event_mask)9070 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9071 { 9072 bool is_pmmu = false; 9073 u32 error_count = 0; 9074 u64 mmu_base; 9075 9076 switch (event_type) { 9077 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9078 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9079 mmu_base = get_hmmu_base(event_type); 9080 break; 9081 9082 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9083 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9084 is_pmmu = true; 9085 mmu_base = mmPMMU_HBW_MMU_BASE; 9086 break; 9087 default: 9088 return 0; 9089 } 9090 9091 if (mmu_base == ULONG_MAX) 9092 return 0; 9093 9094 error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base, 9095 is_pmmu, event_mask); 9096 hl_check_for_glbl_errors(hdev); 9097 9098 return error_count; 9099 } 9100 9101 9102 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */ gaudi2_hbm_sei_handle_read_err(struct hl_device * hdev,struct hl_eq_hbm_sei_read_err_intr_info * rd_err_data,u32 err_cnt)9103 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, 9104 struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt) 9105 { 9106 u32 addr, beat, beat_shift; 9107 bool rc = false; 9108 9109 dev_err_ratelimited(hdev->dev, 9110 "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n", 9111 FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt), 9112 FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt), 9113 FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt)); 9114 9115 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); 9116 dev_err_ratelimited(hdev->dev, 9117 "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n", 9118 FIELD_GET(HBM_RD_ADDR_SID_MASK, addr), 9119 FIELD_GET(HBM_RD_ADDR_BG_MASK, addr), 9120 FIELD_GET(HBM_RD_ADDR_BA_MASK, addr), 9121 FIELD_GET(HBM_RD_ADDR_COL_MASK, addr), 9122 FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr)); 9123 9124 /* For each beat (RDQS edge), look for possible errors and print relevant info */ 9125 for (beat = 0 ; beat < 4 ; beat++) { 9126 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9127 (HBM_RD_ERR_SERR_BEAT0_MASK << beat)) 9128 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", 9129 beat, 9130 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9131 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9132 9133 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9134 (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) { 9135 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", 9136 beat, 9137 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9138 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9139 rc |= true; 9140 } 9141 9142 beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT; 9143 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9144 (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) { 9145 dev_err_ratelimited(hdev->dev, 9146 "Beat%d read PARITY: DM: %#x, PAR data: %#x\n", 9147 beat, 9148 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9149 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9150 (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >> 9151 (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift)); 9152 rc |= true; 9153 } 9154 9155 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); 9156 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9157 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); 9158 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9159 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); 9160 } 9161 9162 return rc; 9163 } 9164 gaudi2_hbm_sei_print_wr_par_info(struct hl_device * hdev,struct hl_eq_hbm_sei_wr_par_intr_info * wr_par_err_data,u32 err_cnt)9165 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, 9166 struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt) 9167 { 9168 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; 9169 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; 9170 9171 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); 9172 9173 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", 9174 derr & 0x3, derr & 0xc); 9175 9176 /* JIRA H6-3286 - the following prints may not be valid */ 9177 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); 9178 for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) { 9179 curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr); 9180 dev_err_ratelimited(hdev->dev, 9181 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n", 9182 i, 9183 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr), 9184 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr), 9185 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr), 9186 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr)); 9187 } 9188 } 9189 gaudi2_hbm_sei_print_ca_par_info(struct hl_device * hdev,struct hl_eq_hbm_sei_ca_par_intr_info * ca_par_err_data,u32 err_cnt)9190 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, 9191 struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt) 9192 { 9193 __le32 *col_cmd = ca_par_err_data->dbg_col; 9194 __le16 *row_cmd = ca_par_err_data->dbg_row; 9195 u32 i; 9196 9197 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); 9198 9199 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); 9200 for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++) 9201 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, 9202 le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0), 9203 le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0)); 9204 } 9205 9206 /* Returns true if hard reset is needed or false otherwise */ gaudi2_handle_hbm_mc_sei_err(struct hl_device * hdev,u16 event_type,struct hl_eq_hbm_sei_data * sei_data)9207 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, 9208 struct hl_eq_hbm_sei_data *sei_data) 9209 { 9210 bool require_hard_reset = false; 9211 u32 hbm_id, mc_id, cause_idx; 9212 9213 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; 9214 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; 9215 9216 cause_idx = sei_data->hdr.sei_cause; 9217 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { 9218 gaudi2_print_event(hdev, event_type, true, 9219 "err cause: %s", 9220 "Invalid HBM SEI event cause (%d) provided by FW", cause_idx); 9221 return true; 9222 } 9223 9224 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, 9225 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", 9226 sei_data->hdr.is_critical ? "Critical" : "Non-critical", 9227 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, 9228 hbm_mc_sei_cause[cause_idx]); 9229 9230 /* Print error-specific info */ 9231 switch (cause_idx) { 9232 case HBM_SEI_CATTRIP: 9233 require_hard_reset = true; 9234 break; 9235 9236 case HBM_SEI_CMD_PARITY_EVEN: 9237 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, 9238 le32_to_cpu(sei_data->hdr.cnt)); 9239 require_hard_reset = true; 9240 break; 9241 9242 case HBM_SEI_CMD_PARITY_ODD: 9243 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, 9244 le32_to_cpu(sei_data->hdr.cnt)); 9245 require_hard_reset = true; 9246 break; 9247 9248 case HBM_SEI_WRITE_DATA_PARITY_ERR: 9249 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, 9250 le32_to_cpu(sei_data->hdr.cnt)); 9251 require_hard_reset = true; 9252 break; 9253 9254 case HBM_SEI_READ_ERR: 9255 /* Unlike other SEI events, read error requires further processing of the 9256 * raw data in order to determine the root cause. 9257 */ 9258 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev, 9259 &sei_data->read_err_info, 9260 le32_to_cpu(sei_data->hdr.cnt)); 9261 break; 9262 9263 default: 9264 break; 9265 } 9266 9267 require_hard_reset |= !!sei_data->hdr.is_critical; 9268 9269 return require_hard_reset; 9270 } 9271 gaudi2_handle_hbm_cattrip(struct hl_device * hdev,u16 event_type,u64 intr_cause_data)9272 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, 9273 u64 intr_cause_data) 9274 { 9275 if (intr_cause_data) { 9276 gaudi2_print_event(hdev, event_type, true, 9277 "temperature error cause: %#llx", intr_cause_data); 9278 return 1; 9279 } 9280 9281 return 0; 9282 } 9283 gaudi2_handle_hbm_mc_spi(struct hl_device * hdev,u64 intr_cause_data)9284 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) 9285 { 9286 u32 i, error_count = 0; 9287 9288 for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++) 9289 if (intr_cause_data & hbm_mc_spi[i].mask) { 9290 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", 9291 hbm_mc_spi[i].cause); 9292 error_count++; 9293 } 9294 9295 return error_count; 9296 } 9297 gaudi2_print_clk_change_info(struct hl_device * hdev,u16 event_type,u64 * event_mask)9298 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9299 { 9300 ktime_t zero_time = ktime_set(0, 0); 9301 9302 mutex_lock(&hdev->clk_throttling.lock); 9303 9304 switch (event_type) { 9305 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 9306 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 9307 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 9308 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 9309 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 9310 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); 9311 break; 9312 9313 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 9314 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 9315 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 9316 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); 9317 break; 9318 9319 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 9320 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 9321 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 9322 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 9323 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 9324 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9325 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); 9326 break; 9327 9328 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 9329 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 9330 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 9331 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9332 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); 9333 break; 9334 9335 default: 9336 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); 9337 break; 9338 } 9339 9340 mutex_unlock(&hdev->clk_throttling.lock); 9341 } 9342 gaudi2_print_out_of_sync_info(struct hl_device * hdev,u16 event_type,struct cpucp_pkt_sync_err * sync_err)9343 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, 9344 struct cpucp_pkt_sync_err *sync_err) 9345 { 9346 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9347 9348 gaudi2_print_event(hdev, event_type, false, 9349 "FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9350 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), 9351 q->pi, atomic_read(&q->ci)); 9352 } 9353 gaudi2_handle_pcie_p2p_msix(struct hl_device * hdev,u16 event_type)9354 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) 9355 { 9356 u32 p2p_intr, msix_gw_intr, error_count = 0; 9357 9358 p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR); 9359 msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR); 9360 9361 if (p2p_intr) { 9362 gaudi2_print_event(hdev, event_type, true, 9363 "pcie p2p transaction terminated due to security, req_id(0x%x)", 9364 RREG32(mmPCIE_WRAP_P2P_REQ_ID)); 9365 9366 WREG32(mmPCIE_WRAP_P2P_INTR, 0x1); 9367 error_count++; 9368 } 9369 9370 if (msix_gw_intr) { 9371 gaudi2_print_event(hdev, event_type, true, 9372 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", 9373 RREG32(mmPCIE_WRAP_MSIX_GW_VEC)); 9374 9375 WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1); 9376 error_count++; 9377 } 9378 9379 return error_count; 9380 } 9381 gaudi2_handle_pcie_drain(struct hl_device * hdev,struct hl_eq_pcie_drain_ind_data * drain_data)9382 static int gaudi2_handle_pcie_drain(struct hl_device *hdev, 9383 struct hl_eq_pcie_drain_ind_data *drain_data) 9384 { 9385 u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause, error_count = 0; 9386 9387 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); 9388 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); 9389 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); 9390 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); 9391 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); 9392 9393 if (cause & BIT_ULL(0)) { 9394 dev_err_ratelimited(hdev->dev, 9395 "PCIE AXI drain LBW completed, read_err %u, write_err %u\n", 9396 !!lbw_rd, !!lbw_wr); 9397 error_count++; 9398 } 9399 9400 if (cause & BIT_ULL(1)) { 9401 dev_err_ratelimited(hdev->dev, 9402 "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n", 9403 hbw_rd, hbw_wr); 9404 error_count++; 9405 } 9406 9407 return error_count; 9408 } 9409 gaudi2_handle_psoc_drain(struct hl_device * hdev,u64 intr_cause_data)9410 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) 9411 { 9412 u32 error_count = 0; 9413 int i; 9414 9415 for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) { 9416 if (intr_cause_data & BIT_ULL(i)) { 9417 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", 9418 gaudi2_psoc_axi_drain_interrupts_cause[i]); 9419 error_count++; 9420 } 9421 } 9422 9423 hl_check_for_glbl_errors(hdev); 9424 9425 return error_count; 9426 } 9427 gaudi2_print_cpu_pkt_failure_info(struct hl_device * hdev,u16 event_type,struct cpucp_pkt_sync_err * sync_err)9428 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, 9429 struct cpucp_pkt_sync_err *sync_err) 9430 { 9431 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9432 9433 gaudi2_print_event(hdev, event_type, false, 9434 "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9435 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 9436 } 9437 hl_arc_event_handle(struct hl_device * hdev,u16 event_type,struct hl_eq_engine_arc_intr_data * data)9438 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, 9439 struct hl_eq_engine_arc_intr_data *data) 9440 { 9441 struct hl_engine_arc_dccm_queue_full_irq *q; 9442 u32 intr_type, engine_id; 9443 u64 payload; 9444 9445 intr_type = le32_to_cpu(data->intr_type); 9446 engine_id = le32_to_cpu(data->engine_id); 9447 payload = le64_to_cpu(data->payload); 9448 9449 switch (intr_type) { 9450 case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ: 9451 q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload; 9452 9453 gaudi2_print_event(hdev, event_type, true, 9454 "ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u", 9455 engine_id, intr_type, q->queue_index); 9456 return 1; 9457 default: 9458 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type"); 9459 return 0; 9460 } 9461 } 9462 gaudi2_handle_eqe(struct hl_device * hdev,struct hl_eq_entry * eq_entry)9463 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 9464 { 9465 struct gaudi2_device *gaudi2 = hdev->asic_specific; 9466 bool reset_required = false, is_critical = false; 9467 u32 index, ctl, reset_flags = 0, error_count = 0; 9468 u64 event_mask = 0; 9469 u16 event_type; 9470 9471 ctl = le32_to_cpu(eq_entry->hdr.ctl); 9472 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT); 9473 9474 if (event_type >= GAUDI2_EVENT_SIZE) { 9475 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 9476 event_type, GAUDI2_EVENT_SIZE - 1); 9477 return; 9478 } 9479 9480 gaudi2->events_stat[event_type]++; 9481 gaudi2->events_stat_aggregate[event_type]++; 9482 9483 switch (event_type) { 9484 case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR: 9485 fallthrough; 9486 case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR: 9487 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9488 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9489 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 9490 is_critical = eq_entry->ecc_data.is_critical; 9491 error_count++; 9492 break; 9493 9494 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM: 9495 fallthrough; 9496 case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9497 fallthrough; 9498 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9499 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask); 9500 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9501 break; 9502 9503 case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0: 9504 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type); 9505 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9506 break; 9507 9508 case GAUDI2_EVENT_CPU_AXI_ERR_RSP: 9509 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type); 9510 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9511 event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR; 9512 break; 9513 9514 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9515 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9516 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask); 9517 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9518 break; 9519 9520 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9521 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9522 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 9523 error_count = gaudi2_handle_rot_err(hdev, index, event_type, 9524 &eq_entry->razwi_with_intr_cause, &event_mask); 9525 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9526 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9527 break; 9528 9529 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9530 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9531 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9532 &eq_entry->razwi_with_intr_cause, &event_mask); 9533 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9534 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9535 break; 9536 9537 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9538 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9539 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9540 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9541 break; 9542 9543 case GAUDI2_EVENT_TPC0_KERNEL_ERR: 9544 case GAUDI2_EVENT_TPC1_KERNEL_ERR: 9545 case GAUDI2_EVENT_TPC2_KERNEL_ERR: 9546 case GAUDI2_EVENT_TPC3_KERNEL_ERR: 9547 case GAUDI2_EVENT_TPC4_KERNEL_ERR: 9548 case GAUDI2_EVENT_TPC5_KERNEL_ERR: 9549 case GAUDI2_EVENT_TPC6_KERNEL_ERR: 9550 case GAUDI2_EVENT_TPC7_KERNEL_ERR: 9551 case GAUDI2_EVENT_TPC8_KERNEL_ERR: 9552 case GAUDI2_EVENT_TPC9_KERNEL_ERR: 9553 case GAUDI2_EVENT_TPC10_KERNEL_ERR: 9554 case GAUDI2_EVENT_TPC11_KERNEL_ERR: 9555 case GAUDI2_EVENT_TPC12_KERNEL_ERR: 9556 case GAUDI2_EVENT_TPC13_KERNEL_ERR: 9557 case GAUDI2_EVENT_TPC14_KERNEL_ERR: 9558 case GAUDI2_EVENT_TPC15_KERNEL_ERR: 9559 case GAUDI2_EVENT_TPC16_KERNEL_ERR: 9560 case GAUDI2_EVENT_TPC17_KERNEL_ERR: 9561 case GAUDI2_EVENT_TPC18_KERNEL_ERR: 9562 case GAUDI2_EVENT_TPC19_KERNEL_ERR: 9563 case GAUDI2_EVENT_TPC20_KERNEL_ERR: 9564 case GAUDI2_EVENT_TPC21_KERNEL_ERR: 9565 case GAUDI2_EVENT_TPC22_KERNEL_ERR: 9566 case GAUDI2_EVENT_TPC23_KERNEL_ERR: 9567 case GAUDI2_EVENT_TPC24_KERNEL_ERR: 9568 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / 9569 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); 9570 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9571 &eq_entry->razwi_with_intr_cause, &event_mask); 9572 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9573 break; 9574 9575 case GAUDI2_EVENT_DEC0_SPI: 9576 case GAUDI2_EVENT_DEC1_SPI: 9577 case GAUDI2_EVENT_DEC2_SPI: 9578 case GAUDI2_EVENT_DEC3_SPI: 9579 case GAUDI2_EVENT_DEC4_SPI: 9580 case GAUDI2_EVENT_DEC5_SPI: 9581 case GAUDI2_EVENT_DEC6_SPI: 9582 case GAUDI2_EVENT_DEC7_SPI: 9583 case GAUDI2_EVENT_DEC8_SPI: 9584 case GAUDI2_EVENT_DEC9_SPI: 9585 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / 9586 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); 9587 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9588 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9589 break; 9590 9591 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9592 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9593 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9594 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9595 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 9596 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 9597 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 9598 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9599 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9600 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9601 break; 9602 9603 case GAUDI2_EVENT_MME0_QMAN_SW_ERROR: 9604 case GAUDI2_EVENT_MME1_QMAN_SW_ERROR: 9605 case GAUDI2_EVENT_MME2_QMAN_SW_ERROR: 9606 case GAUDI2_EVENT_MME3_QMAN_SW_ERROR: 9607 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / 9608 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - 9609 GAUDI2_EVENT_MME0_QMAN_SW_ERROR); 9610 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9611 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9612 break; 9613 9614 case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9615 case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9616 case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9617 case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9618 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / 9619 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - 9620 GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID); 9621 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask); 9622 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9623 break; 9624 9625 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 9626 case GAUDI2_EVENT_KDMA0_CORE: 9627 error_count = gaudi2_handle_kdma_core_event(hdev, event_type, 9628 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9629 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9630 break; 9631 9632 case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE: 9633 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 9634 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9635 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9636 break; 9637 9638 case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE: 9639 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 9640 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9641 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9642 break; 9643 9644 case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR: 9645 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type, 9646 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); 9647 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9648 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9649 break; 9650 9651 case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9652 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9653 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9654 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9655 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask); 9656 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9657 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9658 break; 9659 9660 case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL: 9661 error_count = gaudi2_handle_hif_fatal(hdev, event_type, 9662 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9663 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9664 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9665 break; 9666 9667 case GAUDI2_EVENT_PMMU_FATAL_0: 9668 error_count = gaudi2_handle_pif_fatal(hdev, event_type, 9669 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9670 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9671 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9672 break; 9673 9674 case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT: 9675 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask); 9676 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9677 break; 9678 9679 case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE: 9680 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9681 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { 9682 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9683 reset_required = true; 9684 } 9685 error_count++; 9686 break; 9687 9688 case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5: 9689 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type, 9690 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9691 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9692 break; 9693 9694 case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI: 9695 error_count = gaudi2_handle_hbm_mc_spi(hdev, 9696 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9697 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9698 break; 9699 9700 case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE: 9701 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); 9702 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9703 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9704 break; 9705 9706 case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN: 9707 error_count = gaudi2_handle_psoc_drain(hdev, 9708 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9709 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9710 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9711 break; 9712 9713 case GAUDI2_EVENT_CPU_AXI_ECC: 9714 error_count = GAUDI2_NA_EVENT_CAUSE; 9715 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9716 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9717 break; 9718 case GAUDI2_EVENT_CPU_L2_RAM_ECC: 9719 error_count = GAUDI2_NA_EVENT_CAUSE; 9720 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9721 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9722 break; 9723 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP: 9724 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP: 9725 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP: 9726 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP: 9727 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type, 9728 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9729 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9730 break; 9731 case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B: 9732 error_count = GAUDI2_NA_EVENT_CAUSE; 9733 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9734 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9735 break; 9736 case GAUDI2_EVENT_PSOC_AXI_ERR_RSP: 9737 error_count = GAUDI2_NA_EVENT_CAUSE; 9738 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9739 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9740 break; 9741 case GAUDI2_EVENT_PSOC_PRSTN_FALL: 9742 error_count = GAUDI2_NA_EVENT_CAUSE; 9743 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9744 break; 9745 case GAUDI2_EVENT_PCIE_APB_TIMEOUT: 9746 error_count = GAUDI2_NA_EVENT_CAUSE; 9747 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9748 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9749 break; 9750 case GAUDI2_EVENT_PCIE_FATAL_ERR: 9751 error_count = GAUDI2_NA_EVENT_CAUSE; 9752 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9753 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9754 break; 9755 case GAUDI2_EVENT_TPC0_BMON_SPMU: 9756 case GAUDI2_EVENT_TPC1_BMON_SPMU: 9757 case GAUDI2_EVENT_TPC2_BMON_SPMU: 9758 case GAUDI2_EVENT_TPC3_BMON_SPMU: 9759 case GAUDI2_EVENT_TPC4_BMON_SPMU: 9760 case GAUDI2_EVENT_TPC5_BMON_SPMU: 9761 case GAUDI2_EVENT_TPC6_BMON_SPMU: 9762 case GAUDI2_EVENT_TPC7_BMON_SPMU: 9763 case GAUDI2_EVENT_TPC8_BMON_SPMU: 9764 case GAUDI2_EVENT_TPC9_BMON_SPMU: 9765 case GAUDI2_EVENT_TPC10_BMON_SPMU: 9766 case GAUDI2_EVENT_TPC11_BMON_SPMU: 9767 case GAUDI2_EVENT_TPC12_BMON_SPMU: 9768 case GAUDI2_EVENT_TPC13_BMON_SPMU: 9769 case GAUDI2_EVENT_TPC14_BMON_SPMU: 9770 case GAUDI2_EVENT_TPC15_BMON_SPMU: 9771 case GAUDI2_EVENT_TPC16_BMON_SPMU: 9772 case GAUDI2_EVENT_TPC17_BMON_SPMU: 9773 case GAUDI2_EVENT_TPC18_BMON_SPMU: 9774 case GAUDI2_EVENT_TPC19_BMON_SPMU: 9775 case GAUDI2_EVENT_TPC20_BMON_SPMU: 9776 case GAUDI2_EVENT_TPC21_BMON_SPMU: 9777 case GAUDI2_EVENT_TPC22_BMON_SPMU: 9778 case GAUDI2_EVENT_TPC23_BMON_SPMU: 9779 case GAUDI2_EVENT_TPC24_BMON_SPMU: 9780 case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU: 9781 case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU: 9782 case GAUDI2_EVENT_MME0_WAP_BMON_SPMU: 9783 case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU: 9784 case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU: 9785 case GAUDI2_EVENT_MME1_WAP_BMON_SPMU: 9786 case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU: 9787 case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU: 9788 case GAUDI2_EVENT_MME2_WAP_BMON_SPMU: 9789 case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU: 9790 case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU: 9791 case GAUDI2_EVENT_MME3_WAP_BMON_SPMU: 9792 case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU: 9793 fallthrough; 9794 case GAUDI2_EVENT_DEC0_BMON_SPMU: 9795 case GAUDI2_EVENT_DEC1_BMON_SPMU: 9796 case GAUDI2_EVENT_DEC2_BMON_SPMU: 9797 case GAUDI2_EVENT_DEC3_BMON_SPMU: 9798 case GAUDI2_EVENT_DEC4_BMON_SPMU: 9799 case GAUDI2_EVENT_DEC5_BMON_SPMU: 9800 case GAUDI2_EVENT_DEC6_BMON_SPMU: 9801 case GAUDI2_EVENT_DEC7_BMON_SPMU: 9802 case GAUDI2_EVENT_DEC8_BMON_SPMU: 9803 case GAUDI2_EVENT_DEC9_BMON_SPMU: 9804 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU: 9805 error_count = GAUDI2_NA_EVENT_CAUSE; 9806 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9807 break; 9808 9809 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 9810 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 9811 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 9812 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 9813 gaudi2_print_clk_change_info(hdev, event_type, &event_mask); 9814 error_count = GAUDI2_NA_EVENT_CAUSE; 9815 break; 9816 9817 case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC: 9818 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); 9819 error_count = GAUDI2_NA_EVENT_CAUSE; 9820 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9821 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9822 break; 9823 9824 case GAUDI2_EVENT_PCIE_FLR_REQUESTED: 9825 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9826 error_count = GAUDI2_NA_EVENT_CAUSE; 9827 /* Do nothing- FW will handle it */ 9828 break; 9829 9830 case GAUDI2_EVENT_PCIE_P2P_MSIX: 9831 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type); 9832 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9833 break; 9834 9835 case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE: 9836 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; 9837 error_count = gaudi2_handle_sm_err(hdev, event_type, index); 9838 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9839 break; 9840 9841 case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR: 9842 error_count = GAUDI2_NA_EVENT_CAUSE; 9843 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9844 break; 9845 9846 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 9847 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", 9848 le64_to_cpu(eq_entry->data[0])); 9849 error_count = GAUDI2_NA_EVENT_CAUSE; 9850 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9851 break; 9852 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT: 9853 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", 9854 le64_to_cpu(eq_entry->data[0])); 9855 error_count = GAUDI2_NA_EVENT_CAUSE; 9856 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9857 break; 9858 9859 case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED: 9860 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); 9861 error_count = GAUDI2_NA_EVENT_CAUSE; 9862 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9863 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9864 break; 9865 9866 case GAUDI2_EVENT_ARC_DCCM_FULL: 9867 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); 9868 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9869 break; 9870 9871 case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED: 9872 case GAUDI2_EVENT_CPU_DEV_RESET_REQ: 9873 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9874 error_count = GAUDI2_NA_EVENT_CAUSE; 9875 is_critical = true; 9876 break; 9877 9878 default: 9879 if (gaudi2_irq_map_table[event_type].valid) { 9880 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", 9881 event_type); 9882 error_count = GAUDI2_NA_EVENT_CAUSE; 9883 } 9884 } 9885 9886 /* Make sure to dump an error in case no error cause was printed so far. 9887 * Note that although we have counted the errors, we use this number as 9888 * a boolean. 9889 */ 9890 if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type)) 9891 gaudi2_print_event(hdev, event_type, true, "%d", event_type); 9892 else if (error_count == 0) 9893 gaudi2_print_event(hdev, event_type, true, 9894 "No error cause for H/W event %u", event_type); 9895 9896 if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) || 9897 reset_required) { 9898 if (reset_required || 9899 (gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD)) 9900 reset_flags |= HL_DRV_RESET_HARD; 9901 9902 if (hdev->hard_reset_on_fw_events || 9903 (hdev->asic_prop.fw_security_enabled && is_critical)) 9904 goto reset_device; 9905 } 9906 9907 /* Send unmask irq only for interrupts not classified as MSG */ 9908 if (!gaudi2_irq_map_table[event_type].msg) 9909 hl_fw_unmask_irq(hdev, event_type); 9910 9911 if (event_mask) 9912 hl_notifier_event_send_all(hdev, event_mask); 9913 9914 return; 9915 9916 reset_device: 9917 if (hdev->asic_prop.fw_security_enabled && is_critical) { 9918 reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW; 9919 event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE; 9920 } else { 9921 reset_flags |= HL_DRV_RESET_DELAY; 9922 } 9923 /* escalate general hw errors to critical/fatal error */ 9924 if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR) 9925 hl_handle_critical_hw_err(hdev, event_type, &event_mask); 9926 9927 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 9928 hl_device_cond_reset(hdev, reset_flags, event_mask); 9929 } 9930 gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device * hdev,struct packet_lin_dma * lin_dma_pkt,dma_addr_t pkt_dma_addr,u32 hw_queue_id,u32 size,u64 addr,u32 val)9931 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, 9932 struct packet_lin_dma *lin_dma_pkt, dma_addr_t pkt_dma_addr, 9933 u32 hw_queue_id, u32 size, u64 addr, u32 val) 9934 { 9935 u32 ctl, pkt_size; 9936 int rc = 0; 9937 9938 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 9939 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); 9940 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1); 9941 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1); 9942 9943 lin_dma_pkt->ctl = cpu_to_le32(ctl); 9944 lin_dma_pkt->src_addr = cpu_to_le64(val); 9945 lin_dma_pkt->dst_addr = cpu_to_le64(addr); 9946 lin_dma_pkt->tsize = cpu_to_le32(size); 9947 9948 pkt_size = sizeof(struct packet_lin_dma); 9949 9950 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr); 9951 if (rc) 9952 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", 9953 hw_queue_id); 9954 9955 return rc; 9956 } 9957 gaudi2_memset_device_memory(struct hl_device * hdev,u64 addr,u64 size,u64 val)9958 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) 9959 { 9960 u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 9961 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 9962 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 9963 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0}; 9964 u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val, 9965 old_mmubp, mmubp, num_of_pkts, busy, pkt_size; 9966 u64 comp_addr, cur_addr = addr, end_addr = addr + size; 9967 struct asic_fixed_properties *prop = &hdev->asic_prop; 9968 void *lin_dma_pkts_arr; 9969 dma_addr_t pkt_dma_addr; 9970 int rc = 0, dma_num = 0; 9971 9972 if (prop->edma_enabled_mask == 0) { 9973 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); 9974 return -EIO; 9975 } 9976 9977 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; 9978 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 9979 comp_addr = CFG_BASE + sob_addr; 9980 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 9981 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 9982 mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) | 9983 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1); 9984 9985 /* Calculate how many lin dma pkts we'll need */ 9986 num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G); 9987 pkt_size = sizeof(struct packet_lin_dma); 9988 9989 lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts, 9990 &pkt_dma_addr, GFP_KERNEL); 9991 if (!lin_dma_pkts_arr) 9992 return -ENOMEM; 9993 9994 /* 9995 * set mmu bypass for the scrubbing - all ddmas are configured the same so save 9996 * only the first one to restore later 9997 * also set the sob addr for all edma cores for completion. 9998 * set QM as trusted to allow it to access physical address with MMU bp. 9999 */ 10000 old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP); 10001 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10002 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10003 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10004 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10005 10006 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10007 continue; 10008 10009 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + 10010 edma_offset, mmubp); 10011 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 10012 lower_32_bits(comp_addr)); 10013 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 10014 upper_32_bits(comp_addr)); 10015 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 10016 comp_val); 10017 gaudi2_qman_set_test_mode(hdev, 10018 edma_queues_id[dcore] + 4 * edma_idx, true); 10019 } 10020 } 10021 10022 WREG32(sob_addr, 0); 10023 10024 while (cur_addr < end_addr) { 10025 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10026 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10027 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10028 10029 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10030 continue; 10031 10032 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); 10033 10034 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev, 10035 (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num, 10036 pkt_dma_addr + dma_num * pkt_size, 10037 edma_queues_id[dcore] + edma_idx * 4, 10038 chunk_size, cur_addr, val); 10039 if (rc) 10040 goto end; 10041 10042 dma_num++; 10043 cur_addr += chunk_size; 10044 if (cur_addr == end_addr) 10045 break; 10046 } 10047 } 10048 } 10049 10050 rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000); 10051 if (rc) { 10052 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n"); 10053 goto end; 10054 } 10055 end: 10056 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10057 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10058 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10059 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10060 10061 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10062 continue; 10063 10064 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp); 10065 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0); 10066 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0); 10067 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0); 10068 gaudi2_qman_set_test_mode(hdev, 10069 edma_queues_id[dcore] + 4 * edma_idx, false); 10070 } 10071 } 10072 10073 WREG32(sob_addr, 0); 10074 hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr); 10075 10076 return rc; 10077 } 10078 gaudi2_scrub_device_dram(struct hl_device * hdev,u64 val)10079 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) 10080 { 10081 int rc; 10082 struct asic_fixed_properties *prop = &hdev->asic_prop; 10083 u64 size = prop->dram_end_address - prop->dram_user_base_address; 10084 10085 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); 10086 10087 if (rc) 10088 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", 10089 prop->dram_user_base_address, size); 10090 return rc; 10091 } 10092 gaudi2_scrub_device_mem(struct hl_device * hdev)10093 static int gaudi2_scrub_device_mem(struct hl_device *hdev) 10094 { 10095 int rc; 10096 struct asic_fixed_properties *prop = &hdev->asic_prop; 10097 u64 val = hdev->memory_scrub_val; 10098 u64 addr, size; 10099 10100 if (!hdev->memory_scrub) 10101 return 0; 10102 10103 /* scrub SRAM */ 10104 addr = prop->sram_user_base_address; 10105 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); 10106 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", 10107 addr, addr + size, val); 10108 rc = gaudi2_memset_device_memory(hdev, addr, size, val); 10109 if (rc) { 10110 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); 10111 return rc; 10112 } 10113 10114 /* scrub DRAM */ 10115 rc = gaudi2_scrub_device_dram(hdev, val); 10116 if (rc) { 10117 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); 10118 return rc; 10119 } 10120 return 0; 10121 } 10122 gaudi2_restore_user_sm_registers(struct hl_device * hdev)10123 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) 10124 { 10125 u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr, 10126 cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr; 10127 u32 val, size, offset; 10128 int dcore_id; 10129 10130 offset = hdev->asic_prop.first_available_cq[0] * 4; 10131 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset; 10132 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset; 10133 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset; 10134 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset; 10135 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset; 10136 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset; 10137 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - 10138 (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset); 10139 10140 /* memset dcore0 CQ registers */ 10141 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10142 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10143 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10144 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10145 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10146 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10147 10148 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET; 10149 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET; 10150 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET; 10151 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET; 10152 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET; 10153 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET; 10154 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; 10155 10156 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10157 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10158 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10159 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10160 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10161 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10162 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10163 10164 cq_lbw_l_addr += DCORE_OFFSET; 10165 cq_lbw_h_addr += DCORE_OFFSET; 10166 cq_lbw_data_addr += DCORE_OFFSET; 10167 cq_base_l_addr += DCORE_OFFSET; 10168 cq_base_h_addr += DCORE_OFFSET; 10169 cq_size_addr += DCORE_OFFSET; 10170 } 10171 10172 offset = hdev->asic_prop.first_available_user_mon[0] * 4; 10173 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset; 10174 val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT; 10175 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); 10176 10177 /* memset dcore0 monitors */ 10178 gaudi2_memset_device_lbw(hdev, addr, size, val); 10179 10180 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset; 10181 gaudi2_memset_device_lbw(hdev, addr, size, 0); 10182 10183 mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET; 10184 mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET; 10185 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; 10186 10187 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10188 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val); 10189 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0); 10190 mon_sts_addr += DCORE_OFFSET; 10191 mon_cfg_addr += DCORE_OFFSET; 10192 } 10193 10194 offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10195 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset; 10196 val = 0; 10197 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - 10198 (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10199 10200 /* memset dcore0 sobs */ 10201 gaudi2_memset_device_lbw(hdev, addr, size, val); 10202 10203 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET; 10204 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; 10205 10206 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10207 gaudi2_memset_device_lbw(hdev, addr, size, val); 10208 addr += DCORE_OFFSET; 10209 } 10210 10211 /* Flush all WREG to prevent race */ 10212 val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10213 } 10214 gaudi2_restore_user_qm_registers(struct hl_device * hdev)10215 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) 10216 { 10217 u32 reg_base, hw_queue_id; 10218 10219 for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0; 10220 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10221 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10222 continue; 10223 10224 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10225 10226 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10227 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10228 } 10229 10230 /* Flush all WREG to prevent race */ 10231 RREG32(mmPDMA0_QM_ARB_CFG_0); 10232 } 10233 gaudi2_restore_nic_qm_registers(struct hl_device * hdev)10234 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) 10235 { 10236 u32 reg_base, hw_queue_id; 10237 10238 for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3; 10239 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10240 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10241 continue; 10242 10243 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10244 10245 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10246 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10247 } 10248 10249 /* Flush all WREG to prevent race */ 10250 RREG32(mmPDMA0_QM_ARB_CFG_0); 10251 } 10252 gaudi2_context_switch(struct hl_device * hdev,u32 asid)10253 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) 10254 { 10255 return 0; 10256 } 10257 gaudi2_restore_phase_topology(struct hl_device * hdev)10258 static void gaudi2_restore_phase_topology(struct hl_device *hdev) 10259 { 10260 } 10261 gaudi2_init_block_instances(struct hl_device * hdev,u32 block_idx,struct dup_block_ctx * cfg_ctx)10262 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, 10263 struct dup_block_ctx *cfg_ctx) 10264 { 10265 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; 10266 u8 seq; 10267 int i; 10268 10269 for (i = 0 ; i < cfg_ctx->instances ; i++) { 10270 seq = block_idx * cfg_ctx->instances + i; 10271 10272 /* skip disabled instance */ 10273 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) 10274 continue; 10275 10276 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, 10277 cfg_ctx->data); 10278 } 10279 } 10280 gaudi2_init_blocks_with_mask(struct hl_device * hdev,struct dup_block_ctx * cfg_ctx,u64 mask)10281 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, 10282 u64 mask) 10283 { 10284 int i; 10285 10286 cfg_ctx->enabled_mask = mask; 10287 10288 for (i = 0 ; i < cfg_ctx->blocks ; i++) 10289 gaudi2_init_block_instances(hdev, i, cfg_ctx); 10290 } 10291 gaudi2_init_blocks(struct hl_device * hdev,struct dup_block_ctx * cfg_ctx)10292 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) 10293 { 10294 gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX); 10295 } 10296 gaudi2_debugfs_read_dma(struct hl_device * hdev,u64 addr,u32 size,void * blob_addr)10297 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) 10298 { 10299 void *host_mem_virtual_addr; 10300 dma_addr_t host_mem_dma_addr; 10301 u64 reserved_va_base; 10302 u32 pos, size_left, size_to_dma; 10303 struct hl_ctx *ctx; 10304 int rc = 0; 10305 10306 /* Fetch the ctx */ 10307 ctx = hl_get_compute_ctx(hdev); 10308 if (!ctx) { 10309 dev_err(hdev->dev, "No ctx available\n"); 10310 return -EINVAL; 10311 } 10312 10313 /* Allocate buffers for read and for poll */ 10314 host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr, 10315 GFP_KERNEL | __GFP_ZERO); 10316 if (host_mem_virtual_addr == NULL) { 10317 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); 10318 rc = -ENOMEM; 10319 goto put_ctx; 10320 } 10321 10322 /* Reserve VM region on asic side */ 10323 reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M, 10324 HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10325 if (!reserved_va_base) { 10326 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); 10327 rc = -ENOMEM; 10328 goto free_data_buffer; 10329 } 10330 10331 /* Create mapping on asic side */ 10332 mutex_lock(&hdev->mmu_lock); 10333 10334 rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M); 10335 if (rc) { 10336 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); 10337 goto unreserve_va; 10338 } 10339 10340 rc = hl_mmu_invalidate_cache_range(hdev, false, 10341 MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV, 10342 ctx->asid, reserved_va_base, SZ_2M); 10343 if (rc) { 10344 hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10345 goto unreserve_va; 10346 } 10347 10348 mutex_unlock(&hdev->mmu_lock); 10349 10350 /* Enable MMU on KDMA */ 10351 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); 10352 10353 pos = 0; 10354 size_left = size; 10355 size_to_dma = SZ_2M; 10356 10357 while (size_left > 0) { 10358 if (size_left < SZ_2M) 10359 size_to_dma = size_left; 10360 10361 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false); 10362 if (rc) 10363 break; 10364 10365 memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma); 10366 10367 if (size_left <= SZ_2M) 10368 break; 10369 10370 pos += SZ_2M; 10371 addr += SZ_2M; 10372 size_left -= SZ_2M; 10373 } 10374 10375 gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID); 10376 10377 mutex_lock(&hdev->mmu_lock); 10378 10379 rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10380 if (rc) 10381 goto unreserve_va; 10382 10383 rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR, 10384 ctx->asid, reserved_va_base, SZ_2M); 10385 10386 unreserve_va: 10387 mutex_unlock(&hdev->mmu_lock); 10388 hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M); 10389 free_data_buffer: 10390 hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr); 10391 put_ctx: 10392 hl_ctx_put(ctx); 10393 10394 return rc; 10395 } 10396 gaudi2_internal_cb_pool_init(struct hl_device * hdev,struct hl_ctx * ctx)10397 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) 10398 { 10399 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10400 int min_alloc_order, rc; 10401 10402 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10403 return 0; 10404 10405 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, 10406 HOST_SPACE_INTERNAL_CB_SZ, 10407 &hdev->internal_cb_pool_dma_addr, 10408 GFP_KERNEL | __GFP_ZERO); 10409 10410 if (!hdev->internal_cb_pool_virt_addr) 10411 return -ENOMEM; 10412 10413 min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev), 10414 gaudi2_get_wait_cb_size(hdev))); 10415 10416 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); 10417 if (!hdev->internal_cb_pool) { 10418 dev_err(hdev->dev, "Failed to create internal CB pool\n"); 10419 rc = -ENOMEM; 10420 goto free_internal_cb_pool; 10421 } 10422 10423 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, 10424 HOST_SPACE_INTERNAL_CB_SZ, -1); 10425 if (rc) { 10426 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); 10427 rc = -EFAULT; 10428 goto destroy_internal_cb_pool; 10429 } 10430 10431 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 10432 HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10433 10434 if (!hdev->internal_cb_va_base) { 10435 rc = -ENOMEM; 10436 goto destroy_internal_cb_pool; 10437 } 10438 10439 mutex_lock(&hdev->mmu_lock); 10440 10441 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, 10442 HOST_SPACE_INTERNAL_CB_SZ); 10443 if (rc) 10444 goto unreserve_internal_cb_pool; 10445 10446 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); 10447 if (rc) 10448 goto unmap_internal_cb_pool; 10449 10450 mutex_unlock(&hdev->mmu_lock); 10451 10452 return 0; 10453 10454 unmap_internal_cb_pool: 10455 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10456 unreserve_internal_cb_pool: 10457 mutex_unlock(&hdev->mmu_lock); 10458 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10459 destroy_internal_cb_pool: 10460 gen_pool_destroy(hdev->internal_cb_pool); 10461 free_internal_cb_pool: 10462 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10463 hdev->internal_cb_pool_dma_addr); 10464 10465 return rc; 10466 } 10467 gaudi2_internal_cb_pool_fini(struct hl_device * hdev,struct hl_ctx * ctx)10468 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) 10469 { 10470 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10471 10472 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10473 return; 10474 10475 mutex_lock(&hdev->mmu_lock); 10476 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10477 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10478 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 10479 mutex_unlock(&hdev->mmu_lock); 10480 10481 gen_pool_destroy(hdev->internal_cb_pool); 10482 10483 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10484 hdev->internal_cb_pool_dma_addr); 10485 } 10486 gaudi2_restore_user_registers(struct hl_device * hdev)10487 static void gaudi2_restore_user_registers(struct hl_device *hdev) 10488 { 10489 gaudi2_restore_user_sm_registers(hdev); 10490 gaudi2_restore_user_qm_registers(hdev); 10491 } 10492 gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx * ctx)10493 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10494 { 10495 struct hl_device *hdev = ctx->hdev; 10496 struct asic_fixed_properties *prop = &hdev->asic_prop; 10497 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10498 int rc; 10499 10500 rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10501 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); 10502 if (rc) 10503 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", 10504 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10505 10506 return rc; 10507 } 10508 gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx * ctx)10509 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10510 { 10511 struct hl_device *hdev = ctx->hdev; 10512 struct asic_fixed_properties *prop = &hdev->asic_prop; 10513 int rc; 10514 10515 rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10516 prop->pmmu.page_size, true); 10517 if (rc) 10518 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", 10519 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10520 } 10521 gaudi2_ctx_init(struct hl_ctx * ctx)10522 static int gaudi2_ctx_init(struct hl_ctx *ctx) 10523 { 10524 int rc; 10525 10526 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); 10527 if (rc) 10528 return rc; 10529 10530 /* No need to clear user registers if the device has just 10531 * performed reset, we restore only nic qm registers 10532 */ 10533 if (ctx->hdev->reset_upon_device_release) 10534 gaudi2_restore_nic_qm_registers(ctx->hdev); 10535 else 10536 gaudi2_restore_user_registers(ctx->hdev); 10537 10538 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); 10539 if (rc) 10540 return rc; 10541 10542 rc = gaudi2_map_virtual_msix_doorbell_memory(ctx); 10543 if (rc) 10544 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10545 10546 return rc; 10547 } 10548 gaudi2_ctx_fini(struct hl_ctx * ctx)10549 static void gaudi2_ctx_fini(struct hl_ctx *ctx) 10550 { 10551 if (ctx->asid == HL_KERNEL_ASID_ID) 10552 return; 10553 10554 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10555 10556 gaudi2_unmap_virtual_msix_doorbell_memory(ctx); 10557 } 10558 gaudi2_pre_schedule_cs(struct hl_cs * cs)10559 static int gaudi2_pre_schedule_cs(struct hl_cs *cs) 10560 { 10561 struct hl_device *hdev = cs->ctx->hdev; 10562 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); 10563 u32 mon_payload, sob_id, mon_id; 10564 10565 if (!cs_needs_completion(cs)) 10566 return 0; 10567 10568 /* 10569 * First 64 SOB/MON are reserved for driver for QMAN auto completion 10570 * mechanism. Each SOB/MON pair are used for a pending CS with the same 10571 * cyclic index. The SOB value is increased when each of the CS jobs is 10572 * completed. When the SOB reaches the number of CS jobs, the monitor 10573 * generates MSI-X interrupt. 10574 */ 10575 10576 sob_id = mon_id = index; 10577 mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) | 10578 (1 << CQ_ENTRY_READY_SHIFT) | index; 10579 10580 gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload, 10581 cs->jobs_cnt); 10582 10583 return 0; 10584 } 10585 gaudi2_get_queue_id_for_cq(struct hl_device * hdev,u32 cq_idx)10586 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 10587 { 10588 return HL_INVALID_QUEUE; 10589 } 10590 gaudi2_gen_signal_cb(struct hl_device * hdev,void * data,u16 sob_id,u32 size,bool eb)10591 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) 10592 { 10593 struct hl_cb *cb = data; 10594 struct packet_msg_short *pkt; 10595 u32 value, ctl, pkt_size = sizeof(*pkt); 10596 10597 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); 10598 memset(pkt, 0, pkt_size); 10599 10600 /* Inc by 1, Mode ADD */ 10601 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); 10602 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1); 10603 10604 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); 10605 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */ 10606 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10607 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb); 10608 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10609 10610 pkt->value = cpu_to_le32(value); 10611 pkt->ctl = cpu_to_le32(ctl); 10612 10613 return size + pkt_size; 10614 } 10615 gaudi2_add_mon_msg_short(struct packet_msg_short * pkt,u32 value,u16 addr)10616 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) 10617 { 10618 u32 ctl, pkt_size = sizeof(*pkt); 10619 10620 memset(pkt, 0, pkt_size); 10621 10622 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 10623 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 10624 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10625 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10626 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0); 10627 10628 pkt->value = cpu_to_le32(value); 10629 pkt->ctl = cpu_to_le32(ctl); 10630 10631 return pkt_size; 10632 } 10633 gaudi2_add_arm_monitor_pkt(struct hl_device * hdev,struct packet_msg_short * pkt,u16 sob_base,u8 sob_mask,u16 sob_val,u16 addr)10634 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, 10635 u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr) 10636 { 10637 u32 ctl, value, pkt_size = sizeof(*pkt); 10638 u8 mask; 10639 10640 if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) { 10641 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); 10642 return 0; 10643 } 10644 10645 memset(pkt, 0, pkt_size); 10646 10647 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8); 10648 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val); 10649 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/ 10650 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask); 10651 10652 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 10653 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 10654 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10655 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10656 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10657 10658 pkt->value = cpu_to_le32(value); 10659 pkt->ctl = cpu_to_le32(ctl); 10660 10661 return pkt_size; 10662 } 10663 gaudi2_add_fence_pkt(struct packet_fence * pkt)10664 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) 10665 { 10666 u32 ctl, cfg, pkt_size = sizeof(*pkt); 10667 10668 memset(pkt, 0, pkt_size); 10669 10670 cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1); 10671 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); 10672 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2); 10673 10674 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE); 10675 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10676 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10677 10678 pkt->cfg = cpu_to_le32(cfg); 10679 pkt->ctl = cpu_to_le32(ctl); 10680 10681 return pkt_size; 10682 } 10683 gaudi2_gen_wait_cb(struct hl_device * hdev,struct hl_gen_wait_properties * prop)10684 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) 10685 { 10686 struct hl_cb *cb = prop->data; 10687 void *buf = (void *) (uintptr_t) (cb->kernel_address); 10688 10689 u64 monitor_base, fence_addr = 0; 10690 u32 stream_index, size = prop->size; 10691 u16 msg_addr_offset; 10692 10693 stream_index = prop->q_idx % 4; 10694 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + 10695 QM_FENCE2_OFFSET + stream_index * 4; 10696 10697 /* 10698 * monitor_base should be the content of the base0 address registers, 10699 * so it will be added to the msg short offsets 10700 */ 10701 monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 10702 10703 /* First monitor config packet: low address of the sync */ 10704 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - 10705 monitor_base; 10706 10707 size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset); 10708 10709 /* Second monitor config packet: high address of the sync */ 10710 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - 10711 monitor_base; 10712 10713 size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset); 10714 10715 /* 10716 * Third monitor config packet: the payload, i.e. what to write when the 10717 * sync triggers 10718 */ 10719 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - 10720 monitor_base; 10721 10722 size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset); 10723 10724 /* Fourth monitor config packet: bind the monitor to a sync object */ 10725 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; 10726 10727 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, 10728 prop->sob_val, msg_addr_offset); 10729 10730 /* Fence packet */ 10731 size += gaudi2_add_fence_pkt(buf + size); 10732 10733 return size; 10734 } 10735 gaudi2_reset_sob(struct hl_device * hdev,void * data)10736 static void gaudi2_reset_sob(struct hl_device *hdev, void *data) 10737 { 10738 struct hl_hw_sob *hw_sob = data; 10739 10740 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); 10741 10742 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); 10743 10744 kref_init(&hw_sob->kref); 10745 } 10746 gaudi2_reset_sob_group(struct hl_device * hdev,u16 sob_group)10747 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) 10748 { 10749 } 10750 gaudi2_get_device_time(struct hl_device * hdev)10751 static u64 gaudi2_get_device_time(struct hl_device *hdev) 10752 { 10753 u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 10754 10755 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 10756 } 10757 gaudi2_collective_wait_init_cs(struct hl_cs * cs)10758 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs) 10759 { 10760 return 0; 10761 } 10762 gaudi2_collective_wait_create_jobs(struct hl_device * hdev,struct hl_ctx * ctx,struct hl_cs * cs,u32 wait_queue_id,u32 collective_engine_id,u32 encaps_signal_offset)10763 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, 10764 struct hl_cs *cs, u32 wait_queue_id, 10765 u32 collective_engine_id, u32 encaps_signal_offset) 10766 { 10767 return -EINVAL; 10768 } 10769 10770 /* 10771 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address 10772 * to DMMU page-size address (64MB) before mapping it in 10773 * the MMU. 10774 * The operation is performed on both the virtual and physical addresses. 10775 * for device with 6 HBMs the scramble is: 10776 * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48] 10777 * 10778 * Example: 10779 * ============================================================================= 10780 * Allocated DRAM Reserved VA scrambled VA for MMU mapping Scrambled PA 10781 * Phys address in MMU last 10782 * HOP 10783 * ============================================================================= 10784 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1 10785 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3 10786 * ============================================================================= 10787 */ gaudi2_mmu_scramble_addr(struct hl_device * hdev,u64 raw_addr)10788 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) 10789 { 10790 struct asic_fixed_properties *prop = &hdev->asic_prop; 10791 u32 divisor, mod_va; 10792 u64 div_va; 10793 10794 /* accept any address in the DRAM address space */ 10795 if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE, 10796 VA_HBM_SPACE_END)) { 10797 10798 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 10799 div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va); 10800 return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) | 10801 (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) | 10802 (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT); 10803 } 10804 10805 return raw_addr; 10806 } 10807 gaudi2_mmu_descramble_addr(struct hl_device * hdev,u64 scrambled_addr)10808 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) 10809 { 10810 struct asic_fixed_properties *prop = &hdev->asic_prop; 10811 u32 divisor, mod_va; 10812 u64 div_va; 10813 10814 /* accept any address in the DRAM address space */ 10815 if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE, 10816 VA_HBM_SPACE_END)) { 10817 10818 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 10819 div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, 10820 PAGE_SIZE_64MB, &mod_va); 10821 10822 return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) + 10823 (div_va * divisor + mod_va)); 10824 } 10825 10826 return scrambled_addr; 10827 } 10828 gaudi2_get_dec_base_addr(struct hl_device * hdev,u32 core_id)10829 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) 10830 { 10831 u32 base = 0, dcore_id, dec_id; 10832 10833 if (core_id >= NUMBER_OF_DEC) { 10834 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); 10835 goto out; 10836 } 10837 10838 if (core_id < 8) { 10839 dcore_id = core_id / NUM_OF_DEC_PER_DCORE; 10840 dec_id = core_id % NUM_OF_DEC_PER_DCORE; 10841 10842 base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET + 10843 dec_id * DCORE_VDEC_OFFSET; 10844 } else { 10845 /* PCIe Shared Decoder */ 10846 base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET); 10847 } 10848 out: 10849 return base; 10850 } 10851 gaudi2_get_hw_block_id(struct hl_device * hdev,u64 block_addr,u32 * block_size,u32 * block_id)10852 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 10853 u32 *block_size, u32 *block_id) 10854 { 10855 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10856 int i; 10857 10858 for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) { 10859 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { 10860 *block_id = i; 10861 if (block_size) 10862 *block_size = gaudi2->mapped_blocks[i].size; 10863 return 0; 10864 } 10865 } 10866 10867 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); 10868 10869 return -EINVAL; 10870 } 10871 gaudi2_block_mmap(struct hl_device * hdev,struct vm_area_struct * vma,u32 block_id,u32 block_size)10872 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 10873 u32 block_id, u32 block_size) 10874 { 10875 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10876 u64 offset_in_bar; 10877 u64 address; 10878 int rc; 10879 10880 if (block_id >= NUM_USER_MAPPED_BLOCKS) { 10881 dev_err(hdev->dev, "Invalid block id %u", block_id); 10882 return -EINVAL; 10883 } 10884 10885 /* we allow mapping only an entire block */ 10886 if (block_size != gaudi2->mapped_blocks[block_id].size) { 10887 dev_err(hdev->dev, "Invalid block size %u", block_size); 10888 return -EINVAL; 10889 } 10890 10891 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; 10892 10893 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; 10894 10895 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 10896 VM_DONTCOPY | VM_NORESERVE); 10897 10898 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, 10899 block_size, vma->vm_page_prot); 10900 if (rc) 10901 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 10902 10903 return rc; 10904 } 10905 gaudi2_enable_events_from_fw(struct hl_device * hdev)10906 static void gaudi2_enable_events_from_fw(struct hl_device *hdev) 10907 { 10908 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10909 10910 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 10911 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); 10912 10913 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 10914 WREG32(irq_handler_offset, 10915 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id); 10916 } 10917 gaudi2_get_mmu_base(struct hl_device * hdev,u64 mmu_id,u32 * mmu_base)10918 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) 10919 { 10920 switch (mmu_id) { 10921 case HW_CAP_DCORE0_DMMU0: 10922 *mmu_base = mmDCORE0_HMMU0_MMU_BASE; 10923 break; 10924 case HW_CAP_DCORE0_DMMU1: 10925 *mmu_base = mmDCORE0_HMMU1_MMU_BASE; 10926 break; 10927 case HW_CAP_DCORE0_DMMU2: 10928 *mmu_base = mmDCORE0_HMMU2_MMU_BASE; 10929 break; 10930 case HW_CAP_DCORE0_DMMU3: 10931 *mmu_base = mmDCORE0_HMMU3_MMU_BASE; 10932 break; 10933 case HW_CAP_DCORE1_DMMU0: 10934 *mmu_base = mmDCORE1_HMMU0_MMU_BASE; 10935 break; 10936 case HW_CAP_DCORE1_DMMU1: 10937 *mmu_base = mmDCORE1_HMMU1_MMU_BASE; 10938 break; 10939 case HW_CAP_DCORE1_DMMU2: 10940 *mmu_base = mmDCORE1_HMMU2_MMU_BASE; 10941 break; 10942 case HW_CAP_DCORE1_DMMU3: 10943 *mmu_base = mmDCORE1_HMMU3_MMU_BASE; 10944 break; 10945 case HW_CAP_DCORE2_DMMU0: 10946 *mmu_base = mmDCORE2_HMMU0_MMU_BASE; 10947 break; 10948 case HW_CAP_DCORE2_DMMU1: 10949 *mmu_base = mmDCORE2_HMMU1_MMU_BASE; 10950 break; 10951 case HW_CAP_DCORE2_DMMU2: 10952 *mmu_base = mmDCORE2_HMMU2_MMU_BASE; 10953 break; 10954 case HW_CAP_DCORE2_DMMU3: 10955 *mmu_base = mmDCORE2_HMMU3_MMU_BASE; 10956 break; 10957 case HW_CAP_DCORE3_DMMU0: 10958 *mmu_base = mmDCORE3_HMMU0_MMU_BASE; 10959 break; 10960 case HW_CAP_DCORE3_DMMU1: 10961 *mmu_base = mmDCORE3_HMMU1_MMU_BASE; 10962 break; 10963 case HW_CAP_DCORE3_DMMU2: 10964 *mmu_base = mmDCORE3_HMMU2_MMU_BASE; 10965 break; 10966 case HW_CAP_DCORE3_DMMU3: 10967 *mmu_base = mmDCORE3_HMMU3_MMU_BASE; 10968 break; 10969 case HW_CAP_PMMU: 10970 *mmu_base = mmPMMU_HBW_MMU_BASE; 10971 break; 10972 default: 10973 return -EINVAL; 10974 } 10975 10976 return 0; 10977 } 10978 gaudi2_ack_mmu_error(struct hl_device * hdev,u64 mmu_id)10979 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) 10980 { 10981 bool is_pmmu = (mmu_id == HW_CAP_PMMU); 10982 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10983 u32 mmu_base; 10984 10985 if (!(gaudi2->hw_cap_initialized & mmu_id)) 10986 return; 10987 10988 if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base)) 10989 return; 10990 10991 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL); 10992 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 10993 } 10994 gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device * hdev,u64 mmu_cap_mask)10995 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 10996 { 10997 u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES; 10998 10999 /* check all HMMUs */ 11000 for (i = 0 ; i < num_of_hmmus ; i++) { 11001 mmu_id = HW_CAP_DCORE0_DMMU0 << i; 11002 11003 if (mmu_cap_mask & mmu_id) 11004 gaudi2_ack_mmu_error(hdev, mmu_id); 11005 } 11006 11007 /* check PMMU */ 11008 if (mmu_cap_mask & HW_CAP_PMMU) 11009 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU); 11010 11011 return 0; 11012 } 11013 gaudi2_get_msi_info(__le32 * table)11014 static void gaudi2_get_msi_info(__le32 *table) 11015 { 11016 table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX); 11017 } 11018 gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx)11019 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx) 11020 { 11021 switch (pll_idx) { 11022 case HL_GAUDI2_CPU_PLL: return CPU_PLL; 11023 case HL_GAUDI2_PCI_PLL: return PCI_PLL; 11024 case HL_GAUDI2_NIC_PLL: return NIC_PLL; 11025 case HL_GAUDI2_DMA_PLL: return DMA_PLL; 11026 case HL_GAUDI2_MESH_PLL: return MESH_PLL; 11027 case HL_GAUDI2_MME_PLL: return MME_PLL; 11028 case HL_GAUDI2_TPC_PLL: return TPC_PLL; 11029 case HL_GAUDI2_IF_PLL: return IF_PLL; 11030 case HL_GAUDI2_SRAM_PLL: return SRAM_PLL; 11031 case HL_GAUDI2_HBM_PLL: return HBM_PLL; 11032 case HL_GAUDI2_VID_PLL: return VID_PLL; 11033 case HL_GAUDI2_MSS_PLL: return MSS_PLL; 11034 default: return -EINVAL; 11035 } 11036 } 11037 gaudi2_gen_sync_to_engine_map(struct hl_device * hdev,struct hl_sync_to_engine_map * map)11038 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) 11039 { 11040 /* Not implemented */ 11041 return 0; 11042 } 11043 gaudi2_monitor_valid(struct hl_mon_state_dump * mon)11044 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon) 11045 { 11046 /* Not implemented */ 11047 return 0; 11048 } 11049 gaudi2_print_single_monitor(char ** buf,size_t * size,size_t * offset,struct hl_device * hdev,struct hl_mon_state_dump * mon)11050 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset, 11051 struct hl_device *hdev, struct hl_mon_state_dump *mon) 11052 { 11053 /* Not implemented */ 11054 return 0; 11055 } 11056 11057 gaudi2_print_fences_single_engine(struct hl_device * hdev,u64 base_offset,u64 status_base_offset,enum hl_sync_engine_type engine_type,u32 engine_id,char ** buf,size_t * size,size_t * offset)11058 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, 11059 u64 status_base_offset, enum hl_sync_engine_type engine_type, 11060 u32 engine_id, char **buf, size_t *size, size_t *offset) 11061 { 11062 /* Not implemented */ 11063 return 0; 11064 } 11065 11066 11067 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = { 11068 .monitor_valid = gaudi2_monitor_valid, 11069 .print_single_monitor = gaudi2_print_single_monitor, 11070 .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map, 11071 .print_fences_single_engine = gaudi2_print_fences_single_engine, 11072 }; 11073 gaudi2_state_dump_init(struct hl_device * hdev)11074 static void gaudi2_state_dump_init(struct hl_device *hdev) 11075 { 11076 /* Not implemented */ 11077 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; 11078 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; 11079 } 11080 gaudi2_get_sob_addr(struct hl_device * hdev,u32 sob_id)11081 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) 11082 { 11083 return 0; 11084 } 11085 gaudi2_get_stream_master_qid_arr(void)11086 static u32 *gaudi2_get_stream_master_qid_arr(void) 11087 { 11088 return NULL; 11089 } 11090 gaudi2_add_device_attr(struct hl_device * hdev,struct attribute_group * dev_clk_attr_grp,struct attribute_group * dev_vrm_attr_grp)11091 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 11092 struct attribute_group *dev_vrm_attr_grp) 11093 { 11094 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp); 11095 hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp); 11096 } 11097 gaudi2_mmu_get_real_page_size(struct hl_device * hdev,struct hl_mmu_properties * mmu_prop,u32 page_size,u32 * real_page_size,bool is_dram_addr)11098 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 11099 u32 page_size, u32 *real_page_size, bool is_dram_addr) 11100 { 11101 struct asic_fixed_properties *prop = &hdev->asic_prop; 11102 11103 /* for host pages the page size must be */ 11104 if (!is_dram_addr) { 11105 if (page_size % mmu_prop->page_size) 11106 goto page_size_err; 11107 11108 *real_page_size = mmu_prop->page_size; 11109 return 0; 11110 } 11111 11112 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) 11113 goto page_size_err; 11114 11115 /* 11116 * MMU page size is different from DRAM page size (more precisely, DMMU page is greater 11117 * than DRAM page size). 11118 * for this reason work with the DRAM page size and let the MMU scrambling routine handle 11119 * this mismatch when calculating the address to place in the MMU page table. 11120 * (in that case also make sure that the dram_page_size is not greater than the 11121 * mmu page size) 11122 */ 11123 *real_page_size = prop->dram_page_size; 11124 11125 return 0; 11126 11127 page_size_err: 11128 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", 11129 page_size, mmu_prop->page_size >> 10); 11130 return -EFAULT; 11131 } 11132 gaudi2_get_monitor_dump(struct hl_device * hdev,void * data)11133 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) 11134 { 11135 return -EOPNOTSUPP; 11136 } 11137 gaudi2_send_device_activity(struct hl_device * hdev,bool open)11138 int gaudi2_send_device_activity(struct hl_device *hdev, bool open) 11139 { 11140 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11141 11142 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 11143 return 0; 11144 11145 return hl_fw_send_device_activity(hdev, open); 11146 } 11147 11148 static const struct hl_asic_funcs gaudi2_funcs = { 11149 .early_init = gaudi2_early_init, 11150 .early_fini = gaudi2_early_fini, 11151 .late_init = gaudi2_late_init, 11152 .late_fini = gaudi2_late_fini, 11153 .sw_init = gaudi2_sw_init, 11154 .sw_fini = gaudi2_sw_fini, 11155 .hw_init = gaudi2_hw_init, 11156 .hw_fini = gaudi2_hw_fini, 11157 .halt_engines = gaudi2_halt_engines, 11158 .suspend = gaudi2_suspend, 11159 .resume = gaudi2_resume, 11160 .mmap = gaudi2_mmap, 11161 .ring_doorbell = gaudi2_ring_doorbell, 11162 .pqe_write = gaudi2_pqe_write, 11163 .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent, 11164 .asic_dma_free_coherent = gaudi2_dma_free_coherent, 11165 .scrub_device_mem = gaudi2_scrub_device_mem, 11166 .scrub_device_dram = gaudi2_scrub_device_dram, 11167 .get_int_queue_base = NULL, 11168 .test_queues = gaudi2_test_queues, 11169 .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc, 11170 .asic_dma_pool_free = gaudi2_dma_pool_free, 11171 .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc, 11172 .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free, 11173 .asic_dma_unmap_single = gaudi2_dma_unmap_single, 11174 .asic_dma_map_single = gaudi2_dma_map_single, 11175 .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable, 11176 .cs_parser = gaudi2_cs_parser, 11177 .asic_dma_map_sgtable = hl_dma_map_sgtable, 11178 .add_end_of_cb_packets = NULL, 11179 .update_eq_ci = gaudi2_update_eq_ci, 11180 .context_switch = gaudi2_context_switch, 11181 .restore_phase_topology = gaudi2_restore_phase_topology, 11182 .debugfs_read_dma = gaudi2_debugfs_read_dma, 11183 .add_device_attr = gaudi2_add_device_attr, 11184 .handle_eqe = gaudi2_handle_eqe, 11185 .get_events_stat = gaudi2_get_events_stat, 11186 .read_pte = NULL, 11187 .write_pte = NULL, 11188 .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache, 11189 .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range, 11190 .mmu_prefetch_cache_range = NULL, 11191 .send_heartbeat = gaudi2_send_heartbeat, 11192 .debug_coresight = gaudi2_debug_coresight, 11193 .is_device_idle = gaudi2_is_device_idle, 11194 .compute_reset_late_init = gaudi2_compute_reset_late_init, 11195 .hw_queues_lock = gaudi2_hw_queues_lock, 11196 .hw_queues_unlock = gaudi2_hw_queues_unlock, 11197 .get_pci_id = gaudi2_get_pci_id, 11198 .get_eeprom_data = gaudi2_get_eeprom_data, 11199 .get_monitor_dump = gaudi2_get_monitor_dump, 11200 .send_cpu_message = gaudi2_send_cpu_message, 11201 .pci_bars_map = gaudi2_pci_bars_map, 11202 .init_iatu = gaudi2_init_iatu, 11203 .rreg = hl_rreg, 11204 .wreg = hl_wreg, 11205 .halt_coresight = gaudi2_halt_coresight, 11206 .ctx_init = gaudi2_ctx_init, 11207 .ctx_fini = gaudi2_ctx_fini, 11208 .pre_schedule_cs = gaudi2_pre_schedule_cs, 11209 .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq, 11210 .load_firmware_to_device = NULL, 11211 .load_boot_fit_to_device = NULL, 11212 .get_signal_cb_size = gaudi2_get_signal_cb_size, 11213 .get_wait_cb_size = gaudi2_get_wait_cb_size, 11214 .gen_signal_cb = gaudi2_gen_signal_cb, 11215 .gen_wait_cb = gaudi2_gen_wait_cb, 11216 .reset_sob = gaudi2_reset_sob, 11217 .reset_sob_group = gaudi2_reset_sob_group, 11218 .get_device_time = gaudi2_get_device_time, 11219 .pb_print_security_errors = gaudi2_pb_print_security_errors, 11220 .collective_wait_init_cs = gaudi2_collective_wait_init_cs, 11221 .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs, 11222 .get_dec_base_addr = gaudi2_get_dec_base_addr, 11223 .scramble_addr = gaudi2_mmu_scramble_addr, 11224 .descramble_addr = gaudi2_mmu_descramble_addr, 11225 .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors, 11226 .get_hw_block_id = gaudi2_get_hw_block_id, 11227 .hw_block_mmap = gaudi2_block_mmap, 11228 .enable_events_from_fw = gaudi2_enable_events_from_fw, 11229 .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error, 11230 .get_msi_info = gaudi2_get_msi_info, 11231 .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx, 11232 .init_firmware_preload_params = gaudi2_init_firmware_preload_params, 11233 .init_firmware_loader = gaudi2_init_firmware_loader, 11234 .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm, 11235 .state_dump_init = gaudi2_state_dump_init, 11236 .get_sob_addr = &gaudi2_get_sob_addr, 11237 .set_pci_memory_regions = gaudi2_set_pci_memory_regions, 11238 .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr, 11239 .check_if_razwi_happened = gaudi2_check_if_razwi_happened, 11240 .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size, 11241 .access_dev_mem = hl_access_dev_mem, 11242 .set_dram_bar_base = gaudi2_set_hbm_bar_base, 11243 .set_engine_cores = gaudi2_set_engine_cores, 11244 .set_engines = gaudi2_set_engines, 11245 .send_device_activity = gaudi2_send_device_activity, 11246 .set_dram_properties = gaudi2_set_dram_properties, 11247 .set_binning_masks = gaudi2_set_binning_masks, 11248 }; 11249 gaudi2_set_asic_funcs(struct hl_device * hdev)11250 void gaudi2_set_asic_funcs(struct hl_device *hdev) 11251 { 11252 hdev->asic_funcs = &gaudi2_funcs; 11253 } 11254