1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2020-2022 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include "gaudi2P.h"
9 #include "gaudi2_masks.h"
10 #include "../include/gaudi2/gaudi2_special_blocks.h"
11 #include "../include/hw_ip/mmu/mmu_general.h"
12 #include "../include/hw_ip/mmu/mmu_v2_0.h"
13 #include "../include/gaudi2/gaudi2_packets.h"
14 #include "../include/gaudi2/gaudi2_reg_map.h"
15 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h"
16 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h"
17 
18 #include <linux/module.h>
19 #include <linux/pci.h>
20 #include <linux/hwmon.h>
21 #include <linux/iommu.h>
22 
23 #define GAUDI2_DMA_POOL_BLK_SIZE		SZ_256		/* 256 bytes */
24 
25 #define GAUDI2_RESET_TIMEOUT_MSEC		2000		/* 2000ms */
26 
27 #define GAUDI2_RESET_POLL_TIMEOUT_USEC		500000		/* 500ms */
28 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC		25000		/* 25s */
29 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC		25000		/* 25s */
30 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC	3000000		/* 3s */
31 #define GAUDI2_RESET_POLL_CNT			3
32 #define GAUDI2_RESET_WAIT_MSEC			1		/* 1ms */
33 #define GAUDI2_CPU_RESET_WAIT_MSEC		100		/* 100ms */
34 #define GAUDI2_PLDM_RESET_WAIT_MSEC		1000		/* 1s */
35 #define GAUDI2_CB_POOL_CB_CNT			512
36 #define GAUDI2_CB_POOL_CB_SIZE			SZ_128K		/* 128KB */
37 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC		4000000		/* 4s */
38 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC		25000000	/* 25s */
39 #define GAUDI2_TEST_QUEUE_WAIT_USEC		100000		/* 100ms */
40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC	1000000		/* 1s */
41 
42 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT		3
43 
44 /*
45  * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
46  * and the code relies on that value (for array size etc..) we define another value
47  * for MAX faulty TPCs which reflects the cluster binning requirements
48  */
49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS		1
50 #define MAX_FAULTY_XBARS			1
51 #define MAX_FAULTY_EDMAS			1
52 #define MAX_FAULTY_DECODERS			1
53 
54 #define GAUDI2_TPC_FULL_MASK			0x1FFFFFF
55 #define GAUDI2_HIF_HMMU_FULL_MASK		0xFFFF
56 #define GAUDI2_DECODER_FULL_MASK		0x3FF
57 
58 #define GAUDI2_NA_EVENT_CAUSE			0xFF
59 #define GAUDI2_NUM_OF_QM_ERR_CAUSE		18
60 #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE	25
61 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE		3
62 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE		14
63 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE		3
64 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE		2
65 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE		22
66 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE		31
67 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE		25
68 #define GAUDI2_NUM_OF_MME_ERR_CAUSE		16
69 #define GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE	5
70 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE		7
71 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE	8
72 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE		19
73 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE		9
74 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE		3
75 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE	3
76 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE	2
77 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE	2
78 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE	2
79 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE		5
80 
81 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC	(MMU_CONFIG_TIMEOUT_USEC * 10)
82 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC		(MMU_CONFIG_TIMEOUT_USEC * 200)
83 #define GAUDI2_ARB_WDT_TIMEOUT			(0x1000000)
84 
85 #define GAUDI2_VDEC_TIMEOUT_USEC		10000		/* 10ms */
86 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC		(GAUDI2_VDEC_TIMEOUT_USEC * 100)
87 
88 #define KDMA_TIMEOUT_USEC			USEC_PER_SEC
89 
90 #define IS_DMA_IDLE(dma_core_sts0)	\
91 	(!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK)))
92 
93 #define IS_DMA_HALTED(dma_core_sts1)	\
94 	((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK))
95 
96 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
97 
98 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK))
99 
100 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \
101 	((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \
102 	(((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \
103 	(((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK)))
104 
105 #define PCIE_DEC_EN_MASK			0x300
106 #define DEC_WORK_STATE_IDLE			0
107 #define DEC_WORK_STATE_PEND			3
108 #define IS_DEC_IDLE(dec_swreg15) \
109 	(((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \
110 	((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) ==  DEC_WORK_STATE_PEND)
111 
112 /* HBM MMU address scrambling parameters */
113 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE		SZ_8M
114 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT		26
115 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT		0
116 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK	DRAM_VA_HINT_MASK
117 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR	16
118 #define MMU_RANGE_INV_VA_LSB_SHIFT		12
119 #define MMU_RANGE_INV_VA_MSB_SHIFT		44
120 #define MMU_RANGE_INV_EN_SHIFT			0
121 #define MMU_RANGE_INV_ASID_EN_SHIFT		1
122 #define MMU_RANGE_INV_ASID_SHIFT		2
123 
124 /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has
125  * a 2 entries FIFO, and hence it is not enabled for it.
126  */
127 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK		GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
128 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK		GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
129 
130 #define GAUDI2_MAX_STRING_LEN			64
131 
132 #define GAUDI2_VDEC_MSIX_ENTRIES		(GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
133 							GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1)
134 
135 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
136 
137 /* RAZWI initiator coordinates */
138 #define RAZWI_GET_AXUSER_XY(x) \
139 	((x & 0xF8001FF0) >> 4)
140 
141 #define RAZWI_GET_AXUSER_LOW_XY(x) \
142 	((x & 0x00001FF0) >> 4)
143 
144 #define RAZWI_INITIATOR_AXUER_L_X_SHIFT		0
145 #define RAZWI_INITIATOR_AXUER_L_X_MASK		0x1F
146 #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT		5
147 #define RAZWI_INITIATOR_AXUER_L_Y_MASK		0xF
148 
149 #define RAZWI_INITIATOR_AXUER_H_X_SHIFT		23
150 #define RAZWI_INITIATOR_AXUER_H_X_MASK		0x1F
151 
152 #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \
153 	((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \
154 		(((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT))
155 
156 #define RAZWI_INITIATOR_ID_X_HIGH(x) \
157 		(((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT)
158 
159 #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \
160 	(RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh))
161 
162 #define PSOC_RAZWI_ENG_STR_SIZE 128
163 #define PSOC_RAZWI_MAX_ENG_PER_RTR 5
164 
165 /* HW scrambles only bits 0-25 */
166 #define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26)
167 
168 struct gaudi2_razwi_info {
169 	u32 axuser_xy;
170 	u32 rtr_ctrl;
171 	u16 eng_id;
172 	char *eng_name;
173 };
174 
175 static struct gaudi2_razwi_info common_razwi_info[] = {
176 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE,
177 				GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"},
178 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE,
179 				GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"},
180 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE,
181 				GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"},
182 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE,
183 				GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"},
184 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE,
185 				GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"},
186 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE,
187 				GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"},
188 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE,
189 				GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"},
190 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE,
191 				GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"},
192 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE,
193 				GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"},
194 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE,
195 				GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"},
196 		{RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE,
197 				GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"},
198 		{RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE,
199 				GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"},
200 		{RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE,
201 				GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"},
202 		{RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE,
203 				GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"},
204 		{RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE,
205 				GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"},
206 		{RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE,
207 				GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"},
208 		{RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE,
209 				GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"},
210 		{RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE,
211 				GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"},
212 		{RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE,
213 				GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"},
214 		{RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE,
215 				GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"},
216 		{RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE,
217 				GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"},
218 		{RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE,
219 				GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"},
220 		{RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE,
221 				GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"},
222 		{RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE,
223 				GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"},
224 		{RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE,
225 				GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"},
226 		{RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE,
227 				GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"},
228 		{RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE,
229 				GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"},
230 		{RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE,
231 				GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"},
232 		{RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE,
233 				GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"},
234 		{RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE,
235 				GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"},
236 		{RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE,
237 				GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"},
238 		{RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE,
239 				GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"},
240 		{RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE,
241 				GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"},
242 		{RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE,
243 				GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"},
244 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE,
245 				GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"},
246 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE,
247 				GAUDI2_ENGINE_ID_NIC0_0, "NIC0"},
248 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE,
249 				GAUDI2_ENGINE_ID_NIC0_1, "NIC1"},
250 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE,
251 				GAUDI2_ENGINE_ID_NIC1_0, "NIC2"},
252 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE,
253 				GAUDI2_ENGINE_ID_NIC1_1, "NIC3"},
254 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE,
255 				GAUDI2_ENGINE_ID_NIC2_0, "NIC4"},
256 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
257 				GAUDI2_ENGINE_ID_NIC2_1, "NIC5"},
258 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE,
259 				GAUDI2_ENGINE_ID_NIC3_0, "NIC6"},
260 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE,
261 				GAUDI2_ENGINE_ID_NIC3_1, "NIC7"},
262 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE,
263 				GAUDI2_ENGINE_ID_NIC4_0, "NIC8"},
264 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE,
265 				GAUDI2_ENGINE_ID_NIC4_1, "NIC9"},
266 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE,
267 				GAUDI2_ENGINE_ID_NIC5_0, "NIC10"},
268 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE,
269 				GAUDI2_ENGINE_ID_NIC5_1, "NIC11"},
270 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE,
271 				GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"},
272 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE,
273 				GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"},
274 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE,
275 				GAUDI2_ENGINE_ID_SIZE, "PMMU"},
276 		{RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE,
277 				GAUDI2_ENGINE_ID_SIZE, "PCIE"},
278 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE,
279 				GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"},
280 		{RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE,
281 				GAUDI2_ENGINE_ID_KDMA, "KDMA"},
282 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE,
283 				GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"},
284 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE,
285 				GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"},
286 		{RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE,
287 				GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"},
288 		{RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE,
289 				GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"},
290 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
291 				GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"},
292 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
293 				GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"},
294 		{RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
295 				GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"},
296 		{RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
297 				GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"},
298 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
299 				GAUDI2_ENGINE_ID_SIZE, "HMMU0"},
300 		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
301 				GAUDI2_ENGINE_ID_SIZE, "HMMU1"},
302 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
303 				GAUDI2_ENGINE_ID_SIZE, "HMMU2"},
304 		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
305 				GAUDI2_ENGINE_ID_SIZE, "HMMU3"},
306 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
307 				GAUDI2_ENGINE_ID_SIZE, "HMMU4"},
308 		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
309 				GAUDI2_ENGINE_ID_SIZE, "HMMU5"},
310 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
311 				GAUDI2_ENGINE_ID_SIZE, "HMMU6"},
312 		{RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE,
313 				GAUDI2_ENGINE_ID_SIZE, "HMMU7"},
314 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
315 				GAUDI2_ENGINE_ID_SIZE, "HMMU8"},
316 		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
317 				GAUDI2_ENGINE_ID_SIZE, "HMMU9"},
318 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
319 				GAUDI2_ENGINE_ID_SIZE, "HMMU10"},
320 		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
321 				GAUDI2_ENGINE_ID_SIZE, "HMMU11"},
322 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
323 				GAUDI2_ENGINE_ID_SIZE, "HMMU12"},
324 		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
325 				GAUDI2_ENGINE_ID_SIZE, "HMMU13"},
326 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
327 				GAUDI2_ENGINE_ID_SIZE, "HMMU14"},
328 		{RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE,
329 				GAUDI2_ENGINE_ID_SIZE, "HMMU15"},
330 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
331 				GAUDI2_ENGINE_ID_ROT_0, "ROT0"},
332 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE,
333 				GAUDI2_ENGINE_ID_ROT_1, "ROT1"},
334 		{RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE,
335 				GAUDI2_ENGINE_ID_PSOC, "CPU"},
336 		{RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE,
337 				GAUDI2_ENGINE_ID_PSOC, "PSOC"}
338 };
339 
340 static struct gaudi2_razwi_info mme_razwi_info[] = {
341 		/* MME X high coordinate is N/A, hence using only low coordinates */
342 		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE,
343 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"},
344 		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
345 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"},
346 		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE,
347 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"},
348 		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
349 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"},
350 		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE,
351 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"},
352 		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE,
353 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"},
354 		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE,
355 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"},
356 		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE,
357 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"},
358 		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE,
359 				GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"},
360 		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE,
361 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"},
362 		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
363 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"},
364 		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE,
365 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"},
366 		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
367 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"},
368 		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE,
369 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"},
370 		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE,
371 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"},
372 		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE,
373 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"},
374 		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE,
375 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"},
376 		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE,
377 				GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"},
378 		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE,
379 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"},
380 		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
381 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"},
382 		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE,
383 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"},
384 		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
385 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"},
386 		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE,
387 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"},
388 		{RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE,
389 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"},
390 		{RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE,
391 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"},
392 		{RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE,
393 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"},
394 		{RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE,
395 				GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"},
396 		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE,
397 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"},
398 		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
399 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"},
400 		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE,
401 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"},
402 		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
403 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"},
404 		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE,
405 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"},
406 		{RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE,
407 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"},
408 		{RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE,
409 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"},
410 		{RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE,
411 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"},
412 		{RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE,
413 				GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"}
414 };
415 
416 enum hl_pmmu_fatal_cause {
417 	LATENCY_RD_OUT_FIFO_OVERRUN,
418 	LATENCY_WR_OUT_FIFO_OVERRUN,
419 };
420 
421 enum hl_pcie_drain_ind_cause {
422 	LBW_AXI_DRAIN_IND,
423 	HBW_AXI_DRAIN_IND
424 };
425 
426 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = {
427 	[HBM_ID0] = 0xFFFC,
428 	[HBM_ID1] = 0xFFCF,
429 	[HBM_ID2] = 0xF7F7,
430 	[HBM_ID3] = 0x7F7F,
431 	[HBM_ID4] = 0xFCFF,
432 	[HBM_ID5] = 0xCFFF,
433 };
434 
435 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = {
436 	[0] = HBM_ID0,
437 	[1] = HBM_ID1,
438 	[2] = HBM_ID4,
439 	[3] = HBM_ID5,
440 };
441 
442 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = {
443 	[EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0,
444 	[EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2,
445 	[EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1,
446 	[EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3,
447 	[EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2,
448 	[EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4,
449 	[EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3,
450 	[EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5,
451 };
452 
453 static const int gaudi2_qman_async_event_id[] = {
454 	[GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM,
455 	[GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM,
456 	[GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM,
457 	[GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM,
458 	[GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM,
459 	[GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM,
460 	[GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM,
461 	[GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM,
462 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM,
463 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM,
464 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM,
465 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM,
466 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM,
467 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM,
468 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM,
469 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM,
470 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM,
471 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM,
472 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM,
473 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM,
474 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM,
475 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM,
476 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM,
477 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM,
478 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM,
479 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM,
480 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM,
481 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM,
482 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM,
483 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM,
484 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM,
485 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM,
486 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM,
487 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM,
488 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM,
489 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM,
490 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM,
491 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM,
492 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM,
493 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM,
494 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM,
495 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM,
496 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM,
497 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM,
498 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM,
499 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM,
500 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM,
501 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM,
502 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM,
503 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM,
504 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM,
505 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM,
506 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM,
507 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM,
508 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM,
509 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM,
510 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM,
511 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM,
512 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM,
513 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM,
514 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM,
515 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM,
516 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM,
517 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM,
518 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM,
519 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM,
520 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM,
521 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM,
522 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM,
523 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM,
524 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM,
525 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM,
526 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM,
527 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM,
528 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM,
529 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM,
530 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM,
531 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM,
532 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM,
533 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM,
534 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM,
535 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM,
536 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM,
537 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM,
538 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM,
539 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM,
540 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM,
541 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM,
542 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM,
543 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM,
544 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM,
545 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM,
546 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM,
547 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM,
548 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM,
549 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM,
550 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM,
551 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM,
552 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM,
553 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM,
554 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM,
555 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM,
556 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM,
557 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM,
558 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM,
559 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM,
560 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM,
561 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM,
562 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM,
563 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM,
564 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM,
565 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM,
566 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM,
567 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM,
568 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM,
569 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM,
570 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM,
571 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM,
572 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM,
573 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM,
574 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM,
575 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM,
576 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM,
577 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM,
578 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM,
579 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM,
580 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM,
581 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM,
582 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM,
583 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM,
584 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM,
585 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM,
586 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM,
587 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM,
588 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM,
589 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM,
590 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM,
591 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM,
592 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM,
593 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM,
594 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM,
595 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM,
596 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM,
597 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM,
598 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM,
599 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM,
600 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM,
601 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM,
602 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM,
603 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM,
604 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM,
605 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM,
606 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM,
607 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM,
608 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM,
609 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM,
610 	[GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0,
611 	[GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0,
612 	[GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0,
613 	[GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0,
614 	[GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1,
615 	[GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1,
616 	[GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1,
617 	[GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1,
618 	[GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0,
619 	[GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0,
620 	[GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0,
621 	[GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0,
622 	[GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1,
623 	[GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1,
624 	[GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1,
625 	[GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1,
626 	[GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0,
627 	[GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0,
628 	[GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0,
629 	[GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0,
630 	[GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1,
631 	[GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1,
632 	[GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1,
633 	[GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1,
634 	[GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0,
635 	[GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0,
636 	[GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0,
637 	[GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0,
638 	[GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1,
639 	[GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1,
640 	[GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1,
641 	[GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1,
642 	[GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0,
643 	[GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0,
644 	[GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0,
645 	[GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0,
646 	[GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1,
647 	[GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1,
648 	[GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1,
649 	[GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1,
650 	[GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0,
651 	[GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0,
652 	[GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0,
653 	[GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0,
654 	[GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1,
655 	[GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1,
656 	[GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1,
657 	[GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1,
658 	[GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0,
659 	[GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0,
660 	[GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0,
661 	[GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0,
662 	[GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1,
663 	[GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1,
664 	[GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1,
665 	[GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1,
666 	[GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0,
667 	[GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0,
668 	[GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0,
669 	[GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0,
670 	[GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1,
671 	[GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1,
672 	[GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1,
673 	[GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1,
674 	[GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0,
675 	[GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0,
676 	[GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0,
677 	[GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0,
678 	[GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1,
679 	[GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1,
680 	[GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1,
681 	[GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1,
682 	[GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0,
683 	[GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0,
684 	[GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0,
685 	[GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0,
686 	[GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1,
687 	[GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1,
688 	[GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1,
689 	[GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1,
690 	[GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0,
691 	[GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0,
692 	[GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0,
693 	[GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0,
694 	[GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1,
695 	[GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1,
696 	[GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1,
697 	[GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1,
698 	[GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0,
699 	[GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0,
700 	[GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0,
701 	[GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0,
702 	[GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1,
703 	[GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1,
704 	[GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1,
705 	[GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1,
706 	[GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
707 	[GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
708 	[GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
709 	[GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
710 	[GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
711 	[GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
712 	[GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
713 	[GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM
714 };
715 
716 static const int gaudi2_dma_core_async_event_id[] = {
717 	[DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE,
718 	[DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE,
719 	[DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE,
720 	[DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE,
721 	[DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE,
722 	[DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE,
723 	[DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE,
724 	[DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE,
725 	[DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE,
726 	[DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE,
727 	[DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE,
728 };
729 
730 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = {
731 	"qman sei intr",
732 	"arc sei intr"
733 };
734 
735 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = {
736 	"AXI_TERMINATOR WR",
737 	"AXI_TERMINATOR RD",
738 	"AXI SPLIT SEI Status"
739 };
740 
741 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = {
742 	"cbu_bresp_sei_intr_cause",
743 	"cbu_rresp_sei_intr_cause",
744 	"lbu_bresp_sei_intr_cause",
745 	"lbu_rresp_sei_intr_cause",
746 	"cbu_axi_split_intr_cause",
747 	"lbu_axi_split_intr_cause",
748 	"arc_ip_excptn_sei_intr_cause",
749 	"dmi_bresp_sei_intr_cause",
750 	"aux2apb_err_sei_intr_cause",
751 	"cfg_lbw_wr_terminated_intr_cause",
752 	"cfg_lbw_rd_terminated_intr_cause",
753 	"cfg_dccm_wr_terminated_intr_cause",
754 	"cfg_dccm_rd_terminated_intr_cause",
755 	"cfg_hbw_rd_terminated_intr_cause"
756 };
757 
758 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = {
759 	"msix_vcd_hbw_sei",
760 	"msix_l2c_hbw_sei",
761 	"msix_nrm_hbw_sei",
762 	"msix_abnrm_hbw_sei",
763 	"msix_vcd_lbw_sei",
764 	"msix_l2c_lbw_sei",
765 	"msix_nrm_lbw_sei",
766 	"msix_abnrm_lbw_sei",
767 	"apb_vcd_lbw_sei",
768 	"apb_l2c_lbw_sei",
769 	"apb_nrm_lbw_sei",
770 	"apb_abnrm_lbw_sei",
771 	"dec_sei",
772 	"dec_apb_sei",
773 	"trc_apb_sei",
774 	"lbw_mstr_if_sei",
775 	"axi_split_bresp_err_sei",
776 	"hbw_axi_wr_viol_sei",
777 	"hbw_axi_rd_viol_sei",
778 	"lbw_axi_wr_viol_sei",
779 	"lbw_axi_rd_viol_sei",
780 	"vcd_spi",
781 	"l2c_spi",
782 	"nrm_spi",
783 	"abnrm_spi",
784 };
785 
786 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = {
787 	"PQ AXI HBW error",
788 	"CQ AXI HBW error",
789 	"CP AXI HBW error",
790 	"CP error due to undefined OPCODE",
791 	"CP encountered STOP OPCODE",
792 	"CP AXI LBW error",
793 	"CP WRREG32 or WRBULK returned error",
794 	"N/A",
795 	"FENCE 0 inc over max value and clipped",
796 	"FENCE 1 inc over max value and clipped",
797 	"FENCE 2 inc over max value and clipped",
798 	"FENCE 3 inc over max value and clipped",
799 	"FENCE 0 dec under min value and clipped",
800 	"FENCE 1 dec under min value and clipped",
801 	"FENCE 2 dec under min value and clipped",
802 	"FENCE 3 dec under min value and clipped",
803 	"CPDMA Up overflow",
804 	"PQC L2H error"
805 };
806 
807 static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = {
808 	"RSVD0",
809 	"CQ AXI HBW error",
810 	"CP AXI HBW error",
811 	"CP error due to undefined OPCODE",
812 	"CP encountered STOP OPCODE",
813 	"CP AXI LBW error",
814 	"CP WRREG32 or WRBULK returned error",
815 	"N/A",
816 	"FENCE 0 inc over max value and clipped",
817 	"FENCE 1 inc over max value and clipped",
818 	"FENCE 2 inc over max value and clipped",
819 	"FENCE 3 inc over max value and clipped",
820 	"FENCE 0 dec under min value and clipped",
821 	"FENCE 1 dec under min value and clipped",
822 	"FENCE 2 dec under min value and clipped",
823 	"FENCE 3 dec under min value and clipped",
824 	"CPDMA Up overflow",
825 	"RSVD17",
826 	"CQ_WR_IFIFO_CI_ERR",
827 	"CQ_WR_CTL_CI_ERR",
828 	"ARC_CQF_RD_ERR",
829 	"ARC_CQ_WR_IFIFO_CI_ERR",
830 	"ARC_CQ_WR_CTL_CI_ERR",
831 	"ARC_AXI_ERR",
832 	"CP_SWITCH_WDT_ERR"
833 };
834 
835 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = {
836 	"Choice push while full error",
837 	"Choice Q watchdog error",
838 	"MSG AXI LBW returned with error"
839 };
840 
841 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = {
842 	"qm_axi_err",
843 	"qm_trace_fence_events",
844 	"qm_sw_err",
845 	"qm_cp_sw_stop",
846 	"lbw_mstr_rresp_err",
847 	"lbw_mstr_bresp_err",
848 	"lbw_msg_slverr",
849 	"hbw_msg_slverr",
850 	"wbc_slverr",
851 	"hbw_mstr_rresp_err",
852 	"hbw_mstr_bresp_err",
853 	"sb_resp_intr",
854 	"mrsb_resp_intr",
855 	"core_dw_status_0",
856 	"core_dw_status_1",
857 	"core_dw_status_2",
858 	"core_dw_status_3",
859 	"core_dw_status_4",
860 	"core_dw_status_5",
861 	"core_dw_status_6",
862 	"core_dw_status_7",
863 	"async_arc2cpu_sei_intr",
864 };
865 
866 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = {
867 	"tpc_address_exceed_slm",
868 	"tpc_div_by_0",
869 	"tpc_spu_mac_overflow",
870 	"tpc_spu_addsub_overflow",
871 	"tpc_spu_abs_overflow",
872 	"tpc_spu_fma_fp_dst_nan",
873 	"tpc_spu_fma_fp_dst_inf",
874 	"tpc_spu_convert_fp_dst_nan",
875 	"tpc_spu_convert_fp_dst_inf",
876 	"tpc_spu_fp_dst_denorm",
877 	"tpc_vpu_mac_overflow",
878 	"tpc_vpu_addsub_overflow",
879 	"tpc_vpu_abs_overflow",
880 	"tpc_vpu_convert_fp_dst_nan",
881 	"tpc_vpu_convert_fp_dst_inf",
882 	"tpc_vpu_fma_fp_dst_nan",
883 	"tpc_vpu_fma_fp_dst_inf",
884 	"tpc_vpu_fp_dst_denorm",
885 	"tpc_assertions",
886 	"tpc_illegal_instruction",
887 	"tpc_pc_wrap_around",
888 	"tpc_qm_sw_err",
889 	"tpc_hbw_rresp_err",
890 	"tpc_hbw_bresp_err",
891 	"tpc_lbw_rresp_err",
892 	"tpc_lbw_bresp_err",
893 	"st_unlock_already_locked",
894 	"invalid_lock_access",
895 	"LD_L protection violation",
896 	"ST_L protection violation",
897 	"D$ L0CS mismatch",
898 };
899 
900 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = {
901 	"agu_resp_intr",
902 	"qman_axi_err",
903 	"wap sei (wbc axi err)",
904 	"arc sei",
905 	"cfg access error",
906 	"qm_sw_err",
907 	"sbte_dbg_intr_0",
908 	"sbte_dbg_intr_1",
909 	"sbte_dbg_intr_2",
910 	"sbte_dbg_intr_3",
911 	"sbte_dbg_intr_4",
912 	"sbte_prtn_intr_0",
913 	"sbte_prtn_intr_1",
914 	"sbte_prtn_intr_2",
915 	"sbte_prtn_intr_3",
916 	"sbte_prtn_intr_4",
917 };
918 
919 static const char * const guadi2_mme_sbte_error_cause[GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE] = {
920 	"i0",
921 	"i1",
922 	"i2",
923 	"i3",
924 	"i4",
925 };
926 
927 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = {
928 	"WBC ERR RESP_0",
929 	"WBC ERR RESP_1",
930 	"AP SOURCE POS INF",
931 	"AP SOURCE NEG INF",
932 	"AP SOURCE NAN",
933 	"AP RESULT POS INF",
934 	"AP RESULT NEG INF",
935 };
936 
937 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = {
938 	"HBW Read returned with error RRESP",
939 	"HBW write returned with error BRESP",
940 	"LBW write returned with error BRESP",
941 	"descriptor_fifo_overflow",
942 	"KDMA SB LBW Read returned with error",
943 	"KDMA WBC LBW Write returned with error",
944 	"TRANSPOSE ENGINE DESC FIFO OVERFLOW",
945 	"WRONG CFG FOR COMMIT IN LIN DMA"
946 };
947 
948 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = {
949 	"HBW/LBW Read returned with error RRESP",
950 	"HBW/LBW write returned with error BRESP",
951 	"LBW write returned with error BRESP",
952 	"descriptor_fifo_overflow",
953 	"KDMA SB LBW Read returned with error",
954 	"KDMA WBC LBW Write returned with error",
955 	"TRANSPOSE ENGINE DESC FIFO OVERFLOW",
956 	"WRONG CFG FOR COMMIT IN LIN DMA"
957 };
958 
959 struct gaudi2_sm_sei_cause_data {
960 	const char *cause_name;
961 	const char *log_name;
962 };
963 
964 static const struct gaudi2_sm_sei_cause_data
965 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = {
966 	{"calculated SO value overflow/underflow", "SOB ID"},
967 	{"payload address of monitor is not aligned to 4B", "monitor addr"},
968 	{"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"},
969 };
970 
971 static const char * const
972 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = {
973 	"LATENCY_RD_OUT_FIFO_OVERRUN",
974 	"LATENCY_WR_OUT_FIFO_OVERRUN",
975 };
976 
977 static const char * const
978 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = {
979 	"LATENCY_RD_OUT_FIFO_OVERRUN",
980 	"LATENCY_WR_OUT_FIFO_OVERRUN",
981 };
982 
983 static const char * const
984 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = {
985 	"AXI drain HBW",
986 	"AXI drain LBW",
987 };
988 
989 static const char * const
990 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = {
991 	"HBW error response",
992 	"LBW error response",
993 	"TLP is blocked by RR"
994 };
995 
996 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = {
997 	[GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE,
998 	[GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE,
999 	[GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE,
1000 	[GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE,
1001 	[GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE,
1002 	[GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE,
1003 	[GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE,
1004 	[GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE,
1005 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE,
1006 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE,
1007 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE,
1008 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE,
1009 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE,
1010 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE,
1011 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE,
1012 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE,
1013 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE,
1014 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE,
1015 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE,
1016 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE,
1017 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE,
1018 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE,
1019 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE,
1020 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE,
1021 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE,
1022 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE,
1023 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE,
1024 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE,
1025 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE,
1026 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE,
1027 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE,
1028 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE,
1029 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE,
1030 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE,
1031 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE,
1032 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE,
1033 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE,
1034 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE,
1035 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE,
1036 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE,
1037 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE,
1038 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE,
1039 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE,
1040 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE,
1041 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE,
1042 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE,
1043 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE,
1044 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE,
1045 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE,
1046 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE,
1047 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE,
1048 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE,
1049 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE,
1050 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE,
1051 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE,
1052 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE,
1053 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE,
1054 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE,
1055 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE,
1056 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE,
1057 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE,
1058 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE,
1059 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE,
1060 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE,
1061 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE,
1062 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE,
1063 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE,
1064 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE,
1065 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE,
1066 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE,
1067 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE,
1068 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE,
1069 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE,
1070 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE,
1071 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE,
1072 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE,
1073 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE,
1074 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE,
1075 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE,
1076 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE,
1077 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE,
1078 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE,
1079 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE,
1080 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE,
1081 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE,
1082 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE,
1083 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE,
1084 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE,
1085 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE,
1086 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE,
1087 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE,
1088 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE,
1089 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE,
1090 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE,
1091 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE,
1092 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE,
1093 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE,
1094 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE,
1095 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE,
1096 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE,
1097 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE,
1098 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE,
1099 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE,
1100 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE,
1101 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE,
1102 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE,
1103 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE,
1104 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE,
1105 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE,
1106 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE,
1107 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE,
1108 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE,
1109 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE,
1110 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE,
1111 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE,
1112 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE,
1113 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE,
1114 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE,
1115 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE,
1116 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE,
1117 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE,
1118 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE,
1119 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE,
1120 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE,
1121 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE,
1122 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE,
1123 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE,
1124 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE,
1125 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE,
1126 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE,
1127 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE,
1128 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE,
1129 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE,
1130 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE,
1131 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE,
1132 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE,
1133 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE,
1134 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE,
1135 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE,
1136 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE,
1137 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE,
1138 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE,
1139 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE,
1140 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE,
1141 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE,
1142 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE,
1143 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE,
1144 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE,
1145 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE,
1146 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE,
1147 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE,
1148 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE,
1149 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE,
1150 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE,
1151 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE,
1152 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE,
1153 	[GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE,
1154 	[GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE,
1155 	[GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE,
1156 	[GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE,
1157 	[GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE,
1158 	[GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE,
1159 	[GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE,
1160 	[GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE,
1161 	[GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE,
1162 	[GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE,
1163 	[GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE,
1164 	[GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE,
1165 	[GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE,
1166 	[GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE,
1167 	[GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE,
1168 	[GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE,
1169 	[GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE,
1170 	[GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE,
1171 	[GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE,
1172 	[GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE,
1173 	[GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE,
1174 	[GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE,
1175 	[GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE,
1176 	[GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE,
1177 	[GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE,
1178 	[GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE,
1179 	[GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE,
1180 	[GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE,
1181 	[GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE,
1182 	[GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE,
1183 	[GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE,
1184 	[GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE,
1185 	[GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE,
1186 	[GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE,
1187 	[GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE,
1188 	[GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE,
1189 	[GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE,
1190 	[GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE,
1191 	[GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE,
1192 	[GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE,
1193 	[GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE,
1194 	[GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE,
1195 	[GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE,
1196 	[GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE,
1197 	[GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE,
1198 	[GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE,
1199 	[GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE,
1200 	[GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE,
1201 	[GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE,
1202 	[GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE,
1203 	[GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE,
1204 	[GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE,
1205 	[GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE,
1206 	[GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE,
1207 	[GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE,
1208 	[GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE,
1209 	[GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE,
1210 	[GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE,
1211 	[GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE,
1212 	[GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE,
1213 	[GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE,
1214 	[GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE,
1215 	[GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE,
1216 	[GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE,
1217 	[GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE,
1218 	[GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE,
1219 	[GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE,
1220 	[GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE,
1221 	[GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE,
1222 	[GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE,
1223 	[GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE,
1224 	[GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE,
1225 	[GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE,
1226 	[GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE,
1227 	[GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE,
1228 	[GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE,
1229 	[GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE,
1230 	[GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE,
1231 	[GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE,
1232 	[GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE,
1233 	[GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE,
1234 	[GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE,
1235 	[GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE,
1236 	[GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE,
1237 	[GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE,
1238 	[GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE,
1239 	[GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE,
1240 	[GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE,
1241 	[GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE,
1242 	[GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE,
1243 	[GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE,
1244 	[GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE,
1245 	[GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE,
1246 	[GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE,
1247 	[GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE,
1248 	[GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE,
1249 	[GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE,
1250 	[GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE,
1251 	[GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE,
1252 	[GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE,
1253 	[GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE,
1254 	[GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE,
1255 	[GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE,
1256 	[GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE
1257 };
1258 
1259 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = {
1260 	[CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE,
1261 	[CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE,
1262 	[CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE,
1263 	[CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE,
1264 	[CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE,
1265 	[CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE,
1266 	[CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE,
1267 	[CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE,
1268 	[CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE,
1269 	[CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE,
1270 	[CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE,
1271 	[CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE,
1272 	[CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE,
1273 	[CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE,
1274 	[CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE,
1275 	[CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE,
1276 	[CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE,
1277 	[CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE,
1278 	[CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE,
1279 	[CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE,
1280 	[CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE,
1281 	[CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE,
1282 	[CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE,
1283 	[CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE,
1284 	[CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE,
1285 	[CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE,
1286 	[CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE,
1287 	[CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE,
1288 	[CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE,
1289 	[CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE,
1290 	[CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE,
1291 	[CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE,
1292 	[CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE,
1293 	[CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE,
1294 	[CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE,
1295 	[CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE,
1296 	[CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE,
1297 	[CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE,
1298 	[CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE,
1299 	[CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE,
1300 	[CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE,
1301 	[CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE,
1302 	[CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE,
1303 	[CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE,
1304 	[CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE,
1305 	[CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE,
1306 	[CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE,
1307 	[CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE,
1308 	[CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE,
1309 	[CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE,
1310 	[CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE,
1311 	[CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE,
1312 	[CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE,
1313 	[CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE,
1314 	[CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE,
1315 	[CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE,
1316 	[CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE,
1317 	[CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE,
1318 	[CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE,
1319 	[CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE,
1320 	[CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE,
1321 	[CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE,
1322 	[CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE,
1323 	[CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE,
1324 	[CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE,
1325 	[CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE,
1326 	[CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE,
1327 	[CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE,
1328 	[CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE,
1329 };
1330 
1331 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = {
1332 	[CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE,
1333 	[CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE,
1334 	[CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE,
1335 	[CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE,
1336 	[CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE,
1337 	[CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE,
1338 	[CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE,
1339 	[CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE,
1340 	[CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE,
1341 	[CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE,
1342 	[CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE,
1343 	[CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE,
1344 	[CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE,
1345 	[CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE,
1346 	[CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE,
1347 	[CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE,
1348 	[CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE,
1349 	[CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE,
1350 	[CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE,
1351 	[CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE,
1352 	[CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE,
1353 	[CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE,
1354 	[CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE,
1355 	[CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE,
1356 	[CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE,
1357 	[CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE,
1358 	[CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE,
1359 	[CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE,
1360 	[CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE,
1361 	[CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE,
1362 	[CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE,
1363 	[CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE,
1364 	[CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE,
1365 	[CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE,
1366 	[CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE,
1367 	[CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE,
1368 	[CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE,
1369 	[CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE,
1370 	[CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE,
1371 	[CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE,
1372 	[CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE,
1373 	[CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE,
1374 	[CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE,
1375 	[CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE,
1376 	[CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE,
1377 	[CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE,
1378 	[CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE,
1379 	[CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE,
1380 	[CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE,
1381 	[CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE,
1382 	[CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE,
1383 	[CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE,
1384 	[CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE,
1385 	[CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE,
1386 	[CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE,
1387 	[CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE,
1388 	[CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE,
1389 	[CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE,
1390 	[CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE,
1391 	[CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE,
1392 	[CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE,
1393 	[CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE,
1394 	[CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE,
1395 	[CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE,
1396 	[CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE,
1397 	[CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE,
1398 	[CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE,
1399 	[CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE,
1400 	[CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE,
1401 };
1402 
1403 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = {
1404 	[MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE,
1405 	[MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE,
1406 	[MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE,
1407 	[MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE,
1408 };
1409 
1410 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = {
1411 	[GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0,
1412 	[GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0,
1413 	[GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0,
1414 	[GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0,
1415 	[GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1,
1416 	[GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1,
1417 	[GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1,
1418 	[GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1,
1419 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0,
1420 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0,
1421 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0,
1422 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0,
1423 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1,
1424 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1,
1425 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1,
1426 	[GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1,
1427 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0,
1428 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0,
1429 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0,
1430 	[GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0,
1431 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0,
1432 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0,
1433 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0,
1434 	[GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0,
1435 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1,
1436 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1,
1437 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1,
1438 	[GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1,
1439 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2,
1440 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2,
1441 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2,
1442 	[GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2,
1443 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3,
1444 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3,
1445 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3,
1446 	[GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3,
1447 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4,
1448 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4,
1449 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4,
1450 	[GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4,
1451 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5,
1452 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5,
1453 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5,
1454 	[GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5,
1455 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24,
1456 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24,
1457 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24,
1458 	[GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24,
1459 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2,
1460 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2,
1461 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2,
1462 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2,
1463 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3,
1464 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3,
1465 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3,
1466 	[GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3,
1467 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4,
1468 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4,
1469 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4,
1470 	[GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4,
1471 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6,
1472 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6,
1473 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6,
1474 	[GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6,
1475 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7,
1476 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7,
1477 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7,
1478 	[GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7,
1479 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8,
1480 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8,
1481 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8,
1482 	[GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8,
1483 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9,
1484 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9,
1485 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9,
1486 	[GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9,
1487 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10,
1488 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10,
1489 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10,
1490 	[GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10,
1491 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11,
1492 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11,
1493 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11,
1494 	[GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11,
1495 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4,
1496 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4,
1497 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4,
1498 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4,
1499 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5,
1500 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5,
1501 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5,
1502 	[GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5,
1503 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1,
1504 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1,
1505 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1,
1506 	[GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1,
1507 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12,
1508 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12,
1509 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12,
1510 	[GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12,
1511 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13,
1512 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13,
1513 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13,
1514 	[GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13,
1515 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14,
1516 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14,
1517 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14,
1518 	[GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14,
1519 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15,
1520 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15,
1521 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15,
1522 	[GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15,
1523 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16,
1524 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16,
1525 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16,
1526 	[GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16,
1527 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17,
1528 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17,
1529 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17,
1530 	[GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17,
1531 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6,
1532 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6,
1533 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6,
1534 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6,
1535 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7,
1536 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7,
1537 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7,
1538 	[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7,
1539 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5,
1540 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5,
1541 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5,
1542 	[GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5,
1543 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18,
1544 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18,
1545 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18,
1546 	[GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18,
1547 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19,
1548 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19,
1549 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19,
1550 	[GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19,
1551 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20,
1552 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20,
1553 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20,
1554 	[GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20,
1555 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21,
1556 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21,
1557 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21,
1558 	[GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21,
1559 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22,
1560 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22,
1561 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22,
1562 	[GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22,
1563 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23,
1564 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23,
1565 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23,
1566 	[GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23,
1567 	[GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0,
1568 	[GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0,
1569 	[GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0,
1570 	[GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0,
1571 	[GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1,
1572 	[GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1,
1573 	[GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1,
1574 	[GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1,
1575 	[GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2,
1576 	[GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2,
1577 	[GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2,
1578 	[GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2,
1579 	[GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3,
1580 	[GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3,
1581 	[GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3,
1582 	[GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3,
1583 	[GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4,
1584 	[GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4,
1585 	[GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4,
1586 	[GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4,
1587 	[GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5,
1588 	[GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5,
1589 	[GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5,
1590 	[GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5,
1591 	[GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6,
1592 	[GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6,
1593 	[GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6,
1594 	[GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6,
1595 	[GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7,
1596 	[GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7,
1597 	[GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7,
1598 	[GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7,
1599 	[GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8,
1600 	[GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8,
1601 	[GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8,
1602 	[GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8,
1603 	[GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9,
1604 	[GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9,
1605 	[GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9,
1606 	[GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9,
1607 	[GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10,
1608 	[GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10,
1609 	[GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10,
1610 	[GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10,
1611 	[GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11,
1612 	[GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11,
1613 	[GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11,
1614 	[GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11,
1615 	[GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12,
1616 	[GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12,
1617 	[GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12,
1618 	[GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12,
1619 	[GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13,
1620 	[GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13,
1621 	[GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13,
1622 	[GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13,
1623 	[GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14,
1624 	[GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14,
1625 	[GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14,
1626 	[GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14,
1627 	[GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15,
1628 	[GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15,
1629 	[GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15,
1630 	[GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15,
1631 	[GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16,
1632 	[GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16,
1633 	[GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16,
1634 	[GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16,
1635 	[GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17,
1636 	[GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17,
1637 	[GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17,
1638 	[GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17,
1639 	[GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18,
1640 	[GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18,
1641 	[GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18,
1642 	[GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18,
1643 	[GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19,
1644 	[GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19,
1645 	[GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19,
1646 	[GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19,
1647 	[GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20,
1648 	[GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20,
1649 	[GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20,
1650 	[GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20,
1651 	[GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21,
1652 	[GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21,
1653 	[GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21,
1654 	[GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21,
1655 	[GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22,
1656 	[GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22,
1657 	[GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22,
1658 	[GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22,
1659 	[GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23,
1660 	[GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23,
1661 	[GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23,
1662 	[GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23,
1663 	[GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0,
1664 	[GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0,
1665 	[GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0,
1666 	[GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0,
1667 	[GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1,
1668 	[GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1,
1669 	[GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1,
1670 	[GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1
1671 };
1672 
1673 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = {
1674 	[DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE,
1675 	[DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE,
1676 	[DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE,
1677 	[DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE,
1678 	[DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE,
1679 	[DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE,
1680 	[DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE,
1681 	[DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE,
1682 	[DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE,
1683 	[DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE,
1684 	[DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE
1685 };
1686 
1687 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = {
1688 	[MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE,
1689 	[MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE,
1690 	[MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE,
1691 	[MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE
1692 };
1693 
1694 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = {
1695 	[TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE,
1696 	[TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE,
1697 	[TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE,
1698 	[TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE,
1699 	[TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE,
1700 	[TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE,
1701 	[TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE,
1702 	[TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE,
1703 	[TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE,
1704 	[TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE,
1705 	[TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE,
1706 	[TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE,
1707 	[TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE,
1708 	[TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE,
1709 	[TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE,
1710 	[TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE,
1711 	[TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE,
1712 	[TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE,
1713 	[TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE,
1714 	[TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE,
1715 	[TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE,
1716 	[TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE,
1717 	[TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE,
1718 	[TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE,
1719 	[TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE,
1720 };
1721 
1722 static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = {
1723 	[TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE,
1724 	[TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE,
1725 	[TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE,
1726 	[TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE,
1727 	[TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE,
1728 	[TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE,
1729 	[TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE,
1730 	[TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE,
1731 	[TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE,
1732 	[TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE,
1733 	[TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE,
1734 	[TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE,
1735 	[TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE,
1736 	[TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE,
1737 	[TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE,
1738 	[TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE,
1739 	[TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE,
1740 	[TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE,
1741 	[TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE,
1742 	[TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE,
1743 	[TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE,
1744 	[TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE,
1745 	[TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE,
1746 	[TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE,
1747 	[TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE,
1748 };
1749 
1750 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = {
1751 	[ROTATOR_ID_0] = mmROT0_BASE,
1752 	[ROTATOR_ID_1] = mmROT1_BASE
1753 };
1754 
1755 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = {
1756 	[TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0,
1757 	[TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0,
1758 	[TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0,
1759 	[TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0,
1760 	[TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0,
1761 	[TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0,
1762 	[TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0,
1763 	[TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0,
1764 	[TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0,
1765 	[TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0,
1766 	[TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0,
1767 	[TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0,
1768 	[TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0,
1769 	[TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0,
1770 	[TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0,
1771 	[TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0,
1772 	[TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0,
1773 	[TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0,
1774 	[TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0,
1775 	[TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0,
1776 	[TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0,
1777 	[TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0,
1778 	[TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0,
1779 	[TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0,
1780 	[TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0,
1781 };
1782 
1783 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = {
1784 	[ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0,
1785 	[ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0,
1786 };
1787 
1788 static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = {
1789 	[GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0,
1790 	[GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1,
1791 	[GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2,
1792 	[GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3,
1793 	[GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4,
1794 	[GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5,
1795 	[GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0,
1796 	[GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1,
1797 	[GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2,
1798 	[GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3,
1799 	[GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4,
1800 	[GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5,
1801 	[GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0,
1802 	[GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1,
1803 	[GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2,
1804 	[GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3,
1805 	[GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4,
1806 	[GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5,
1807 	[GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0,
1808 	[GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1,
1809 	[GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2,
1810 	[GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3,
1811 	[GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4,
1812 	[GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5,
1813 	/* the PCI TPC is placed last (mapped liked HW) */
1814 	[GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6,
1815 };
1816 
1817 static const u32 gaudi2_mme_engine_id_to_mme_id[] = {
1818 	[GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0,
1819 	[GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1,
1820 	[GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2,
1821 	[GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3,
1822 };
1823 
1824 static const u32 gaudi2_edma_engine_id_to_edma_id[] = {
1825 	[GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0,
1826 	[GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1,
1827 	[GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0,
1828 	[GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1,
1829 	[GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2,
1830 	[GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3,
1831 	[GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4,
1832 	[GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5,
1833 	[GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6,
1834 	[GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7,
1835 	[GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA,
1836 };
1837 
1838 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
1839 	GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
1840 	GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0,
1841 	GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0,
1842 	GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0,
1843 	GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0,
1844 	GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0,
1845 	GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0,
1846 	GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0,
1847 };
1848 
1849 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = {
1850 	"gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal",
1851 	"gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal",
1852 	"gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal",
1853 	"gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal",
1854 	"gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal",
1855 	"gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal",
1856 	"gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal",
1857 	"gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal",
1858 	"gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal",
1859 	"gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal"
1860 };
1861 
1862 enum rtr_id {
1863 	DCORE0_RTR0,
1864 	DCORE0_RTR1,
1865 	DCORE0_RTR2,
1866 	DCORE0_RTR3,
1867 	DCORE0_RTR4,
1868 	DCORE0_RTR5,
1869 	DCORE0_RTR6,
1870 	DCORE0_RTR7,
1871 	DCORE1_RTR0,
1872 	DCORE1_RTR1,
1873 	DCORE1_RTR2,
1874 	DCORE1_RTR3,
1875 	DCORE1_RTR4,
1876 	DCORE1_RTR5,
1877 	DCORE1_RTR6,
1878 	DCORE1_RTR7,
1879 	DCORE2_RTR0,
1880 	DCORE2_RTR1,
1881 	DCORE2_RTR2,
1882 	DCORE2_RTR3,
1883 	DCORE2_RTR4,
1884 	DCORE2_RTR5,
1885 	DCORE2_RTR6,
1886 	DCORE2_RTR7,
1887 	DCORE3_RTR0,
1888 	DCORE3_RTR1,
1889 	DCORE3_RTR2,
1890 	DCORE3_RTR3,
1891 	DCORE3_RTR4,
1892 	DCORE3_RTR5,
1893 	DCORE3_RTR6,
1894 	DCORE3_RTR7,
1895 };
1896 
1897 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
1898 	DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3,
1899 	DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4,
1900 	DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1,
1901 	DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6,
1902 	DCORE0_RTR0
1903 };
1904 
1905 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
1906 	DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2,
1907 	DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5,
1908 	DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0,
1909 	DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7,
1910 	DCORE0_RTR0
1911 };
1912 
1913 static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = {
1914 	DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0,
1915 	DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0
1916 };
1917 
1918 static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = {
1919 	DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1,
1920 	DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0
1921 };
1922 
1923 static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = {
1924 	DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0,
1925 	DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7
1926 };
1927 
1928 static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = {
1929 	DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0,
1930 	DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7
1931 };
1932 
1933 static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
1934 	mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE,
1935 	mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1936 	mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE,
1937 	mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1938 	mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1939 	mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE,
1940 	mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE,
1941 	mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE
1942 };
1943 
1944 static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = {
1945 	DCORE0_RTR0, DCORE0_RTR0
1946 };
1947 
1948 static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = {
1949 	DCORE0_RTR2, DCORE0_RTR2
1950 };
1951 
1952 static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = {
1953 	DCORE2_RTR0, DCORE3_RTR7
1954 };
1955 
1956 static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = {
1957 	DCORE2_RTR2, DCORE3_RTR5
1958 };
1959 
1960 struct mme_initiators_rtr_id {
1961 	u32 wap0;
1962 	u32 wap1;
1963 	u32 write;
1964 	u32 read;
1965 	u32 sbte0;
1966 	u32 sbte1;
1967 	u32 sbte2;
1968 	u32 sbte3;
1969 	u32 sbte4;
1970 };
1971 
1972 enum mme_initiators {
1973 	MME_WAP0 = 0,
1974 	MME_WAP1,
1975 	MME_WRITE,
1976 	MME_READ,
1977 	MME_SBTE0,
1978 	MME_SBTE1,
1979 	MME_SBTE2,
1980 	MME_SBTE3,
1981 	MME_SBTE4,
1982 	MME_INITIATORS_MAX
1983 };
1984 
1985 static const struct mme_initiators_rtr_id
1986 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = {
1987 	{ .wap0 = 5, .wap1 = 7, .write = 6, .read = 7,
1988 	.sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6},
1989 	{ .wap0 = 10, .wap1 = 8, .write = 9, .read = 8,
1990 	.sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8},
1991 	{ .wap0 = 21, .wap1 = 23, .write = 22, .read = 23,
1992 	.sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23},
1993 	{ .wap0 = 30, .wap1 = 28, .write = 29, .read = 30,
1994 	.sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28},
1995 };
1996 
1997 enum razwi_event_sources {
1998 	RAZWI_TPC,
1999 	RAZWI_MME,
2000 	RAZWI_EDMA,
2001 	RAZWI_PDMA,
2002 	RAZWI_NIC,
2003 	RAZWI_DEC,
2004 	RAZWI_ROT
2005 };
2006 
2007 struct hbm_mc_error_causes {
2008 	u32 mask;
2009 	char cause[50];
2010 };
2011 
2012 static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS;
2013 
2014 /* Special blocks iterator is currently used to configure security protection bits,
2015  * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2016  * must be skipped. Following configurations are commonly used for both PB config
2017  * and global error reading, since currently they both share the same settings.
2018  * Once it changes, we must remember to use separate configurations for either one.
2019  */
2020 static int gaudi2_iterator_skip_block_types[] = {
2021 		GAUDI2_BLOCK_TYPE_PLL,
2022 		GAUDI2_BLOCK_TYPE_EU_BIST,
2023 		GAUDI2_BLOCK_TYPE_HBM,
2024 		GAUDI2_BLOCK_TYPE_XFT
2025 };
2026 
2027 static struct range gaudi2_iterator_skip_block_ranges[] = {
2028 		/* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */
2029 		{mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE},
2030 		{mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE},
2031 		/* Skip all CPU blocks except for CPU_IF */
2032 		{mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE},
2033 		{mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE}
2034 };
2035 
2036 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = {
2037 	{HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"},
2038 	{HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2039 	{HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2040 	{HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"},
2041 	{HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"},
2042 };
2043 
2044 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = {
2045 	[HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even",
2046 	[HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd",
2047 	[HBM_SEI_READ_ERR] = "SEI read data error",
2048 	[HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error",
2049 	[HBM_SEI_CATTRIP] = "SEI CATTRIP asserted",
2050 	[HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail",
2051 	[HBM_SEI_DFI] = "SEI DFI error",
2052 	[HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read",
2053 	[HBM_SEI_BIST_FAIL] = "SEI BIST fail"
2054 };
2055 
2056 struct mmu_spi_sei_cause {
2057 	char cause[50];
2058 	int clear_bit;
2059 };
2060 
2061 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = {
2062 	{"page fault", 1},		/* INTERRUPT_CLR[1] */
2063 	{"page access", 1},		/* INTERRUPT_CLR[1] */
2064 	{"bypass ddr", 2},		/* INTERRUPT_CLR[2] */
2065 	{"multi hit", 2},		/* INTERRUPT_CLR[2] */
2066 	{"mmu rei0", -1},		/* no clear register bit */
2067 	{"mmu rei1", -1},		/* no clear register bit */
2068 	{"stlb rei0", -1},		/* no clear register bit */
2069 	{"stlb rei1", -1},		/* no clear register bit */
2070 	{"rr privileged write hit", 2},	/* INTERRUPT_CLR[2] */
2071 	{"rr privileged read hit", 2},	/* INTERRUPT_CLR[2] */
2072 	{"rr secure write hit", 2},	/* INTERRUPT_CLR[2] */
2073 	{"rr secure read hit", 2},	/* INTERRUPT_CLR[2] */
2074 	{"bist_fail no use", 2},	/* INTERRUPT_CLR[2] */
2075 	{"bist_fail no use", 2},	/* INTERRUPT_CLR[2] */
2076 	{"bist_fail no use", 2},	/* INTERRUPT_CLR[2] */
2077 	{"bist_fail no use", 2},	/* INTERRUPT_CLR[2] */
2078 	{"slave error", 16},		/* INTERRUPT_CLR[16] */
2079 	{"dec error", 17},		/* INTERRUPT_CLR[17] */
2080 	{"burst fifo full", 2}		/* INTERRUPT_CLR[2] */
2081 };
2082 
2083 struct gaudi2_cache_invld_params {
2084 	u64 start_va;
2085 	u64 end_va;
2086 	u32 inv_start_val;
2087 	u32 flags;
2088 	bool range_invalidation;
2089 };
2090 
2091 struct gaudi2_tpc_idle_data {
2092 	struct engines_data *e;
2093 	unsigned long *mask;
2094 	bool *is_idle;
2095 	const char *tpc_fmt;
2096 };
2097 
2098 struct gaudi2_tpc_mmu_data {
2099 	u32 rw_asid;
2100 };
2101 
2102 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0};
2103 
2104 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val);
2105 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id);
2106 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id);
2107 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2108 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2109 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val);
2110 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size,
2111 										bool is_memset);
2112 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2113 		struct engines_data *e);
2114 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2115 		struct engines_data *e);
2116 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2117 		struct engines_data *e);
2118 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr);
2119 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr);
2120 
2121 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev)
2122 {
2123 
2124 }
2125 
2126 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev)
2127 {
2128 	return sizeof(struct packet_msg_short);
2129 }
2130 
2131 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev)
2132 {
2133 	return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence);
2134 }
2135 
2136 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx)
2137 {
2138 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2139 	int dcore, inst, tpc_seq;
2140 	u32 offset;
2141 
2142 	/* init the return code */
2143 	ctx->rc = 0;
2144 
2145 	for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) {
2146 		for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) {
2147 			tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst;
2148 
2149 			if (!(prop->tpc_enabled_mask & BIT(tpc_seq)))
2150 				continue;
2151 
2152 			offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst);
2153 
2154 			ctx->fn(hdev, dcore, inst, offset, ctx);
2155 			if (ctx->rc) {
2156 				dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n",
2157 							dcore, inst);
2158 				return;
2159 			}
2160 		}
2161 	}
2162 
2163 	if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6)))
2164 		return;
2165 
2166 	/* special check for PCI TPC (DCORE0_TPC6) */
2167 	offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1);
2168 	ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx);
2169 	if (ctx->rc)
2170 		dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n");
2171 }
2172 
2173 static bool gaudi2_host_phys_addr_valid(u64 addr)
2174 {
2175 	if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1))
2176 		return true;
2177 
2178 	return false;
2179 }
2180 
2181 static int set_number_of_functional_hbms(struct hl_device *hdev)
2182 {
2183 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2184 	u8 faulty_hbms = hweight64(hdev->dram_binning);
2185 
2186 	/* check if all HBMs should be used */
2187 	if (!faulty_hbms) {
2188 		dev_dbg(hdev->dev, "All HBM are in use (no binning)\n");
2189 		prop->num_functional_hbms = GAUDI2_HBM_NUM;
2190 		return 0;
2191 	}
2192 
2193 	/*
2194 	 * check for error condition in which number of binning
2195 	 * candidates is higher than the maximum supported by the
2196 	 * driver (in which case binning mask shall be ignored and driver will
2197 	 * set the default)
2198 	 */
2199 	if (faulty_hbms > MAX_FAULTY_HBMS) {
2200 		dev_err(hdev->dev,
2201 			"HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n",
2202 			MAX_FAULTY_HBMS, hdev->dram_binning);
2203 		return -EINVAL;
2204 	}
2205 
2206 	/*
2207 	 * by default, number of functional HBMs in Gaudi2 is always
2208 	 * GAUDI2_HBM_NUM - 1.
2209 	 */
2210 	prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms;
2211 	return 0;
2212 }
2213 
2214 static int gaudi2_set_dram_properties(struct hl_device *hdev)
2215 {
2216 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2217 	u32 basic_hbm_page_size;
2218 	int rc;
2219 
2220 	rc = set_number_of_functional_hbms(hdev);
2221 	if (rc)
2222 		return -EINVAL;
2223 
2224 	/*
2225 	 * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround
2226 	 * in which we are using x16 bigger page size to be able to populate the entire
2227 	 * HBM mappings in the TLB
2228 	 */
2229 	basic_hbm_page_size = prop->num_functional_hbms * SZ_8M;
2230 	prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size;
2231 	prop->device_mem_alloc_default_page_size = prop->dram_page_size;
2232 	prop->dram_size = prop->num_functional_hbms * SZ_16G;
2233 	prop->dram_base_address = DRAM_PHYS_BASE;
2234 	prop->dram_end_address = prop->dram_base_address + prop->dram_size;
2235 	prop->dram_supports_virtual_memory = true;
2236 
2237 	prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size;
2238 	prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK;
2239 	prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START;
2240 	prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END;
2241 
2242 	/* since DRAM page size differs from DMMU page size we need to allocate
2243 	 * DRAM memory in units of dram_page size and mapping this memory in
2244 	 * units of DMMU page size. we overcome this size mismatch using a
2245 	 * scrambling routine which takes a DRAM page and converts it to a DMMU
2246 	 * page.
2247 	 * We therefore:
2248 	 * 1. partition the virtual address space to DRAM-page (whole) pages.
2249 	 *    (suppose we get n such pages)
2250 	 * 2. limit the amount of virtual address space we got from 1 above to
2251 	 *    a multiple of 64M as we don't want the scrambled address to cross
2252 	 *    the DRAM virtual address space.
2253 	 *    ( m = (n * DRAM_page_size) / DMMU_page_size).
2254 	 * 3. determine the and address accordingly
2255 	 *    end_addr = start_addr + m * 48M
2256 	 *
2257 	 *    the DRAM address MSBs (63:48) are not part of the roundup calculation
2258 	 */
2259 	prop->dmmu.start_addr = prop->dram_base_address +
2260 			(prop->dram_page_size *
2261 				DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size));
2262 
2263 	prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size *
2264 			div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size);
2265 
2266 	return 0;
2267 }
2268 
2269 static int gaudi2_set_fixed_properties(struct hl_device *hdev)
2270 {
2271 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2272 	struct hw_queue_properties *q_props;
2273 	u32 num_sync_stream_queues = 0;
2274 	int i;
2275 
2276 	prop->max_queues = GAUDI2_QUEUE_ID_SIZE;
2277 	prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties),
2278 					GFP_KERNEL);
2279 
2280 	if (!prop->hw_queues_props)
2281 		return -ENOMEM;
2282 
2283 	q_props = prop->hw_queues_props;
2284 
2285 	for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) {
2286 		q_props[i].type = QUEUE_TYPE_HW;
2287 		q_props[i].driver_only = 0;
2288 
2289 		if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) {
2290 			q_props[i].supports_sync_stream = 0;
2291 		} else {
2292 			q_props[i].supports_sync_stream = 1;
2293 			num_sync_stream_queues++;
2294 		}
2295 
2296 		q_props[i].cb_alloc_flags = CB_ALLOC_USER;
2297 	}
2298 
2299 	q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU;
2300 	q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1;
2301 	q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL;
2302 
2303 	prop->cache_line_size = DEVICE_CACHE_LINE_SIZE;
2304 	prop->cfg_base_address = CFG_BASE;
2305 	prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0;
2306 	prop->host_base_address = HOST_PHYS_BASE_0;
2307 	prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0;
2308 	prop->max_pending_cs = GAUDI2_MAX_PENDING_CS;
2309 	prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER;
2310 	prop->user_dec_intr_count = NUMBER_OF_DEC;
2311 	prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1;
2312 	prop->completion_mode = HL_COMPLETION_MODE_CS;
2313 	prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER;
2314 	prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER;
2315 
2316 	prop->sram_base_address = SRAM_BASE_ADDR;
2317 	prop->sram_size = SRAM_SIZE;
2318 	prop->sram_end_address = prop->sram_base_address + prop->sram_size;
2319 	prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET;
2320 
2321 	prop->hints_range_reservation = true;
2322 
2323 	prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1;
2324 
2325 	if (hdev->pldm)
2326 		prop->mmu_pgt_size = 0x800000; /* 8MB */
2327 	else
2328 		prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE;
2329 
2330 	prop->mmu_pte_size = HL_PTE_SIZE;
2331 	prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE;
2332 	prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE;
2333 
2334 	prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT;
2335 	prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT;
2336 	prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT;
2337 	prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT;
2338 	prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT;
2339 	prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK;
2340 	prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK;
2341 	prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK;
2342 	prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK;
2343 	prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK;
2344 	prop->dmmu.page_size = PAGE_SIZE_1GB;
2345 	prop->dmmu.num_hops = MMU_ARCH_6_HOPS;
2346 	prop->dmmu.last_mask = LAST_MASK;
2347 	prop->dmmu.host_resident = 1;
2348 	prop->dmmu.hop_table_size = prop->mmu_hop_table_size;
2349 	prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
2350 
2351 	/*
2352 	 * this is done in order to be able to validate FW descriptor (i.e. validating that
2353 	 * the addresses and allocated space for FW image does not cross memory bounds).
2354 	 * for this reason we set the DRAM size to the minimum possible and later it will
2355 	 * be modified according to what reported in the cpucp info packet
2356 	 */
2357 	prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G;
2358 
2359 	hdev->pmmu_huge_range = true;
2360 	prop->pmmu.host_resident = 1;
2361 	prop->pmmu.num_hops = MMU_ARCH_6_HOPS;
2362 	prop->pmmu.last_mask = LAST_MASK;
2363 	prop->pmmu.hop_table_size = prop->mmu_hop_table_size;
2364 	prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
2365 
2366 	prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START;
2367 	prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END;
2368 	prop->hints_host_hpage_reserved_va_range.start_addr =
2369 			RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START;
2370 	prop->hints_host_hpage_reserved_va_range.end_addr =
2371 			RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END;
2372 
2373 	if (PAGE_SIZE == SZ_64K) {
2374 		prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K;
2375 		prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K;
2376 		prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K;
2377 		prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K;
2378 		prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K;
2379 		prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K;
2380 		prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K;
2381 		prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K;
2382 		prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K;
2383 		prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K;
2384 		prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K;
2385 		prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K;
2386 		prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;
2387 		prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;
2388 		prop->pmmu.page_size = PAGE_SIZE_64KB;
2389 
2390 		/* shifts and masks are the same in PMMU and HPMMU */
2391 		memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
2392 		prop->pmmu_huge.page_size = PAGE_SIZE_16MB;
2393 		prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;
2394 		prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
2395 	} else {
2396 		prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K;
2397 		prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K;
2398 		prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K;
2399 		prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K;
2400 		prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K;
2401 		prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K;
2402 		prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K;
2403 		prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K;
2404 		prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K;
2405 		prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K;
2406 		prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K;
2407 		prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K;
2408 		prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;
2409 		prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;
2410 		prop->pmmu.page_size = PAGE_SIZE_4KB;
2411 
2412 		/* shifts and masks are the same in PMMU and HPMMU */
2413 		memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
2414 		prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
2415 		prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;
2416 		prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
2417 	}
2418 
2419 	prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE;
2420 	prop->num_engine_cores = CPU_ID_MAX;
2421 	prop->cfg_size = CFG_SIZE;
2422 	prop->max_asid = MAX_ASID;
2423 	prop->num_of_events = GAUDI2_EVENT_SIZE;
2424 
2425 	prop->supports_engine_modes = true;
2426 
2427 	prop->dc_power_default = DC_POWER_DEFAULT;
2428 
2429 	prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT;
2430 	prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE;
2431 	prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE;
2432 	prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
2433 
2434 	strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
2435 
2436 	prop->mme_master_slave_mode = 1;
2437 
2438 	prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER +
2439 					(num_sync_stream_queues * HL_RSVD_SOBS);
2440 
2441 	prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER +
2442 					(num_sync_stream_queues * HL_RSVD_MONS);
2443 
2444 	prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST;
2445 	prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT;
2446 	prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE;
2447 
2448 	prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER;
2449 
2450 	prop->fw_cpu_boot_dev_sts0_valid = false;
2451 	prop->fw_cpu_boot_dev_sts1_valid = false;
2452 	prop->hard_reset_done_by_fw = false;
2453 	prop->gic_interrupts_enable = true;
2454 
2455 	prop->server_type = HL_SERVER_TYPE_UNKNOWN;
2456 
2457 	prop->max_dec = NUMBER_OF_DEC;
2458 
2459 	prop->clk_pll_index = HL_GAUDI2_MME_PLL;
2460 
2461 	prop->dma_mask = 64;
2462 
2463 	prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0;
2464 
2465 	return 0;
2466 }
2467 
2468 static int gaudi2_pci_bars_map(struct hl_device *hdev)
2469 {
2470 	static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"};
2471 	bool is_wc[3] = {false, false, true};
2472 	int rc;
2473 
2474 	rc = hl_pci_bars_map(hdev, name, is_wc);
2475 	if (rc)
2476 		return rc;
2477 
2478 	hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR);
2479 
2480 	return 0;
2481 }
2482 
2483 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
2484 {
2485 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
2486 	struct hl_inbound_pci_region pci_region;
2487 	u64 old_addr = addr;
2488 	int rc;
2489 
2490 	if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr))
2491 		return old_addr;
2492 
2493 	if (hdev->asic_prop.iatu_done_by_fw)
2494 		return U64_MAX;
2495 
2496 	/* Inbound Region 2 - Bar 4 - Point to DRAM */
2497 	pci_region.mode = PCI_BAR_MATCH_MODE;
2498 	pci_region.bar = DRAM_BAR_ID;
2499 	pci_region.addr = addr;
2500 	rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
2501 	if (rc)
2502 		return U64_MAX;
2503 
2504 	if (gaudi2) {
2505 		old_addr = gaudi2->dram_bar_cur_addr;
2506 		gaudi2->dram_bar_cur_addr = addr;
2507 	}
2508 
2509 	return old_addr;
2510 }
2511 
2512 static int gaudi2_init_iatu(struct hl_device *hdev)
2513 {
2514 	struct hl_inbound_pci_region inbound_region;
2515 	struct hl_outbound_pci_region outbound_region;
2516 	u32 bar_addr_low, bar_addr_high;
2517 	int rc;
2518 
2519 	if (hdev->asic_prop.iatu_done_by_fw)
2520 		return 0;
2521 
2522 	/* Temporary inbound Region 0 - Bar 0 - Point to CFG
2523 	 * We must map this region in BAR match mode in order to
2524 	 * fetch BAR physical base address
2525 	 */
2526 	inbound_region.mode = PCI_BAR_MATCH_MODE;
2527 	inbound_region.bar = SRAM_CFG_BAR_ID;
2528 	/* Base address must be aligned to Bar size which is 256 MB */
2529 	inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF;
2530 	rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2531 	if (rc)
2532 		return rc;
2533 
2534 	/* Fetch physical BAR address */
2535 	bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF);
2536 	bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF;
2537 
2538 	hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low;
2539 
2540 	/* Inbound Region 0 - Bar 0 - Point to CFG */
2541 	inbound_region.mode = PCI_ADDRESS_MATCH_MODE;
2542 	inbound_region.bar = SRAM_CFG_BAR_ID;
2543 	inbound_region.offset_in_bar = 0;
2544 	inbound_region.addr = STM_FLASH_BASE_ADDR;
2545 	inbound_region.size = CFG_REGION_SIZE;
2546 	rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2547 	if (rc)
2548 		return rc;
2549 
2550 	/* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */
2551 	inbound_region.mode = PCI_ADDRESS_MATCH_MODE;
2552 	inbound_region.bar = SRAM_CFG_BAR_ID;
2553 	inbound_region.offset_in_bar = CFG_REGION_SIZE;
2554 	inbound_region.addr = BAR0_RSRVD_BASE_ADDR;
2555 	inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE;
2556 	rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
2557 	if (rc)
2558 		return rc;
2559 
2560 	/* Inbound Region 2 - Bar 4 - Point to DRAM */
2561 	inbound_region.mode = PCI_BAR_MATCH_MODE;
2562 	inbound_region.bar = DRAM_BAR_ID;
2563 	inbound_region.addr = DRAM_PHYS_BASE;
2564 	rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
2565 	if (rc)
2566 		return rc;
2567 
2568 	/* Outbound Region 0 - Point to Host */
2569 	outbound_region.addr = HOST_PHYS_BASE_0;
2570 	outbound_region.size = HOST_PHYS_SIZE_0;
2571 	rc = hl_pci_set_outbound_region(hdev, &outbound_region);
2572 
2573 	return rc;
2574 }
2575 
2576 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev)
2577 {
2578 	return RREG32(mmHW_STATE);
2579 }
2580 
2581 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev)
2582 {
2583 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2584 
2585 	/*
2586 	 * check for error condition in which number of binning candidates
2587 	 * is higher than the maximum supported by the driver
2588 	 */
2589 	if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) {
2590 		dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n",
2591 					MAX_CLUSTER_BINNING_FAULTY_TPCS,
2592 					hdev->tpc_binning);
2593 		return -EINVAL;
2594 	}
2595 
2596 	prop->tpc_binning_mask = hdev->tpc_binning;
2597 	prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK;
2598 
2599 	return 0;
2600 }
2601 
2602 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev)
2603 {
2604 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2605 	struct hw_queue_properties *q_props = prop->hw_queues_props;
2606 	u64 tpc_binning_mask;
2607 	u8 subst_idx = 0;
2608 	int i, rc;
2609 
2610 	rc = gaudi2_tpc_binning_init_prop(hdev);
2611 	if (rc)
2612 		return rc;
2613 
2614 	tpc_binning_mask = prop->tpc_binning_mask;
2615 
2616 	for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) {
2617 		u8 subst_seq, binned, qid_base;
2618 
2619 		if (tpc_binning_mask == 0)
2620 			break;
2621 
2622 		if (subst_idx == 0) {
2623 			subst_seq = TPC_ID_DCORE0_TPC6;
2624 			qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0;
2625 		} else {
2626 			subst_seq = TPC_ID_DCORE3_TPC5;
2627 			qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0;
2628 		}
2629 
2630 
2631 		/* clear bit from mask */
2632 		binned = __ffs(tpc_binning_mask);
2633 		/*
2634 		 * Coverity complains about possible out-of-bound access in
2635 		 * clear_bit
2636 		 */
2637 		if (binned >= TPC_ID_SIZE) {
2638 			dev_err(hdev->dev,
2639 				"Invalid binned TPC (binning mask: %llx)\n",
2640 				tpc_binning_mask);
2641 			return -EINVAL;
2642 		}
2643 		clear_bit(binned, (unsigned long *)&tpc_binning_mask);
2644 
2645 		/* also clear replacing TPC bit from enabled mask */
2646 		clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask);
2647 
2648 		/* bin substite TPC's Qs */
2649 		q_props[qid_base].binned = 1;
2650 		q_props[qid_base + 1].binned = 1;
2651 		q_props[qid_base + 2].binned = 1;
2652 		q_props[qid_base + 3].binned = 1;
2653 
2654 		subst_idx++;
2655 	}
2656 
2657 	return 0;
2658 }
2659 
2660 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev)
2661 {
2662 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2663 	u8 num_faulty;
2664 
2665 	num_faulty = hweight32(hdev->decoder_binning);
2666 
2667 	/*
2668 	 * check for error condition in which number of binning candidates
2669 	 * is higher than the maximum supported by the driver
2670 	 */
2671 	if (num_faulty > MAX_FAULTY_DECODERS) {
2672 		dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n",
2673 						hdev->decoder_binning);
2674 		return -EINVAL;
2675 	}
2676 
2677 	prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK);
2678 
2679 	if (prop->decoder_binning_mask)
2680 		prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1));
2681 	else
2682 		prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK;
2683 
2684 	return 0;
2685 }
2686 
2687 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev)
2688 {
2689 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2690 
2691 	/* check if we should override default binning */
2692 	if (!hdev->dram_binning) {
2693 		prop->dram_binning_mask = 0;
2694 		prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK;
2695 		return;
2696 	}
2697 
2698 	/* set DRAM binning constraints */
2699 	prop->faulty_dram_cluster_map |= hdev->dram_binning;
2700 	prop->dram_binning_mask = hdev->dram_binning;
2701 	prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5);
2702 }
2703 
2704 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev)
2705 {
2706 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2707 	struct hw_queue_properties *q_props;
2708 	u8 seq, num_faulty;
2709 
2710 	num_faulty = hweight32(hdev->edma_binning);
2711 
2712 	/*
2713 	 * check for error condition in which number of binning candidates
2714 	 * is higher than the maximum supported by the driver
2715 	 */
2716 	if (num_faulty > MAX_FAULTY_EDMAS) {
2717 		dev_err(hdev->dev,
2718 			"EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n",
2719 			hdev->edma_binning);
2720 		return -EINVAL;
2721 	}
2722 
2723 	if (!hdev->edma_binning) {
2724 		prop->edma_binning_mask = 0;
2725 		prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK;
2726 		return 0;
2727 	}
2728 
2729 	seq = __ffs((unsigned long)hdev->edma_binning);
2730 
2731 	/* set binning constraints */
2732 	prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]);
2733 	prop->edma_binning_mask = hdev->edma_binning;
2734 	prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1);
2735 
2736 	/* bin substitute EDMA's queue */
2737 	q_props = prop->hw_queues_props;
2738 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1;
2739 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1;
2740 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1;
2741 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1;
2742 
2743 	return 0;
2744 }
2745 
2746 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask)
2747 {
2748 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2749 	u8 num_faulty, seq;
2750 
2751 	/* check if we should override default binning */
2752 	if (!xbar_edge_iso_mask) {
2753 		prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK;
2754 		return 0;
2755 	}
2756 
2757 	/*
2758 	 * note that it can be set to value other than 0 only after cpucp packet (i.e.
2759 	 * only the FW can set a redundancy value). for user it'll always be 0.
2760 	 */
2761 	num_faulty = hweight32(xbar_edge_iso_mask);
2762 
2763 	/*
2764 	 * check for error condition in which number of binning candidates
2765 	 * is higher than the maximum supported by the driver
2766 	 */
2767 	if (num_faulty > MAX_FAULTY_XBARS) {
2768 		dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n",
2769 									MAX_FAULTY_XBARS);
2770 		return -EINVAL;
2771 	}
2772 
2773 	seq = __ffs((unsigned long)xbar_edge_iso_mask);
2774 
2775 	/* set binning constraints */
2776 	prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]);
2777 	prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK;
2778 
2779 	return 0;
2780 }
2781 
2782 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask)
2783 {
2784 	int rc;
2785 
2786 	/*
2787 	 * mark all clusters as good, each component will "fail" cluster
2788 	 * based on eFuse/user values.
2789 	 * If more than single cluster is faulty- the chip is unusable
2790 	 */
2791 	hdev->asic_prop.faulty_dram_cluster_map = 0;
2792 
2793 	gaudi2_set_dram_binning_masks(hdev);
2794 
2795 	rc = gaudi2_set_edma_binning_masks(hdev);
2796 	if (rc)
2797 		return rc;
2798 
2799 	rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask);
2800 	if (rc)
2801 		return rc;
2802 
2803 
2804 	/* always initially set to full mask */
2805 	hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK;
2806 
2807 	return 0;
2808 }
2809 
2810 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev)
2811 {
2812 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2813 	int rc;
2814 
2815 	rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask);
2816 	if (rc)
2817 		return rc;
2818 
2819 	/* if we have DRAM binning reported by FW we should perform cluster config  */
2820 	if (prop->faulty_dram_cluster_map) {
2821 		u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map);
2822 
2823 		prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq];
2824 	}
2825 
2826 	return 0;
2827 }
2828 
2829 static int gaudi2_set_binning_masks(struct hl_device *hdev)
2830 {
2831 	int rc;
2832 
2833 	rc = gaudi2_set_cluster_binning_masks(hdev);
2834 	if (rc)
2835 		return rc;
2836 
2837 	rc = gaudi2_set_tpc_binning_masks(hdev);
2838 	if (rc)
2839 		return rc;
2840 
2841 	rc = gaudi2_set_dec_binning_masks(hdev);
2842 	if (rc)
2843 		return rc;
2844 
2845 	return 0;
2846 }
2847 
2848 static int gaudi2_cpucp_info_get(struct hl_device *hdev)
2849 {
2850 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
2851 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2852 	long max_power;
2853 	u64 dram_size;
2854 	int rc;
2855 
2856 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
2857 		return 0;
2858 
2859 	/* No point of asking this information again when not doing hard reset, as the device
2860 	 * CPU hasn't been reset
2861 	 */
2862 	if (hdev->reset_info.in_compute_reset)
2863 		return 0;
2864 
2865 	rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
2866 										mmCPU_BOOT_ERR1);
2867 	if (rc)
2868 		return rc;
2869 
2870 	dram_size = le64_to_cpu(prop->cpucp_info.dram_size);
2871 	if (dram_size) {
2872 		/* we can have wither 5 or 6 HBMs. other values are invalid */
2873 
2874 		if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) &&
2875 					(dram_size != (GAUDI2_HBM_NUM * SZ_16G))) {
2876 			dev_err(hdev->dev,
2877 				"F/W reported invalid DRAM size %llu. Trying to use default size %llu\n",
2878 				dram_size, prop->dram_size);
2879 			dram_size = prop->dram_size;
2880 		}
2881 
2882 		prop->dram_size = dram_size;
2883 		prop->dram_end_address = prop->dram_base_address + dram_size;
2884 	}
2885 
2886 	if (!strlen(prop->cpucp_info.card_name))
2887 		strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
2888 
2889 	/* Overwrite binning masks with the actual binning values from F/W */
2890 	hdev->dram_binning = prop->cpucp_info.dram_binning_mask;
2891 	hdev->edma_binning = prop->cpucp_info.edma_binning_mask;
2892 	hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask);
2893 	hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask));
2894 
2895 	dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n",
2896 			hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2897 			hdev->decoder_binning);
2898 
2899 	/*
2900 	 * at this point the DRAM parameters need to be updated according to data obtained
2901 	 * from the FW
2902 	 */
2903 	rc = hdev->asic_funcs->set_dram_properties(hdev);
2904 	if (rc)
2905 		return rc;
2906 
2907 	rc = hdev->asic_funcs->set_binning_masks(hdev);
2908 	if (rc)
2909 		return rc;
2910 
2911 	max_power = hl_fw_get_max_power(hdev);
2912 	if (max_power < 0)
2913 		return max_power;
2914 
2915 	prop->max_power_default = (u64) max_power;
2916 
2917 	return 0;
2918 }
2919 
2920 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev)
2921 {
2922 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
2923 	u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS];
2924 	int rc;
2925 
2926 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
2927 		return 0;
2928 
2929 	rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr);
2930 	if (rc)
2931 		return rc;
2932 
2933 	hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3];
2934 
2935 	return 0;
2936 }
2937 
2938 static int gaudi2_early_init(struct hl_device *hdev)
2939 {
2940 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2941 	struct pci_dev *pdev = hdev->pdev;
2942 	resource_size_t pci_bar_size;
2943 	int rc;
2944 
2945 	rc = gaudi2_set_fixed_properties(hdev);
2946 	if (rc)
2947 		return rc;
2948 
2949 	/* Check BAR sizes */
2950 	pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID);
2951 
2952 	if (pci_bar_size != CFG_BAR_SIZE) {
2953 		dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2954 			SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE);
2955 		rc = -ENODEV;
2956 		goto free_queue_props;
2957 	}
2958 
2959 	pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID);
2960 	if (pci_bar_size != MSIX_BAR_SIZE) {
2961 		dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2962 			MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE);
2963 		rc = -ENODEV;
2964 		goto free_queue_props;
2965 	}
2966 
2967 	prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID);
2968 	hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID);
2969 
2970 	/*
2971 	 * Only in pldm driver config iATU
2972 	 */
2973 	if (hdev->pldm)
2974 		hdev->asic_prop.iatu_done_by_fw = false;
2975 	else
2976 		hdev->asic_prop.iatu_done_by_fw = true;
2977 
2978 	rc = hl_pci_init(hdev);
2979 	if (rc)
2980 		goto free_queue_props;
2981 
2982 	/* Before continuing in the initialization, we need to read the preboot
2983 	 * version to determine whether we run with a security-enabled firmware
2984 	 */
2985 	rc = hl_fw_read_preboot_status(hdev);
2986 	if (rc) {
2987 		if (hdev->reset_on_preboot_fail)
2988 			/* we are already on failure flow, so don't check if hw_fini fails. */
2989 			hdev->asic_funcs->hw_fini(hdev, true, false);
2990 		goto pci_fini;
2991 	}
2992 
2993 	if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
2994 		dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
2995 		rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2996 		if (rc) {
2997 			dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);
2998 			goto pci_fini;
2999 		}
3000 	}
3001 
3002 	return 0;
3003 
3004 pci_fini:
3005 	hl_pci_fini(hdev);
3006 free_queue_props:
3007 	kfree(hdev->asic_prop.hw_queues_props);
3008 	return rc;
3009 }
3010 
3011 static int gaudi2_early_fini(struct hl_device *hdev)
3012 {
3013 	kfree(hdev->asic_prop.hw_queues_props);
3014 	hl_pci_fini(hdev);
3015 
3016 	return 0;
3017 }
3018 
3019 static bool gaudi2_is_arc_nic_owned(u64 arc_id)
3020 {
3021 	switch (arc_id) {
3022 	case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
3023 		return true;
3024 	default:
3025 		return false;
3026 	}
3027 }
3028 
3029 static bool gaudi2_is_arc_tpc_owned(u64 arc_id)
3030 {
3031 	switch (arc_id) {
3032 	case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
3033 		return true;
3034 	default:
3035 		return false;
3036 	}
3037 }
3038 
3039 static void gaudi2_init_arcs(struct hl_device *hdev)
3040 {
3041 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3042 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3043 	u64 arc_id;
3044 	u32 i;
3045 
3046 	for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) {
3047 		if (gaudi2_is_arc_enabled(hdev, i))
3048 			continue;
3049 
3050 		gaudi2_set_arc_id_cap(hdev, i);
3051 	}
3052 
3053 	for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) {
3054 		if (!gaudi2_is_queue_enabled(hdev, i))
3055 			continue;
3056 
3057 		arc_id = gaudi2_queue_id_to_arc_id[i];
3058 		if (gaudi2_is_arc_enabled(hdev, arc_id))
3059 			continue;
3060 
3061 		if (gaudi2_is_arc_nic_owned(arc_id) &&
3062 				!(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)))
3063 			continue;
3064 
3065 		if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized &
3066 							BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)))
3067 			continue;
3068 
3069 		gaudi2_set_arc_id_cap(hdev, arc_id);
3070 	}
3071 
3072 	/* Fetch ARC scratchpad address */
3073 	hdev->asic_prop.engine_core_interrupt_reg_addr =
3074 		CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl);
3075 }
3076 
3077 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id)
3078 {
3079 	u32 reg_base, reg_val;
3080 	int rc;
3081 
3082 	switch (cpu_id) {
3083 	case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3:
3084 		/* Each ARC scheduler has 2 consecutive DCCM blocks */
3085 		rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3086 						ARC_DCCM_BLOCK_SIZE * 2, true);
3087 		if (rc)
3088 			return rc;
3089 		break;
3090 	case CPU_ID_SCHED_ARC4:
3091 	case CPU_ID_SCHED_ARC5:
3092 	case CPU_ID_MME_QMAN_ARC0:
3093 	case CPU_ID_MME_QMAN_ARC1:
3094 		reg_base = gaudi2_arc_blocks_bases[cpu_id];
3095 
3096 		/* Scrub lower DCCM block */
3097 		rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3098 						ARC_DCCM_BLOCK_SIZE, true);
3099 		if (rc)
3100 			return rc;
3101 
3102 		/* Switch to upper DCCM block */
3103 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1);
3104 		WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val);
3105 
3106 		/* Scrub upper DCCM block */
3107 		rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3108 						ARC_DCCM_BLOCK_SIZE, true);
3109 		if (rc)
3110 			return rc;
3111 
3112 		/* Switch to lower DCCM block */
3113 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0);
3114 		WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val);
3115 		break;
3116 	default:
3117 		rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
3118 						ARC_DCCM_BLOCK_SIZE, true);
3119 		if (rc)
3120 			return rc;
3121 	}
3122 
3123 	return 0;
3124 }
3125 
3126 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev)
3127 {
3128 	u16 arc_id;
3129 	int rc;
3130 
3131 	for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) {
3132 		if (!gaudi2_is_arc_enabled(hdev, arc_id))
3133 			continue;
3134 
3135 		rc = gaudi2_scrub_arc_dccm(hdev, arc_id);
3136 		if (rc)
3137 			return rc;
3138 	}
3139 
3140 	return 0;
3141 }
3142 
3143 static int gaudi2_late_init(struct hl_device *hdev)
3144 {
3145 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3146 	int rc;
3147 
3148 	hdev->asic_prop.supports_advanced_cpucp_rc = true;
3149 
3150 	rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS,
3151 					gaudi2->virt_msix_db_dma_addr);
3152 	if (rc) {
3153 		dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
3154 		return rc;
3155 	}
3156 
3157 	rc = gaudi2_fetch_psoc_frequency(hdev);
3158 	if (rc) {
3159 		dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
3160 		goto disable_pci_access;
3161 	}
3162 
3163 	gaudi2_init_arcs(hdev);
3164 
3165 	rc = gaudi2_scrub_arcs_dccm(hdev);
3166 	if (rc) {
3167 		dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
3168 		goto disable_pci_access;
3169 	}
3170 
3171 	gaudi2_init_security(hdev);
3172 
3173 	return 0;
3174 
3175 disable_pci_access:
3176 	hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
3177 
3178 	return rc;
3179 }
3180 
3181 static void gaudi2_late_fini(struct hl_device *hdev)
3182 {
3183 	hl_hwmon_release_resources(hdev);
3184 }
3185 
3186 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx)
3187 {
3188 	struct user_mapped_block *blocks = gaudi2->mapped_blocks;
3189 
3190 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3191 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3192 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3193 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3194 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3195 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3196 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3197 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3198 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE);
3199 	HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE);
3200 }
3201 
3202 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev)
3203 {
3204 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3205 	struct user_mapped_block *blocks = gaudi2->mapped_blocks;
3206 	u32 block_size, umr_start_idx, num_umr_blocks;
3207 	int i;
3208 
3209 	for (i = 0 ; i < NUM_ARC_CPUS ; i++) {
3210 		if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3)
3211 			block_size = ARC_DCCM_BLOCK_SIZE * 2;
3212 		else
3213 			block_size = ARC_DCCM_BLOCK_SIZE;
3214 
3215 		blocks[i].address = gaudi2_arc_dccm_bases[i];
3216 		blocks[i].size = block_size;
3217 	}
3218 
3219 	blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE;
3220 	blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE;
3221 
3222 	blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE;
3223 	blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE;
3224 
3225 	blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE;
3226 	blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE;
3227 
3228 	blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE;
3229 	blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE;
3230 
3231 	blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE;
3232 	blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE;
3233 
3234 	blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE;
3235 	blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE;
3236 
3237 	blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE;
3238 	blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE;
3239 
3240 	blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE;
3241 	blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE;
3242 
3243 	umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS;
3244 	num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS;
3245 	for (i = 0 ; i < num_umr_blocks ; i++) {
3246 		u8 nic_id, umr_block_id;
3247 
3248 		nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS;
3249 		umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS;
3250 
3251 		blocks[umr_start_idx + i].address =
3252 			mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE +
3253 			(nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET +
3254 			(nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET +
3255 			umr_block_id * NIC_UMR_OFFSET;
3256 		blocks[umr_start_idx + i].size = HL_BLOCK_SIZE;
3257 	}
3258 
3259 	/* Expose decoder HW configuration block to user */
3260 	gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX);
3261 
3262 	for (i = 1; i < NUM_OF_DCORES; ++i) {
3263 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE;
3264 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE;
3265 
3266 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address =
3267 						mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET;
3268 
3269 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address =
3270 						mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET;
3271 	}
3272 }
3273 
3274 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
3275 {
3276 	dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;
3277 	void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {};
3278 	int i, j, rc = 0;
3279 
3280 	/* The device ARC works with 32-bits addresses, and because there is a single HW register
3281 	 * that holds the extension bits (49..28), these bits must be identical in all the allocated
3282 	 * range.
3283 	 */
3284 
3285 	for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) {
3286 		virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE,
3287 							&dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO);
3288 		if (!virt_addr_arr[i]) {
3289 			rc = -ENOMEM;
3290 			goto free_dma_mem_arr;
3291 		}
3292 
3293 		end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;
3294 		if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr))
3295 			break;
3296 	}
3297 
3298 	if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) {
3299 		dev_err(hdev->dev,
3300 			"MSB of ARC accessible DMA memory are not identical in all range\n");
3301 		rc = -EFAULT;
3302 		goto free_dma_mem_arr;
3303 	}
3304 
3305 	hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
3306 	hdev->cpu_accessible_dma_address = dma_addr_arr[i];
3307 
3308 free_dma_mem_arr:
3309 	for (j = 0 ; j < i ; j++)
3310 		hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j],
3311 						dma_addr_arr[j]);
3312 
3313 	return rc;
3314 }
3315 
3316 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev)
3317 {
3318 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3319 	struct pci_mem_region *region;
3320 
3321 	/* CFG */
3322 	region = &hdev->pci_mem_region[PCI_REGION_CFG];
3323 	region->region_base = CFG_BASE;
3324 	region->region_size = CFG_SIZE;
3325 	region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR;
3326 	region->bar_size = CFG_BAR_SIZE;
3327 	region->bar_id = SRAM_CFG_BAR_ID;
3328 	region->used = 1;
3329 
3330 	/* SRAM */
3331 	region = &hdev->pci_mem_region[PCI_REGION_SRAM];
3332 	region->region_base = SRAM_BASE_ADDR;
3333 	region->region_size = SRAM_SIZE;
3334 	region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE;
3335 	region->bar_size = CFG_BAR_SIZE;
3336 	region->bar_id = SRAM_CFG_BAR_ID;
3337 	region->used = 1;
3338 
3339 	/* DRAM */
3340 	region = &hdev->pci_mem_region[PCI_REGION_DRAM];
3341 	region->region_base = DRAM_PHYS_BASE;
3342 	region->region_size = hdev->asic_prop.dram_size;
3343 	region->offset_in_bar = 0;
3344 	region->bar_size = prop->dram_pci_bar_size;
3345 	region->bar_id = DRAM_BAR_ID;
3346 	region->used = 1;
3347 }
3348 
3349 static void gaudi2_user_interrupt_setup(struct hl_device *hdev)
3350 {
3351 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3352 	int i, j, k;
3353 
3354 	/* Initialize TPC interrupt */
3355 	HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC);
3356 
3357 	/* Initialize unexpected error interrupt */
3358 	HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0,
3359 						HL_USR_INTERRUPT_UNEXPECTED);
3360 
3361 	/* Initialize common user CQ interrupt */
3362 	HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev,
3363 				HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ);
3364 
3365 	/* Initialize common decoder interrupt */
3366 	HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev,
3367 				HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER);
3368 
3369 	/* User interrupts structure holds both decoder and user interrupts from various engines.
3370 	 * We first initialize the decoder interrupts and then we add the user interrupts.
3371 	 * The only limitation is that the last decoder interrupt id must be smaller
3372 	 * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time.
3373 	 */
3374 
3375 	/* Initialize decoder interrupts, expose only normal interrupts,
3376 	 * error interrupts to be handled by driver
3377 	 */
3378 	for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM;
3379 										i += 2, j++)
3380 		HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i,
3381 						HL_USR_INTERRUPT_DECODER);
3382 
3383 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++)
3384 		HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ);
3385 }
3386 
3387 static inline int gaudi2_get_non_zero_random_int(void)
3388 {
3389 	int rand = get_random_u32();
3390 
3391 	return rand ? rand : 1;
3392 }
3393 
3394 static void gaudi2_special_blocks_free(struct hl_device *hdev)
3395 {
3396 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3397 	struct hl_skip_blocks_cfg *skip_special_blocks_cfg =
3398 			&prop->skip_special_blocks_cfg;
3399 
3400 	kfree(prop->special_blocks);
3401 	kfree(skip_special_blocks_cfg->block_types);
3402 	kfree(skip_special_blocks_cfg->block_ranges);
3403 }
3404 
3405 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev)
3406 {
3407 	gaudi2_special_blocks_free(hdev);
3408 }
3409 
3410 static bool gaudi2_special_block_skip(struct hl_device *hdev,
3411 		struct hl_special_blocks_cfg *special_blocks_cfg,
3412 		u32 blk_idx, u32 major, u32 minor, u32 sub_minor)
3413 {
3414 	return false;
3415 }
3416 
3417 static int gaudi2_special_blocks_config(struct hl_device *hdev)
3418 {
3419 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3420 	int i, rc;
3421 
3422 	/* Configure Special blocks */
3423 	prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE;
3424 	prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks);
3425 	prop->special_blocks = kmalloc_array(prop->num_of_special_blocks,
3426 			sizeof(*prop->special_blocks), GFP_KERNEL);
3427 	if (!prop->special_blocks)
3428 		return -ENOMEM;
3429 
3430 	for (i = 0 ; i < prop->num_of_special_blocks ; i++)
3431 		memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i],
3432 				sizeof(*prop->special_blocks));
3433 
3434 	/* Configure when to skip Special blocks */
3435 	memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg));
3436 	prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip;
3437 
3438 	if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) {
3439 		prop->skip_special_blocks_cfg.block_types =
3440 				kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types),
3441 					sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL);
3442 		if (!prop->skip_special_blocks_cfg.block_types) {
3443 			rc = -ENOMEM;
3444 			goto free_special_blocks;
3445 		}
3446 
3447 		memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types,
3448 				sizeof(gaudi2_iterator_skip_block_types));
3449 
3450 		prop->skip_special_blocks_cfg.block_types_len =
3451 					ARRAY_SIZE(gaudi2_iterator_skip_block_types);
3452 	}
3453 
3454 	if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) {
3455 		prop->skip_special_blocks_cfg.block_ranges =
3456 				kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges),
3457 					sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL);
3458 		if (!prop->skip_special_blocks_cfg.block_ranges) {
3459 			rc = -ENOMEM;
3460 			goto free_skip_special_blocks_types;
3461 		}
3462 
3463 		for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++)
3464 			memcpy(&prop->skip_special_blocks_cfg.block_ranges[i],
3465 					&gaudi2_iterator_skip_block_ranges[i],
3466 					sizeof(struct range));
3467 
3468 		prop->skip_special_blocks_cfg.block_ranges_len =
3469 					ARRAY_SIZE(gaudi2_iterator_skip_block_ranges);
3470 	}
3471 
3472 	return 0;
3473 
3474 free_skip_special_blocks_types:
3475 	kfree(prop->skip_special_blocks_cfg.block_types);
3476 free_special_blocks:
3477 	kfree(prop->special_blocks);
3478 
3479 	return rc;
3480 }
3481 
3482 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev)
3483 {
3484 	return gaudi2_special_blocks_config(hdev);
3485 }
3486 
3487 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev)
3488 {
3489 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3490 	struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;
3491 	int i;
3492 
3493 	for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) {
3494 		/* bail-out if this is an allocation failure point */
3495 		if (!msg_info[i].kern_addr)
3496 			break;
3497 
3498 		hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr);
3499 		msg_info[i].kern_addr = NULL;
3500 	}
3501 }
3502 
3503 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev)
3504 {
3505 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3506 	struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;
3507 	int i, rc;
3508 
3509 	/* allocate a message-short buf for each Q we intend to test */
3510 	for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) {
3511 		msg_info[i].kern_addr =
3512 			(void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short),
3513 							GFP_KERNEL, &msg_info[i].dma_addr);
3514 		if (!msg_info[i].kern_addr) {
3515 			dev_err(hdev->dev,
3516 				"Failed to allocate dma memory for H/W queue %d testing\n", i);
3517 			rc = -ENOMEM;
3518 			goto err_exit;
3519 		}
3520 	}
3521 
3522 	return 0;
3523 
3524 err_exit:
3525 	gaudi2_test_queues_msgs_free(hdev);
3526 	return rc;
3527 }
3528 
3529 static int gaudi2_sw_init(struct hl_device *hdev)
3530 {
3531 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3532 	struct gaudi2_device *gaudi2;
3533 	int i, rc;
3534 
3535 	/* Allocate device structure */
3536 	gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL);
3537 	if (!gaudi2)
3538 		return -ENOMEM;
3539 
3540 	for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) {
3541 		if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid)
3542 			continue;
3543 
3544 		if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) {
3545 			dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n",
3546 				GAUDI2_EVENT_SIZE);
3547 			rc = -EINVAL;
3548 			goto free_gaudi2_device;
3549 		}
3550 
3551 		gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id;
3552 	}
3553 
3554 	for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++)
3555 		gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int();
3556 
3557 	gaudi2->cpucp_info_get = gaudi2_cpucp_info_get;
3558 
3559 	hdev->asic_specific = gaudi2;
3560 
3561 	/* Create DMA pool for small allocations.
3562 	 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped
3563 	 * PI/CI registers allocated from this pool have this restriction
3564 	 */
3565 	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev,
3566 					GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0);
3567 	if (!hdev->dma_pool) {
3568 		dev_err(hdev->dev, "failed to create DMA pool\n");
3569 		rc = -ENOMEM;
3570 		goto free_gaudi2_device;
3571 	}
3572 
3573 	rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev);
3574 	if (rc)
3575 		goto free_dma_pool;
3576 
3577 	hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
3578 	if (!hdev->cpu_accessible_dma_pool) {
3579 		dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n");
3580 		rc = -ENOMEM;
3581 		goto free_cpu_dma_mem;
3582 	}
3583 
3584 	rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem,
3585 				HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
3586 	if (rc) {
3587 		dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n");
3588 		rc = -EFAULT;
3589 		goto free_cpu_accessible_dma_pool;
3590 	}
3591 
3592 	gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size,
3593 								&gaudi2->virt_msix_db_dma_addr);
3594 	if (!gaudi2->virt_msix_db_cpu_addr) {
3595 		dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n");
3596 		rc = -ENOMEM;
3597 		goto free_cpu_accessible_dma_pool;
3598 	}
3599 
3600 	spin_lock_init(&gaudi2->hw_queues_lock);
3601 
3602 	gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE,
3603 							&gaudi2->scratchpad_bus_address,
3604 							GFP_KERNEL | __GFP_ZERO);
3605 	if (!gaudi2->scratchpad_kernel_address) {
3606 		rc = -ENOMEM;
3607 		goto free_virt_msix_db_mem;
3608 	}
3609 
3610 	gaudi2_user_mapped_blocks_init(hdev);
3611 
3612 	/* Initialize user interrupts */
3613 	gaudi2_user_interrupt_setup(hdev);
3614 
3615 	hdev->supports_coresight = true;
3616 	hdev->supports_sync_stream = true;
3617 	hdev->supports_cb_mapping = true;
3618 	hdev->supports_wait_for_multi_cs = false;
3619 
3620 	prop->supports_compute_reset = true;
3621 
3622 	/* Event queue sanity check added in FW version 1.11 */
3623 	if (hl_is_fw_sw_ver_below(hdev, 1, 11))
3624 		hdev->event_queue.check_eqe_index = false;
3625 	else
3626 		hdev->event_queue.check_eqe_index = true;
3627 
3628 	hdev->asic_funcs->set_pci_memory_regions(hdev);
3629 
3630 	rc = gaudi2_special_blocks_iterator_config(hdev);
3631 	if (rc)
3632 		goto free_scratchpad_mem;
3633 
3634 	rc = gaudi2_test_queues_msgs_alloc(hdev);
3635 	if (rc)
3636 		goto special_blocks_free;
3637 
3638 	return 0;
3639 
3640 special_blocks_free:
3641 	gaudi2_special_blocks_iterator_free(hdev);
3642 free_scratchpad_mem:
3643 	hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3644 				  gaudi2->scratchpad_bus_address);
3645 free_virt_msix_db_mem:
3646 	hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3647 free_cpu_accessible_dma_pool:
3648 	gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3649 free_cpu_dma_mem:
3650 	hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3651 					hdev->cpu_accessible_dma_address);
3652 free_dma_pool:
3653 	dma_pool_destroy(hdev->dma_pool);
3654 free_gaudi2_device:
3655 	kfree(gaudi2);
3656 	return rc;
3657 }
3658 
3659 static int gaudi2_sw_fini(struct hl_device *hdev)
3660 {
3661 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3662 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3663 
3664 	gaudi2_test_queues_msgs_free(hdev);
3665 
3666 	gaudi2_special_blocks_iterator_free(hdev);
3667 
3668 	hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3669 
3670 	gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3671 
3672 	hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3673 						hdev->cpu_accessible_dma_address);
3674 
3675 	hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3676 					gaudi2->scratchpad_bus_address);
3677 
3678 	dma_pool_destroy(hdev->dma_pool);
3679 
3680 	kfree(gaudi2);
3681 
3682 	return 0;
3683 }
3684 
3685 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base)
3686 {
3687 	WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP |
3688 						QM_GLBL_CFG1_CQF_STOP |
3689 						QM_GLBL_CFG1_CP_STOP);
3690 
3691 	/* stop also the ARC */
3692 	WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP);
3693 }
3694 
3695 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base)
3696 {
3697 	WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH |
3698 						QM_GLBL_CFG1_CQF_FLUSH |
3699 						QM_GLBL_CFG1_CP_FLUSH);
3700 }
3701 
3702 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base)
3703 {
3704 	WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH);
3705 }
3706 
3707 /**
3708  * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3709  *
3710  * @hdev: pointer to the habanalabs device structure
3711  * @queue_id: queue to clear fence counters to
3712  * @skip_fence: if true set maximum fence value to all fence counters to avoid
3713  *              getting stuck on any fence value. otherwise set all fence
3714  *              counters to 0 (standard clear of fence counters)
3715  */
3716 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id,
3717 						bool skip_fence)
3718 {
3719 	u32 size, reg_base;
3720 	u32 addr, val;
3721 
3722 	reg_base = gaudi2_qm_blocks_bases[queue_id];
3723 
3724 	addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET;
3725 	size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0;
3726 
3727 	/*
3728 	 * in case we want to make sure that QM that is stuck on a fence will
3729 	 * be released we should set the fence counter to a higher value that
3730 	 * the value the QM waiting for. to comply with any fence counter of
3731 	 * any value we set maximum fence value to all counters
3732 	 */
3733 	val = skip_fence ? U32_MAX : 0;
3734 	gaudi2_memset_device_lbw(hdev, addr, size, val);
3735 }
3736 
3737 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id)
3738 {
3739 	u32 reg_base = gaudi2_qm_blocks_bases[queue_id];
3740 
3741 	gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true);
3742 	gaudi2_flush_qman_common(hdev, reg_base);
3743 	gaudi2_flush_qman_arc_common(hdev, reg_base);
3744 }
3745 
3746 static void gaudi2_stop_dma_qmans(struct hl_device *hdev)
3747 {
3748 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3749 	int dcore, inst;
3750 
3751 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3752 		goto stop_edma_qmans;
3753 
3754 	/* Stop CPs of PDMA QMANs */
3755 	gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE);
3756 	gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE);
3757 
3758 stop_edma_qmans:
3759 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3760 		return;
3761 
3762 	for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3763 		for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3764 			u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3765 			u32 qm_base;
3766 
3767 			if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3768 				continue;
3769 
3770 			qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET +
3771 					inst * DCORE_EDMA_OFFSET;
3772 
3773 			/* Stop CPs of EDMA QMANs */
3774 			gaudi2_stop_qman_common(hdev, qm_base);
3775 		}
3776 	}
3777 }
3778 
3779 static void gaudi2_stop_mme_qmans(struct hl_device *hdev)
3780 {
3781 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3782 	u32 offset, i;
3783 
3784 	offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;
3785 
3786 	for (i = 0 ; i < NUM_OF_DCORES ; i++) {
3787 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)))
3788 			continue;
3789 
3790 		gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3791 	}
3792 }
3793 
3794 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev)
3795 {
3796 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3797 	u32 reg_base;
3798 	int i;
3799 
3800 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3801 		return;
3802 
3803 	for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3804 		if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3805 			continue;
3806 
3807 		reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]];
3808 		gaudi2_stop_qman_common(hdev, reg_base);
3809 	}
3810 }
3811 
3812 static void gaudi2_stop_rot_qmans(struct hl_device *hdev)
3813 {
3814 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3815 	u32 reg_base;
3816 	int i;
3817 
3818 	if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3819 		return;
3820 
3821 	for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3822 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3823 			continue;
3824 
3825 		reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]];
3826 		gaudi2_stop_qman_common(hdev, reg_base);
3827 	}
3828 }
3829 
3830 static void gaudi2_stop_nic_qmans(struct hl_device *hdev)
3831 {
3832 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3833 	u32 reg_base, queue_id;
3834 	int i;
3835 
3836 	if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
3837 		return;
3838 
3839 	queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
3840 
3841 	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
3842 		if (!(hdev->nic_ports_mask & BIT(i)))
3843 			continue;
3844 
3845 		reg_base = gaudi2_qm_blocks_bases[queue_id];
3846 		gaudi2_stop_qman_common(hdev, reg_base);
3847 	}
3848 }
3849 
3850 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base)
3851 {
3852 	u32 reg_val;
3853 
3854 	reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1);
3855 	WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val);
3856 }
3857 
3858 static void gaudi2_dma_stall(struct hl_device *hdev)
3859 {
3860 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3861 	int dcore, inst;
3862 
3863 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3864 		goto stall_edma;
3865 
3866 	gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE);
3867 	gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE);
3868 
3869 stall_edma:
3870 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3871 		return;
3872 
3873 	for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3874 		for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3875 			u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3876 			u32 core_base;
3877 
3878 			if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3879 				continue;
3880 
3881 			core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET +
3882 					inst * DCORE_EDMA_OFFSET;
3883 
3884 			/* Stall CPs of EDMA QMANs */
3885 			gaudi2_stall_dma_common(hdev, core_base);
3886 		}
3887 	}
3888 }
3889 
3890 static void gaudi2_mme_stall(struct hl_device *hdev)
3891 {
3892 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3893 	u32 offset, i;
3894 
3895 	offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL;
3896 
3897 	for (i = 0 ; i < NUM_OF_DCORES ; i++)
3898 		if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))
3899 			WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1);
3900 }
3901 
3902 static void gaudi2_tpc_stall(struct hl_device *hdev)
3903 {
3904 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3905 	u32 reg_base;
3906 	int i;
3907 
3908 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3909 		return;
3910 
3911 	for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3912 		if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3913 			continue;
3914 
3915 		reg_base = gaudi2_tpc_cfg_blocks_bases[i];
3916 		WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1);
3917 	}
3918 }
3919 
3920 static void gaudi2_rotator_stall(struct hl_device *hdev)
3921 {
3922 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3923 	u32 reg_val;
3924 	int i;
3925 
3926 	if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3927 		return;
3928 
3929 	reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) |
3930 			FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) |
3931 			FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1);
3932 
3933 	for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3934 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3935 			continue;
3936 
3937 		WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val);
3938 	}
3939 }
3940 
3941 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base)
3942 {
3943 	WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0);
3944 }
3945 
3946 static void gaudi2_disable_dma_qmans(struct hl_device *hdev)
3947 {
3948 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3949 	int dcore, inst;
3950 
3951 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3952 		goto stop_edma_qmans;
3953 
3954 	gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE);
3955 	gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE);
3956 
3957 stop_edma_qmans:
3958 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3959 		return;
3960 
3961 	for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3962 		for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3963 			u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3964 			u32 qm_base;
3965 
3966 			if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3967 				continue;
3968 
3969 			qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET +
3970 					inst * DCORE_EDMA_OFFSET;
3971 
3972 			/* Disable CPs of EDMA QMANs */
3973 			gaudi2_disable_qman_common(hdev, qm_base);
3974 		}
3975 	}
3976 }
3977 
3978 static void gaudi2_disable_mme_qmans(struct hl_device *hdev)
3979 {
3980 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3981 	u32 offset, i;
3982 
3983 	offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;
3984 
3985 	for (i = 0 ; i < NUM_OF_DCORES ; i++)
3986 		if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))
3987 			gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3988 }
3989 
3990 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev)
3991 {
3992 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
3993 	u32 reg_base;
3994 	int i;
3995 
3996 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3997 		return;
3998 
3999 	for (i = 0 ; i < TPC_ID_SIZE ; i++) {
4000 		if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
4001 			continue;
4002 
4003 		reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]];
4004 		gaudi2_disable_qman_common(hdev, reg_base);
4005 	}
4006 }
4007 
4008 static void gaudi2_disable_rot_qmans(struct hl_device *hdev)
4009 {
4010 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4011 	u32 reg_base;
4012 	int i;
4013 
4014 	if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
4015 		return;
4016 
4017 	for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
4018 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
4019 			continue;
4020 
4021 		reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]];
4022 		gaudi2_disable_qman_common(hdev, reg_base);
4023 	}
4024 }
4025 
4026 static void gaudi2_disable_nic_qmans(struct hl_device *hdev)
4027 {
4028 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4029 	u32 reg_base, queue_id;
4030 	int i;
4031 
4032 	if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
4033 		return;
4034 
4035 	queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
4036 
4037 	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
4038 		if (!(hdev->nic_ports_mask & BIT(i)))
4039 			continue;
4040 
4041 		reg_base = gaudi2_qm_blocks_bases[queue_id];
4042 		gaudi2_disable_qman_common(hdev, reg_base);
4043 	}
4044 }
4045 
4046 static void gaudi2_enable_timestamp(struct hl_device *hdev)
4047 {
4048 	/* Disable the timestamp counter */
4049 	WREG32(mmPSOC_TIMESTAMP_BASE, 0);
4050 
4051 	/* Zero the lower/upper parts of the 64-bit counter */
4052 	WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0);
4053 	WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0);
4054 
4055 	/* Enable the counter */
4056 	WREG32(mmPSOC_TIMESTAMP_BASE, 1);
4057 }
4058 
4059 static void gaudi2_disable_timestamp(struct hl_device *hdev)
4060 {
4061 	/* Disable the timestamp counter */
4062 	WREG32(mmPSOC_TIMESTAMP_BASE, 0);
4063 }
4064 
4065 static const char *gaudi2_irq_name(u16 irq_number)
4066 {
4067 	switch (irq_number) {
4068 	case GAUDI2_IRQ_NUM_EVENT_QUEUE:
4069 		return "gaudi2 cpu eq";
4070 	case GAUDI2_IRQ_NUM_COMPLETION:
4071 		return "gaudi2 completion";
4072 	case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM:
4073 		return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM];
4074 	case GAUDI2_IRQ_NUM_TPC_ASSERT:
4075 		return "gaudi2 tpc assert";
4076 	case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR:
4077 		return "gaudi2 unexpected error";
4078 	case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST:
4079 		return "gaudi2 user completion";
4080 	default:
4081 		return "invalid";
4082 	}
4083 }
4084 
4085 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num)
4086 {
4087 	int i, irq, relative_idx;
4088 	struct hl_dec *dec;
4089 
4090 	for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) {
4091 		irq = pci_irq_vector(hdev->pdev, i);
4092 		relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
4093 
4094 		dec = hdev->dec + relative_idx / 2;
4095 
4096 		/* We pass different structures depending on the irq handler. For the abnormal
4097 		 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
4098 		 * user_interrupt entry
4099 		 */
4100 		free_irq(irq, ((relative_idx % 2) ?
4101 				(void *) dec :
4102 				(void *) &hdev->user_interrupt[dec->core_id]));
4103 	}
4104 }
4105 
4106 static int gaudi2_dec_enable_msix(struct hl_device *hdev)
4107 {
4108 	int rc, i, irq_init_cnt, irq, relative_idx;
4109 	struct hl_dec *dec;
4110 
4111 	for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0;
4112 			i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM;
4113 			i++, irq_init_cnt++) {
4114 
4115 		irq = pci_irq_vector(hdev->pdev, i);
4116 		relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
4117 
4118 		/* We pass different structures depending on the irq handler. For the abnormal
4119 		 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
4120 		 * user_interrupt entry
4121 		 *
4122 		 * TODO: change the dec abnrm to threaded irq
4123 		 */
4124 
4125 		dec = hdev->dec + relative_idx / 2;
4126 		if (relative_idx % 2) {
4127 			rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0,
4128 						gaudi2_irq_name(i), (void *) dec);
4129 		} else {
4130 			rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
4131 					hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
4132 					gaudi2_irq_name(i),
4133 					(void *) &hdev->user_interrupt[dec->core_id]);
4134 		}
4135 
4136 		if (rc) {
4137 			dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4138 			goto free_dec_irqs;
4139 		}
4140 	}
4141 
4142 	return 0;
4143 
4144 free_dec_irqs:
4145 	gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt));
4146 	return rc;
4147 }
4148 
4149 static int gaudi2_enable_msix(struct hl_device *hdev)
4150 {
4151 	struct asic_fixed_properties *prop = &hdev->asic_prop;
4152 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4153 	int rc, irq, i, j, user_irq_init_cnt;
4154 	struct hl_cq *cq;
4155 
4156 	if (gaudi2->hw_cap_initialized & HW_CAP_MSIX)
4157 		return 0;
4158 
4159 	rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES,
4160 					PCI_IRQ_MSIX);
4161 	if (rc < 0) {
4162 		dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n",
4163 			GAUDI2_MSIX_ENTRIES, rc);
4164 		return rc;
4165 	}
4166 
4167 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4168 	cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
4169 	rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq);
4170 	if (rc) {
4171 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4172 		goto free_irq_vectors;
4173 	}
4174 
4175 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4176 	rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE),
4177 			&hdev->event_queue);
4178 	if (rc) {
4179 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4180 		goto free_completion_irq;
4181 	}
4182 
4183 	rc = gaudi2_dec_enable_msix(hdev);
4184 	if (rc) {
4185 		dev_err(hdev->dev, "Failed to enable decoder IRQ");
4186 		goto free_event_irq;
4187 	}
4188 
4189 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4190 	rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
4191 			hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
4192 			gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt);
4193 	if (rc) {
4194 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4195 		goto free_dec_irq;
4196 	}
4197 
4198 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4199 	rc = request_irq(irq, hl_irq_handler_user_interrupt, 0,
4200 			gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR),
4201 					&hdev->unexpected_error_interrupt);
4202 	if (rc) {
4203 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4204 		goto free_tpc_irq;
4205 	}
4206 
4207 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0;
4208 			user_irq_init_cnt < prop->user_interrupt_count;
4209 			i++, j++, user_irq_init_cnt++) {
4210 
4211 		irq = pci_irq_vector(hdev->pdev, i);
4212 		rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
4213 						hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
4214 						gaudi2_irq_name(i), &hdev->user_interrupt[j]);
4215 
4216 		if (rc) {
4217 			dev_err(hdev->dev, "Failed to request IRQ %d", irq);
4218 			goto free_user_irq;
4219 		}
4220 	}
4221 
4222 	gaudi2->hw_cap_initialized |= HW_CAP_MSIX;
4223 
4224 	return 0;
4225 
4226 free_user_irq:
4227 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count;
4228 			i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) {
4229 
4230 		irq = pci_irq_vector(hdev->pdev, i);
4231 		free_irq(irq, &hdev->user_interrupt[j]);
4232 	}
4233 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4234 	free_irq(irq, &hdev->unexpected_error_interrupt);
4235 free_tpc_irq:
4236 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4237 	free_irq(irq, &hdev->tpc_interrupt);
4238 free_dec_irq:
4239 	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1);
4240 free_event_irq:
4241 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4242 	free_irq(irq, cq);
4243 
4244 free_completion_irq:
4245 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4246 	free_irq(irq, cq);
4247 
4248 free_irq_vectors:
4249 	pci_free_irq_vectors(hdev->pdev);
4250 
4251 	return rc;
4252 }
4253 
4254 static void gaudi2_sync_irqs(struct hl_device *hdev)
4255 {
4256 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4257 	int i, j;
4258 	int irq;
4259 
4260 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))
4261 		return;
4262 
4263 	/* Wait for all pending IRQs to be finished */
4264 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION));
4265 
4266 	for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) {
4267 		irq = pci_irq_vector(hdev->pdev, i);
4268 		synchronize_irq(irq);
4269 	}
4270 
4271 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT));
4272 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR));
4273 
4274 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count;
4275 										i++, j++) {
4276 		irq = pci_irq_vector(hdev->pdev, i);
4277 		synchronize_irq(irq);
4278 	}
4279 
4280 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE));
4281 }
4282 
4283 static void gaudi2_disable_msix(struct hl_device *hdev)
4284 {
4285 	struct asic_fixed_properties *prop = &hdev->asic_prop;
4286 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4287 	struct hl_cq *cq;
4288 	int irq, i, j, k;
4289 
4290 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))
4291 		return;
4292 
4293 	gaudi2_sync_irqs(hdev);
4294 
4295 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
4296 	free_irq(irq, &hdev->event_queue);
4297 
4298 	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
4299 
4300 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
4301 	free_irq(irq, &hdev->tpc_interrupt);
4302 
4303 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
4304 	free_irq(irq, &hdev->unexpected_error_interrupt);
4305 
4306 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0;
4307 			k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) {
4308 
4309 		irq = pci_irq_vector(hdev->pdev, i);
4310 		free_irq(irq, &hdev->user_interrupt[j]);
4311 	}
4312 
4313 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
4314 	cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
4315 	free_irq(irq, cq);
4316 
4317 	pci_free_irq_vectors(hdev->pdev);
4318 
4319 	gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX;
4320 }
4321 
4322 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id)
4323 {
4324 	u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1);
4325 	u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK;
4326 	u32 timeout_usec, dec_id, dec_bit, offset, graceful;
4327 	int rc;
4328 
4329 	if (hdev->pldm)
4330 		timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC;
4331 	else
4332 		timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC;
4333 
4334 	for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
4335 		dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id;
4336 		if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4337 			continue;
4338 
4339 		offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET;
4340 
4341 		WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0);
4342 
4343 		WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val);
4344 
4345 		/* Wait till all traffic from decoder stops
4346 		 * before apply core reset.
4347 		 */
4348 		rc = hl_poll_timeout(
4349 				hdev,
4350 				mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset,
4351 				graceful,
4352 				(graceful & graceful_pend_mask),
4353 				100,
4354 				timeout_usec);
4355 		if (rc)
4356 			dev_err(hdev->dev,
4357 				"Failed to stop traffic from DCORE%d Decoder %d\n",
4358 				dcore_id, dec_id);
4359 	}
4360 }
4361 
4362 static void gaudi2_stop_pcie_dec(struct hl_device *hdev)
4363 {
4364 	u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1);
4365 	u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK;
4366 	u32 timeout_usec, dec_id, dec_bit, offset, graceful;
4367 	int rc;
4368 
4369 	if (hdev->pldm)
4370 		timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC;
4371 	else
4372 		timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC;
4373 
4374 	for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
4375 		dec_bit = PCIE_DEC_SHIFT + dec_id;
4376 		if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4377 			continue;
4378 
4379 		offset = dec_id * PCIE_VDEC_OFFSET;
4380 
4381 		WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0);
4382 
4383 		WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val);
4384 
4385 		/* Wait till all traffic from decoder stops
4386 		 * before apply core reset.
4387 		 */
4388 		rc = hl_poll_timeout(
4389 				hdev,
4390 				mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset,
4391 				graceful,
4392 				(graceful & graceful_pend_mask),
4393 				100,
4394 				timeout_usec);
4395 		if (rc)
4396 			dev_err(hdev->dev,
4397 				"Failed to stop traffic from PCIe Decoder %d\n",
4398 				dec_id);
4399 	}
4400 }
4401 
4402 static void gaudi2_stop_dec(struct hl_device *hdev)
4403 {
4404 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4405 	int dcore_id;
4406 
4407 	if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0)
4408 		return;
4409 
4410 	for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
4411 		gaudi2_stop_dcore_dec(hdev, dcore_id);
4412 
4413 	gaudi2_stop_pcie_dec(hdev);
4414 }
4415 
4416 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode)
4417 {
4418 	u32 reg_base, reg_val;
4419 
4420 	reg_base = gaudi2_arc_blocks_bases[cpu_id];
4421 	if (run_mode == HL_ENGINE_CORE_RUN)
4422 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1);
4423 	else
4424 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1);
4425 
4426 	WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val);
4427 }
4428 
4429 static void gaudi2_halt_arcs(struct hl_device *hdev)
4430 {
4431 	u16 arc_id;
4432 
4433 	for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) {
4434 		if (gaudi2_is_arc_enabled(hdev, arc_id))
4435 			gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT);
4436 	}
4437 }
4438 
4439 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode)
4440 {
4441 	int rc;
4442 	u32 reg_base, val, ack_mask, timeout_usec = 100000;
4443 
4444 	if (hdev->pldm)
4445 		timeout_usec *= 100;
4446 
4447 	reg_base = gaudi2_arc_blocks_bases[cpu_id];
4448 	if (run_mode == HL_ENGINE_CORE_RUN)
4449 		ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK;
4450 	else
4451 		ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK;
4452 
4453 	rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET,
4454 				val, ((val & ack_mask) == ack_mask),
4455 				1000, timeout_usec);
4456 
4457 	if (!rc) {
4458 		/* Clear */
4459 		val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0);
4460 		WREG32(reg_base + ARC_HALT_REQ_OFFSET, val);
4461 	}
4462 
4463 	return rc;
4464 }
4465 
4466 static void gaudi2_reset_arcs(struct hl_device *hdev)
4467 {
4468 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4469 	u16 arc_id;
4470 
4471 	if (!gaudi2)
4472 		return;
4473 
4474 	for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++)
4475 		if (gaudi2_is_arc_enabled(hdev, arc_id))
4476 			gaudi2_clr_arc_id_cap(hdev, arc_id);
4477 }
4478 
4479 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev)
4480 {
4481 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4482 	u32 queue_id;
4483 	int i;
4484 
4485 	if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
4486 		return;
4487 
4488 	queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
4489 
4490 	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
4491 		if (!(hdev->nic_ports_mask & BIT(i)))
4492 			continue;
4493 
4494 		gaudi2_qman_manual_flush_common(hdev, queue_id);
4495 	}
4496 }
4497 
4498 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids,
4499 					u32 num_cores, u32 core_command)
4500 {
4501 	int i, rc;
4502 
4503 	for (i = 0 ; i < num_cores ; i++) {
4504 		if (gaudi2_is_arc_enabled(hdev, core_ids[i]))
4505 			gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command);
4506 	}
4507 
4508 	for (i = 0 ; i < num_cores ; i++) {
4509 		if (gaudi2_is_arc_enabled(hdev, core_ids[i])) {
4510 			rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command);
4511 
4512 			if (rc) {
4513 				dev_err(hdev->dev, "failed to %s arc: %d\n",
4514 					(core_command == HL_ENGINE_CORE_HALT) ?
4515 					"HALT" : "RUN", core_ids[i]);
4516 				return -1;
4517 			}
4518 		}
4519 	}
4520 
4521 	return 0;
4522 }
4523 
4524 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4525 {
4526 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4527 	u32 reg_base, reg_addr, reg_val, tpc_id;
4528 
4529 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
4530 		return 0;
4531 
4532 	tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id];
4533 	if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id)))
4534 		return 0;
4535 
4536 	reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id];
4537 	reg_addr = reg_base + TPC_CFG_STALL_OFFSET;
4538 	reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK,
4539 			(engine_command == HL_ENGINE_STALL) ? 1 : 0);
4540 	WREG32(reg_addr, reg_val);
4541 
4542 	if (engine_command == HL_ENGINE_RESUME) {
4543 		reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id];
4544 		reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET;
4545 		RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK);
4546 	}
4547 
4548 	return 0;
4549 }
4550 
4551 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4552 {
4553 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4554 	u32 reg_base, reg_addr, reg_val, mme_id;
4555 
4556 	mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id];
4557 	if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id)))
4558 		return 0;
4559 
4560 	reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id];
4561 	reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET;
4562 	reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK,
4563 			(engine_command == HL_ENGINE_STALL) ? 1 : 0);
4564 	WREG32(reg_addr, reg_val);
4565 
4566 	return 0;
4567 }
4568 
4569 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command)
4570 {
4571 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4572 	u32 reg_base, reg_addr, reg_val, edma_id;
4573 
4574 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
4575 		return 0;
4576 
4577 	edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id];
4578 	if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id)))
4579 		return 0;
4580 
4581 	reg_base = gaudi2_dma_core_blocks_bases[edma_id];
4582 	reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET;
4583 	reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK,
4584 			(engine_command == HL_ENGINE_STALL) ? 1 : 0);
4585 	WREG32(reg_addr, reg_val);
4586 
4587 	if (engine_command == HL_ENGINE_STALL) {
4588 		reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) |
4589 				FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1);
4590 		WREG32(reg_addr, reg_val);
4591 	}
4592 
4593 	return 0;
4594 }
4595 
4596 static int gaudi2_set_engine_modes(struct hl_device *hdev,
4597 		u32 *engine_ids, u32 num_engines, u32 engine_command)
4598 {
4599 	int i, rc;
4600 
4601 	for (i = 0 ; i < num_engines ; ++i) {
4602 		switch (engine_ids[i]) {
4603 		case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5:
4604 		case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5:
4605 		case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5:
4606 		case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5:
4607 			rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command);
4608 			if (rc)
4609 				return rc;
4610 
4611 			break;
4612 		case GAUDI2_DCORE0_ENGINE_ID_MME:
4613 		case GAUDI2_DCORE1_ENGINE_ID_MME:
4614 		case GAUDI2_DCORE2_ENGINE_ID_MME:
4615 		case GAUDI2_DCORE3_ENGINE_ID_MME:
4616 			rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command);
4617 			if (rc)
4618 				return rc;
4619 
4620 			break;
4621 		case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1:
4622 		case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1:
4623 		case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1:
4624 		case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1:
4625 			rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command);
4626 			if (rc)
4627 				return rc;
4628 
4629 			break;
4630 		default:
4631 			dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]);
4632 			return -EINVAL;
4633 		}
4634 	}
4635 
4636 	return 0;
4637 }
4638 
4639 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids,
4640 					u32 num_engines, u32 engine_command)
4641 {
4642 	switch (engine_command) {
4643 	case HL_ENGINE_CORE_HALT:
4644 	case HL_ENGINE_CORE_RUN:
4645 		return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command);
4646 
4647 	case HL_ENGINE_STALL:
4648 	case HL_ENGINE_RESUME:
4649 		return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command);
4650 
4651 	default:
4652 		dev_err(hdev->dev, "failed to execute command id %u\n", engine_command);
4653 		return -EINVAL;
4654 	}
4655 }
4656 
4657 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
4658 {
4659 	u32 wait_timeout_ms;
4660 
4661 	if (hdev->pldm)
4662 		wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC;
4663 	else
4664 		wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC;
4665 
4666 	if (fw_reset)
4667 		goto skip_engines;
4668 
4669 	gaudi2_stop_dma_qmans(hdev);
4670 	gaudi2_stop_mme_qmans(hdev);
4671 	gaudi2_stop_tpc_qmans(hdev);
4672 	gaudi2_stop_rot_qmans(hdev);
4673 	gaudi2_stop_nic_qmans(hdev);
4674 	msleep(wait_timeout_ms);
4675 
4676 	gaudi2_halt_arcs(hdev);
4677 	gaudi2_dma_stall(hdev);
4678 	gaudi2_mme_stall(hdev);
4679 	gaudi2_tpc_stall(hdev);
4680 	gaudi2_rotator_stall(hdev);
4681 
4682 	msleep(wait_timeout_ms);
4683 
4684 	gaudi2_stop_dec(hdev);
4685 
4686 	/*
4687 	 * in case of soft reset do a manual flush for QMANs (currently called
4688 	 * only for NIC QMANs
4689 	 */
4690 	if (!hard_reset)
4691 		gaudi2_nic_qmans_manual_flush(hdev);
4692 
4693 	gaudi2_disable_dma_qmans(hdev);
4694 	gaudi2_disable_mme_qmans(hdev);
4695 	gaudi2_disable_tpc_qmans(hdev);
4696 	gaudi2_disable_rot_qmans(hdev);
4697 	gaudi2_disable_nic_qmans(hdev);
4698 	gaudi2_disable_timestamp(hdev);
4699 
4700 skip_engines:
4701 	if (hard_reset) {
4702 		gaudi2_disable_msix(hdev);
4703 		return;
4704 	}
4705 
4706 	gaudi2_sync_irqs(hdev);
4707 }
4708 
4709 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev)
4710 {
4711 	struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
4712 
4713 	pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;
4714 	pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0;
4715 	pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1;
4716 	pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0;
4717 	pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1;
4718 	pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC;
4719 }
4720 
4721 static void gaudi2_init_firmware_loader(struct hl_device *hdev)
4722 {
4723 	struct fw_load_mgr *fw_loader = &hdev->fw_loader;
4724 	struct dynamic_fw_load_mgr *dynamic_loader;
4725 	struct cpu_dyn_regs *dyn_regs;
4726 
4727 	/* fill common fields */
4728 	fw_loader->fw_comp_loaded = FW_TYPE_NONE;
4729 	fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE;
4730 	fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE;
4731 	fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC;
4732 	fw_loader->skip_bmc = false;
4733 	fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;
4734 	fw_loader->dram_bar_id = DRAM_BAR_ID;
4735 	fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC;
4736 
4737 	/* here we update initial values for few specific dynamic regs (as
4738 	 * before reading the first descriptor from FW those value has to be
4739 	 * hard-coded). in later stages of the protocol those values will be
4740 	 * updated automatically by reading the FW descriptor so data there
4741 	 * will always be up-to-date
4742 	 */
4743 	dynamic_loader = &hdev->fw_loader.dynamic_loader;
4744 	dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;
4745 	dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);
4746 	dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);
4747 	dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC;
4748 }
4749 
4750 static int gaudi2_init_cpu(struct hl_device *hdev)
4751 {
4752 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4753 	int rc;
4754 
4755 	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
4756 		return 0;
4757 
4758 	if (gaudi2->hw_cap_initialized & HW_CAP_CPU)
4759 		return 0;
4760 
4761 	rc = hl_fw_init_cpu(hdev);
4762 	if (rc)
4763 		return rc;
4764 
4765 	gaudi2->hw_cap_initialized |= HW_CAP_CPU;
4766 
4767 	return 0;
4768 }
4769 
4770 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
4771 {
4772 	struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
4773 	struct asic_fixed_properties *prop = &hdev->asic_prop;
4774 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4775 	struct cpu_dyn_regs *dyn_regs;
4776 	struct hl_eq *eq;
4777 	u32 status;
4778 	int err;
4779 
4780 	if (!hdev->cpu_queues_enable)
4781 		return 0;
4782 
4783 	if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)
4784 		return 0;
4785 
4786 	eq = &hdev->event_queue;
4787 
4788 	dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4789 
4790 	WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
4791 	WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
4792 
4793 	WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
4794 	WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
4795 
4796 	WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address));
4797 	WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address));
4798 
4799 	WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
4800 	WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
4801 	WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
4802 
4803 	/* Used for EQ CI */
4804 	WREG32(mmCPU_IF_EQ_RD_OFFS, 0);
4805 
4806 	WREG32(mmCPU_IF_PF_PQ_PI, 0);
4807 
4808 	WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP);
4809 
4810 	/* Let the ARC know we are ready as it is now handling those queues  */
4811 
4812 	WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),
4813 		gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id);
4814 
4815 	err = hl_poll_timeout(
4816 		hdev,
4817 		mmCPU_IF_QUEUE_INIT,
4818 		status,
4819 		(status == PQ_INIT_STATUS_READY_FOR_HOST),
4820 		1000,
4821 		cpu_timeout);
4822 
4823 	if (err) {
4824 		dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n");
4825 		return -EIO;
4826 	}
4827 
4828 	/* update FW application security bits */
4829 	if (prop->fw_cpu_boot_dev_sts0_valid)
4830 		prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);
4831 
4832 	if (prop->fw_cpu_boot_dev_sts1_valid)
4833 		prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);
4834 
4835 	gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q;
4836 	return 0;
4837 }
4838 
4839 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base,
4840 				u32 queue_id_base)
4841 {
4842 	struct hl_hw_queue *q;
4843 	u32 pq_id, pq_offset;
4844 
4845 	for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) {
4846 		q = &hdev->kernel_queues[queue_id_base + pq_id];
4847 		pq_offset = pq_id * 4;
4848 
4849 		WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset,
4850 				lower_32_bits(q->bus_address));
4851 		WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset,
4852 				upper_32_bits(q->bus_address));
4853 		WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH));
4854 		WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0);
4855 		WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0);
4856 	}
4857 }
4858 
4859 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base)
4860 {
4861 	u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi;
4862 
4863 	mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
4864 	mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
4865 	so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4866 	so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4867 
4868 	for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) {
4869 		cp_offset = cp_id * 4;
4870 
4871 		WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo);
4872 		WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset,	mtr_base_hi);
4873 		WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset,	so_base_lo);
4874 		WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset,	so_base_hi);
4875 	}
4876 
4877 	/* allow QMANs to accept work from ARC CQF */
4878 	WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1));
4879 }
4880 
4881 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base,
4882 				u32 queue_id_base)
4883 {
4884 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
4885 	u32 pq_id, pq_offset, so_base_lo, so_base_hi;
4886 
4887 	so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4888 	so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4889 
4890 	for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) {
4891 		pq_offset = pq_id * 4;
4892 
4893 		/* Configure QMAN HBW to scratchpad as it is not needed */
4894 		WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset,
4895 				lower_32_bits(gaudi2->scratchpad_bus_address));
4896 		WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset,
4897 				upper_32_bits(gaudi2->scratchpad_bus_address));
4898 		WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset,
4899 				ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry)));
4900 
4901 		WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0);
4902 		WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA);
4903 		WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo);
4904 		WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi);
4905 	}
4906 
4907 	/* Enable QMAN H/W completion */
4908 	WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);
4909 }
4910 
4911 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base)
4912 {
4913 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4914 	u32 sp_reg_addr;
4915 
4916 	switch (queue_id_base) {
4917 	case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3:
4918 		fallthrough;
4919 	case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3:
4920 		fallthrough;
4921 	case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3:
4922 		fallthrough;
4923 	case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3:
4924 		fallthrough;
4925 	case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3:
4926 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);
4927 		break;
4928 	case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3:
4929 		fallthrough;
4930 	case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3:
4931 		fallthrough;
4932 	case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3:
4933 		fallthrough;
4934 	case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3:
4935 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);
4936 		break;
4937 	case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3:
4938 		fallthrough;
4939 	case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3:
4940 		fallthrough;
4941 	case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3:
4942 		fallthrough;
4943 	case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3:
4944 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl);
4945 		break;
4946 	case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3:
4947 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl);
4948 		break;
4949 	case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3:
4950 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl);
4951 		break;
4952 	default:
4953 		dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base);
4954 		return 0;
4955 	}
4956 
4957 	return sp_reg_addr;
4958 }
4959 
4960 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base,
4961 					u32 queue_id_base)
4962 {
4963 	u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset;
4964 	int map_table_entry;
4965 
4966 	WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot);
4967 
4968 	irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base);
4969 	WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset));
4970 	WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset));
4971 
4972 	map_table_entry = gaudi2_qman_async_event_id[queue_id_base];
4973 	WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET,
4974 		gaudi2_irq_map_table[map_table_entry].cpu_id);
4975 
4976 	WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK);
4977 
4978 	WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT);
4979 	WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0);
4980 	WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0);
4981 
4982 	/* Enable the QMAN channel.
4983 	 * PDMA QMAN configuration is different, as we do not allow user to
4984 	 * access some of the CPs.
4985 	 * PDMA0: CP2/3 are reserved for the ARC usage.
4986 	 * PDMA1: CP1/2/3 are reserved for the ARC usage.
4987 	 */
4988 	if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0])
4989 		WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE);
4990 	else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0])
4991 		WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE);
4992 	else
4993 		WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE);
4994 }
4995 
4996 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base,
4997 		u32 queue_id_base)
4998 {
4999 	u32 pq_id;
5000 
5001 	for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++)
5002 		hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION;
5003 
5004 	gaudi2_init_qman_pq(hdev, reg_base, queue_id_base);
5005 	gaudi2_init_qman_cp(hdev, reg_base);
5006 	gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base);
5007 	gaudi2_init_qman_common(hdev, reg_base, queue_id_base);
5008 }
5009 
5010 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base,
5011 				u32 dma_core_id, bool is_secure)
5012 {
5013 	u32 prot, irq_handler_offset;
5014 	struct cpu_dyn_regs *dyn_regs;
5015 	int map_table_entry;
5016 
5017 	prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT;
5018 	if (is_secure)
5019 		prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT;
5020 
5021 	WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot);
5022 
5023 	dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
5024 	irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);
5025 
5026 	WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET,
5027 			lower_32_bits(CFG_BASE + irq_handler_offset));
5028 
5029 	WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET,
5030 			upper_32_bits(CFG_BASE + irq_handler_offset));
5031 
5032 	map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id];
5033 	WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET,
5034 		gaudi2_irq_map_table[map_table_entry].cpu_id);
5035 
5036 	/* Enable the DMA channel */
5037 	WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT);
5038 }
5039 
5040 static void gaudi2_init_kdma(struct hl_device *hdev)
5041 {
5042 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5043 	u32 reg_base;
5044 
5045 	if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA)
5046 		return;
5047 
5048 	reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA];
5049 
5050 	gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true);
5051 
5052 	gaudi2->hw_cap_initialized |= HW_CAP_KDMA;
5053 }
5054 
5055 static void gaudi2_init_pdma(struct hl_device *hdev)
5056 {
5057 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5058 	u32 reg_base;
5059 
5060 	if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK)
5061 		return;
5062 
5063 	reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0];
5064 	gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false);
5065 
5066 	reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0];
5067 	gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0);
5068 
5069 	reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1];
5070 	gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false);
5071 
5072 	reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0];
5073 	gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0);
5074 
5075 	gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK;
5076 }
5077 
5078 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq)
5079 {
5080 	u32 reg_base, base_edma_core_id, base_edma_qman_id;
5081 
5082 	base_edma_core_id = DMA_CORE_ID_EDMA0 + seq;
5083 	base_edma_qman_id = edma_stream_base[seq];
5084 
5085 	reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id];
5086 	gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false);
5087 
5088 	reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id];
5089 	gaudi2_init_qman(hdev, reg_base, base_edma_qman_id);
5090 }
5091 
5092 static void gaudi2_init_edma(struct hl_device *hdev)
5093 {
5094 	struct asic_fixed_properties *prop = &hdev->asic_prop;
5095 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5096 	int dcore, inst;
5097 
5098 	if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK)
5099 		return;
5100 
5101 	for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
5102 		for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
5103 			u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
5104 
5105 			if (!(prop->edma_enabled_mask & BIT(seq)))
5106 				continue;
5107 
5108 			gaudi2_init_edma_instance(hdev, seq);
5109 
5110 			gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq);
5111 		}
5112 	}
5113 }
5114 
5115 /*
5116  * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5117  * @hdev: pointer to habanalabs device structure.
5118  * @sob_id: sync object ID.
5119  * @first_mon_id: ID of first monitor out of 3 consecutive monitors.
5120  * @interrupt_id: interrupt ID.
5121  *
5122  * Some initiators cannot have HBW address in their completion address registers, and thus cannot
5123  * write directly to the HBW host memory of the virtual MSI-X doorbell.
5124  * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write.
5125  *
5126  * The mechanism in the sync manager block is composed of a master monitor with 3 messages.
5127  * In addition to the HBW write, the other 2 messages are for preparing the monitor to next
5128  * completion, by decrementing the sync object value and re-arming the monitor.
5129  */
5130 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id,
5131 							u32 first_mon_id, u32 interrupt_id)
5132 {
5133 	u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config;
5134 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5135 	u64 addr;
5136 	u8 mask;
5137 
5138 	/* Reset the SOB value */
5139 	sob_offset = sob_id * sizeof(u32);
5140 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0);
5141 
5142 	/* Configure 3 monitors:
5143 	 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor)
5144 	 * 2. Decrement SOB value by 1.
5145 	 * 3. Re-arm the master monitor.
5146 	 */
5147 
5148 	first_mon_offset = first_mon_id * sizeof(u32);
5149 
5150 	/* 2nd monitor: Decrement SOB value by 1 */
5151 	mon_offset = first_mon_offset + sizeof(u32);
5152 
5153 	addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
5154 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
5155 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
5156 
5157 	payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */
5158 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) |
5159 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1);
5160 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
5161 
5162 	/* 3rd monitor: Re-arm the master monitor */
5163 	mon_offset = first_mon_offset + 2 * sizeof(u32);
5164 
5165 	addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset;
5166 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
5167 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
5168 
5169 	sob_group = sob_id / 8;
5170 	mask = ~BIT(sob_id & 0x7);
5171 	mode = 0; /* comparison mode is "greater than or equal to" */
5172 	arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) |
5173 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) |
5174 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) |
5175 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1);
5176 
5177 	payload = arm;
5178 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
5179 
5180 	/* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */
5181 	mon_offset = first_mon_offset;
5182 
5183 	config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */
5184 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config);
5185 
5186 	addr = gaudi2->virt_msix_db_dma_addr;
5187 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
5188 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
5189 
5190 	payload = interrupt_id;
5191 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
5192 
5193 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm);
5194 }
5195 
5196 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev)
5197 {
5198 	u32 decoder_id, sob_id, first_mon_id, interrupt_id;
5199 	struct asic_fixed_properties *prop = &hdev->asic_prop;
5200 
5201 	/* Decoder normal/abnormal interrupts */
5202 	for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) {
5203 		if (!(prop->decoder_enabled_mask & BIT(decoder_id)))
5204 			continue;
5205 
5206 		sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
5207 		first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id;
5208 		interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id;
5209 		gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
5210 
5211 		sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id;
5212 		first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id;
5213 		interrupt_id += 1;
5214 		gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
5215 	}
5216 }
5217 
5218 static void gaudi2_init_sm(struct hl_device *hdev)
5219 {
5220 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5221 	u64 cq_address;
5222 	u32 reg_val;
5223 	int i;
5224 
5225 	/* Enable HBW/LBW CQ for completion monitors */
5226 	reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
5227 	reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1);
5228 
5229 	for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++)
5230 		WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
5231 
5232 	/* Enable only HBW CQ for KDMA completion monitor */
5233 	reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
5234 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
5235 
5236 	/* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */
5237 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr));
5238 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr));
5239 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION);
5240 
5241 	for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) {
5242 		cq_address =
5243 			hdev->completion_queue[i].bus_address;
5244 
5245 		WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i),
5246 							lower_32_bits(cq_address));
5247 		WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i),
5248 							upper_32_bits(cq_address));
5249 		WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i),
5250 							ilog2(HL_CQ_SIZE_IN_BYTES));
5251 	}
5252 
5253 	/* Configure kernel ASID and MMU BP*/
5254 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000);
5255 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0);
5256 
5257 	/* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */
5258 	gaudi2_prepare_sm_for_virt_msix_db(hdev);
5259 }
5260 
5261 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base)
5262 {
5263 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5264 	u32 reg_val;
5265 	int i;
5266 
5267 	reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0);
5268 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1);
5269 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1);
5270 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1);
5271 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1);
5272 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1);
5273 
5274 	WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val);
5275 	WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF);
5276 
5277 	for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) {
5278 		WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i);
5279 		WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]);
5280 	}
5281 }
5282 
5283 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id,
5284 							bool config_qman_only)
5285 {
5286 	u32 queue_id_base, reg_base;
5287 
5288 	switch (dcore_id) {
5289 	case 0:
5290 		queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0;
5291 		break;
5292 	case 1:
5293 		queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
5294 		break;
5295 	case 2:
5296 		queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
5297 		break;
5298 	case 3:
5299 		queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
5300 		break;
5301 	default:
5302 		dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id);
5303 		return;
5304 	}
5305 
5306 	if (!config_qman_only) {
5307 		reg_base = gaudi2_mme_acc_blocks_bases[dcore_id];
5308 		gaudi2_init_mme_acc(hdev, reg_base);
5309 	}
5310 
5311 	reg_base = gaudi2_qm_blocks_bases[queue_id_base];
5312 	gaudi2_init_qman(hdev, reg_base, queue_id_base);
5313 }
5314 
5315 static void gaudi2_init_mme(struct hl_device *hdev)
5316 {
5317 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5318 	int i;
5319 
5320 	if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK)
5321 		return;
5322 
5323 	for (i = 0 ; i < NUM_OF_DCORES ; i++) {
5324 		gaudi2_init_dcore_mme(hdev, i, false);
5325 
5326 		gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i);
5327 	}
5328 }
5329 
5330 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base)
5331 {
5332 	/* Mask arithmetic and QM interrupts in TPC */
5333 	WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE);
5334 
5335 	/* Set 16 cache lines */
5336 	WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET,
5337 			2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT);
5338 }
5339 
5340 struct gaudi2_tpc_init_cfg_data {
5341 	enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES];
5342 };
5343 
5344 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst,
5345 					u32 offset, struct iterate_module_ctx *ctx)
5346 {
5347 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5348 	struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data;
5349 	u32 queue_id_base;
5350 	u8 seq;
5351 
5352 	queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN);
5353 
5354 	if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1))
5355 		/* gets last sequence number */
5356 		seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE;
5357 	else
5358 		seq = dcore * NUM_OF_TPC_PER_DCORE + inst;
5359 
5360 	gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset);
5361 	gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base);
5362 
5363 	gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq);
5364 }
5365 
5366 static void gaudi2_init_tpc(struct hl_device *hdev)
5367 {
5368 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5369 	struct gaudi2_tpc_init_cfg_data init_cfg_data;
5370 	struct iterate_module_ctx tpc_iter;
5371 
5372 	if (!hdev->asic_prop.tpc_enabled_mask)
5373 		return;
5374 
5375 	if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK)
5376 		return;
5377 
5378 	init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0;
5379 	init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0;
5380 	init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0;
5381 	init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0;
5382 	tpc_iter.fn = &gaudi2_init_tpc_config;
5383 	tpc_iter.data = &init_cfg_data;
5384 	gaudi2_iterate_tpcs(hdev, &tpc_iter);
5385 }
5386 
5387 static void gaudi2_init_rotator(struct hl_device *hdev)
5388 {
5389 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5390 	u32 i, reg_base, queue_id;
5391 
5392 	queue_id = GAUDI2_QUEUE_ID_ROT_0_0;
5393 
5394 	for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
5395 		reg_base = gaudi2_qm_blocks_bases[queue_id];
5396 		gaudi2_init_qman(hdev, reg_base, queue_id);
5397 
5398 		gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i);
5399 	}
5400 }
5401 
5402 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id)
5403 {
5404 	u32 sob_id;
5405 
5406 	/* VCMD normal interrupt */
5407 	sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
5408 	WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR,
5409 			mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32));
5410 	WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE);
5411 
5412 	/* VCMD abnormal interrupt */
5413 	sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id;
5414 	WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR,
5415 			mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32));
5416 	WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE);
5417 }
5418 
5419 static void gaudi2_init_dec(struct hl_device *hdev)
5420 {
5421 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5422 	u32 dcore_id, dec_id, dec_bit;
5423 	u64 base_addr;
5424 
5425 	if (!hdev->asic_prop.decoder_enabled_mask)
5426 		return;
5427 
5428 	if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK)
5429 		return;
5430 
5431 	for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
5432 		for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
5433 			dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id;
5434 
5435 			if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
5436 				continue;
5437 
5438 			base_addr =  mmDCORE0_DEC0_CMD_BASE +
5439 					BRDG_CTRL_BLOCK_OFFSET +
5440 					dcore_id * DCORE_OFFSET +
5441 					dec_id * DCORE_VDEC_OFFSET;
5442 
5443 			gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
5444 
5445 			gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);
5446 		}
5447 
5448 	for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) {
5449 		dec_bit = PCIE_DEC_SHIFT + dec_id;
5450 		if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
5451 			continue;
5452 
5453 		base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET +
5454 				dec_id * DCORE_VDEC_OFFSET;
5455 
5456 		gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
5457 
5458 		gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);
5459 	}
5460 }
5461 
5462 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev,
5463 					u32 stlb_base, u32 asid, u64 phys_addr)
5464 {
5465 	u32 status, timeout_usec;
5466 	int rc;
5467 
5468 	if (hdev->pldm || !hdev->pdev)
5469 		timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC;
5470 	else
5471 		timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
5472 
5473 	WREG32(stlb_base + STLB_ASID_OFFSET, asid);
5474 	WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
5475 	WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT);
5476 	WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000);
5477 
5478 	rc = hl_poll_timeout(
5479 		hdev,
5480 		stlb_base + STLB_BUSY_OFFSET,
5481 		status,
5482 		!(status & 0x80000000),
5483 		1000,
5484 		timeout_usec);
5485 
5486 	if (rc) {
5487 		dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid);
5488 		return rc;
5489 	}
5490 
5491 	return 0;
5492 }
5493 
5494 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base,
5495 					u32 start_offset, u32 inv_start_val,
5496 					u32 flags)
5497 {
5498 	/* clear PMMU mem line cache (only needed in mmu range invalidation) */
5499 	if (flags & MMU_OP_CLEAR_MEMCACHE)
5500 		WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1);
5501 
5502 	if (flags & MMU_OP_SKIP_LOW_CACHE_INV)
5503 		return;
5504 
5505 	WREG32(stlb_base + start_offset, inv_start_val);
5506 }
5507 
5508 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base,
5509 						struct gaudi2_cache_invld_params *inv_params)
5510 {
5511 	u32 status, timeout_usec, start_offset;
5512 	int rc;
5513 
5514 	timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC :
5515 					GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC;
5516 
5517 	/* poll PMMU mem line cache (only needed in mmu range invalidation) */
5518 	if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) {
5519 		rc = hl_poll_timeout(
5520 			hdev,
5521 			mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS,
5522 			status,
5523 			status & 0x1,
5524 			1000,
5525 			timeout_usec);
5526 
5527 		if (rc)
5528 			return rc;
5529 
5530 		/* Need to manually reset the status to 0 */
5531 		WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0);
5532 	}
5533 
5534 	/* Lower cache does not work with cache lines, hence we can skip its
5535 	 * invalidation upon map and invalidate only upon unmap
5536 	 */
5537 	if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV)
5538 		return 0;
5539 
5540 	start_offset = inv_params->range_invalidation ?
5541 			STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET;
5542 
5543 	rc = hl_poll_timeout(
5544 		hdev,
5545 		stlb_base + start_offset,
5546 		status,
5547 		!(status & 0x1),
5548 		1000,
5549 		timeout_usec);
5550 
5551 	return rc;
5552 }
5553 
5554 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id)
5555 {
5556 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5557 	u32 hw_cap;
5558 
5559 	hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id);
5560 
5561 	if (gaudi2->hw_cap_initialized & hw_cap)
5562 		return true;
5563 
5564 	return false;
5565 }
5566 
5567 /* this function shall be called only for HMMUs for which capability bit is set */
5568 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id)
5569 {
5570 	u32 offset;
5571 
5572 	offset =  (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET);
5573 	return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset);
5574 }
5575 
5576 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base,
5577 						struct gaudi2_cache_invld_params *inv_params)
5578 {
5579 	u32 start_offset;
5580 
5581 	if (inv_params->range_invalidation) {
5582 		/* Set the addresses range
5583 		 * Note: that the start address we set in register, is not included in
5584 		 * the range of the invalidation, by design.
5585 		 * that's why we need to set lower address than the one we actually
5586 		 * want to be included in the range invalidation.
5587 		 */
5588 		u64 start = inv_params->start_va - 1;
5589 
5590 		start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET;
5591 
5592 		WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET,
5593 				start >> MMU_RANGE_INV_VA_LSB_SHIFT);
5594 
5595 		WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET,
5596 				start >> MMU_RANGE_INV_VA_MSB_SHIFT);
5597 
5598 		WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET,
5599 				inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT);
5600 
5601 		WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET,
5602 				inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT);
5603 	} else {
5604 		start_offset = STLB_INV_ALL_START_OFFSET;
5605 	}
5606 
5607 	gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset,
5608 						inv_params->inv_start_val, inv_params->flags);
5609 }
5610 
5611 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev,
5612 						int dcore_id, int hmmu_id,
5613 						struct gaudi2_cache_invld_params *inv_params)
5614 {
5615 	u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id);
5616 
5617 	gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params);
5618 }
5619 
5620 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev,
5621 						int dcore_id, int hmmu_id,
5622 						struct gaudi2_cache_invld_params *inv_params)
5623 {
5624 	u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id);
5625 
5626 	return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params);
5627 }
5628 
5629 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev,
5630 						struct gaudi2_cache_invld_params *inv_params)
5631 {
5632 	int dcore_id, hmmu_id;
5633 
5634 	/* first send all invalidation commands */
5635 	for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
5636 		for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) {
5637 			if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
5638 				continue;
5639 
5640 			gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params);
5641 		}
5642 	}
5643 
5644 	/* next, poll all invalidations status */
5645 	for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
5646 		for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) {
5647 			int rc;
5648 
5649 			if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
5650 				continue;
5651 
5652 			rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id,
5653 										inv_params);
5654 			if (rc)
5655 				return rc;
5656 		}
5657 	}
5658 
5659 	return 0;
5660 }
5661 
5662 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
5663 {
5664 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5665 	struct gaudi2_cache_invld_params invld_params;
5666 	int rc = 0;
5667 
5668 	if (hdev->reset_info.hard_reset_pending)
5669 		return rc;
5670 
5671 	invld_params.range_invalidation = false;
5672 	invld_params.inv_start_val = 1;
5673 
5674 	if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
5675 		invld_params.flags = flags;
5676 		gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
5677 		rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
5678 										&invld_params);
5679 	} else if (flags & MMU_OP_PHYS_PACK) {
5680 		invld_params.flags = 0;
5681 		rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
5682 	}
5683 
5684 	return rc;
5685 }
5686 
5687 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
5688 				u32 flags, u32 asid, u64 va, u64 size)
5689 {
5690 	struct gaudi2_cache_invld_params invld_params = {0};
5691 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5692 	u64 start_va, end_va;
5693 	u32 inv_start_val;
5694 	int rc = 0;
5695 
5696 	if (hdev->reset_info.hard_reset_pending)
5697 		return 0;
5698 
5699 	inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT |
5700 			1 << MMU_RANGE_INV_ASID_EN_SHIFT |
5701 			asid << MMU_RANGE_INV_ASID_SHIFT);
5702 	start_va = va;
5703 	end_va = start_va + size;
5704 
5705 	if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
5706 		/* As range invalidation does not support zero address we will
5707 		 * do full invalidation in this case
5708 		 */
5709 		if (start_va) {
5710 			invld_params.range_invalidation = true;
5711 			invld_params.start_va = start_va;
5712 			invld_params.end_va = end_va;
5713 			invld_params.inv_start_val = inv_start_val;
5714 			invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE;
5715 		} else {
5716 			invld_params.range_invalidation = false;
5717 			invld_params.inv_start_val = 1;
5718 			invld_params.flags = flags;
5719 		}
5720 
5721 
5722 		gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
5723 		rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
5724 										&invld_params);
5725 		if (rc)
5726 			return rc;
5727 
5728 	} else if (flags & MMU_OP_PHYS_PACK) {
5729 		invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va);
5730 		invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va);
5731 		invld_params.inv_start_val = inv_start_val;
5732 		invld_params.flags = flags;
5733 		rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
5734 	}
5735 
5736 	return rc;
5737 }
5738 
5739 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base)
5740 {
5741 	struct asic_fixed_properties *prop = &hdev->asic_prop;
5742 	u64 hop0_addr;
5743 	u32 asid, max_asid = prop->max_asid;
5744 	int rc;
5745 
5746 	/* it takes too much time to init all of the ASIDs on palladium */
5747 	if (hdev->pldm)
5748 		max_asid = min((u32) 8, max_asid);
5749 
5750 	for (asid = 0 ; asid < max_asid ; asid++) {
5751 		hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr;
5752 		rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr);
5753 		if (rc) {
5754 			dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid);
5755 			return rc;
5756 		}
5757 	}
5758 
5759 	return 0;
5760 }
5761 
5762 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base)
5763 {
5764 	u32 status, timeout_usec;
5765 	int rc;
5766 
5767 	if (hdev->pldm || !hdev->pdev)
5768 		timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC;
5769 	else
5770 		timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC;
5771 
5772 	WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1);
5773 
5774 	rc = hl_poll_timeout(
5775 		hdev,
5776 		stlb_base + STLB_SRAM_INIT_OFFSET,
5777 		status,
5778 		!status,
5779 		1000,
5780 		timeout_usec);
5781 
5782 	if (rc)
5783 		dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n");
5784 
5785 	rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base);
5786 	if (rc)
5787 		return rc;
5788 
5789 	WREG32(mmu_base + MMU_BYPASS_OFFSET, 0);
5790 
5791 	rc = hl_poll_timeout(
5792 		hdev,
5793 		stlb_base + STLB_INV_ALL_START_OFFSET,
5794 		status,
5795 		!status,
5796 		1000,
5797 		timeout_usec);
5798 
5799 	if (rc)
5800 		dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n");
5801 
5802 	WREG32(mmu_base + MMU_ENABLE_OFFSET, 1);
5803 
5804 	return rc;
5805 }
5806 
5807 static int gaudi2_pci_mmu_init(struct hl_device *hdev)
5808 {
5809 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5810 	u32 mmu_base, stlb_base;
5811 	int rc;
5812 
5813 	if (gaudi2->hw_cap_initialized & HW_CAP_PMMU)
5814 		return 0;
5815 
5816 	mmu_base = mmPMMU_HBW_MMU_BASE;
5817 	stlb_base = mmPMMU_HBW_STLB_BASE;
5818 
5819 	RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
5820 		(0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) |
5821 		(5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) |
5822 		(4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) |
5823 		(5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) |
5824 		(5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT),
5825 		PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK |
5826 		PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK |
5827 		PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK |
5828 		PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK |
5829 		PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK);
5830 
5831 	WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0);
5832 
5833 	if (PAGE_SIZE == SZ_64K) {
5834 		/* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */
5835 		RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET,
5836 			FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) |
5837 			FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) |
5838 			FIELD_PREP(
5839 				DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK,
5840 				1),
5841 			DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK |
5842 			DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK |
5843 			DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK);
5844 	}
5845 
5846 	WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK);
5847 
5848 	rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5849 	if (rc)
5850 		return rc;
5851 
5852 	gaudi2->hw_cap_initialized |= HW_CAP_PMMU;
5853 
5854 	return 0;
5855 }
5856 
5857 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id,
5858 				int hmmu_id)
5859 {
5860 	struct asic_fixed_properties *prop = &hdev->asic_prop;
5861 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5862 	u32 offset, mmu_base, stlb_base, hw_cap;
5863 	u8 dmmu_seq;
5864 	int rc;
5865 
5866 	dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id;
5867 	hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq;
5868 
5869 	/*
5870 	 * return if DMMU is already initialized or if it's not out of
5871 	 * isolation (due to cluster binning)
5872 	 */
5873 	if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq)))
5874 		return 0;
5875 
5876 	offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET);
5877 	mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset;
5878 	stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset;
5879 
5880 	RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */,
5881 			MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK);
5882 
5883 	RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
5884 		FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) |
5885 		FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) |
5886 		FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) |
5887 		FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) |
5888 		FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3),
5889 			DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK |
5890 			DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK |
5891 			DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK |
5892 			DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK |
5893 			DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK);
5894 
5895 	RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1,
5896 			STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK);
5897 
5898 	WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK);
5899 
5900 	rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5901 	if (rc)
5902 		return rc;
5903 
5904 	gaudi2->hw_cap_initialized |= hw_cap;
5905 
5906 	return 0;
5907 }
5908 
5909 static int gaudi2_hbm_mmu_init(struct hl_device *hdev)
5910 {
5911 	int rc, dcore_id, hmmu_id;
5912 
5913 	for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
5914 		for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) {
5915 			rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id);
5916 			if (rc)
5917 				return rc;
5918 		}
5919 
5920 	return 0;
5921 }
5922 
5923 static int gaudi2_mmu_init(struct hl_device *hdev)
5924 {
5925 	int rc;
5926 
5927 	rc = gaudi2_pci_mmu_init(hdev);
5928 	if (rc)
5929 		return rc;
5930 
5931 	rc = gaudi2_hbm_mmu_init(hdev);
5932 	if (rc)
5933 		return rc;
5934 
5935 	return 0;
5936 }
5937 
5938 static int gaudi2_hw_init(struct hl_device *hdev)
5939 {
5940 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
5941 	int rc;
5942 
5943 	/* Let's mark in the H/W that we have reached this point. We check
5944 	 * this value in the reset_before_init function to understand whether
5945 	 * we need to reset the chip before doing H/W init. This register is
5946 	 * cleared by the H/W upon H/W reset
5947 	 */
5948 	WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
5949 
5950 	/* Perform read from the device to make sure device is up */
5951 	RREG32(mmHW_STATE);
5952 
5953 	/* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE.
5954 	 * So we set it here and if anyone tries to move it later to
5955 	 * a different address, there will be an error
5956 	 */
5957 	if (hdev->asic_prop.iatu_done_by_fw)
5958 		gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE;
5959 
5960 	/*
5961 	 * Before pushing u-boot/linux to device, need to set the hbm bar to
5962 	 * base address of dram
5963 	 */
5964 	if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
5965 		dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n");
5966 		return -EIO;
5967 	}
5968 
5969 	rc = gaudi2_init_cpu(hdev);
5970 	if (rc) {
5971 		dev_err(hdev->dev, "failed to initialize CPU\n");
5972 		return rc;
5973 	}
5974 
5975 	gaudi2_init_scrambler_hbm(hdev);
5976 	gaudi2_init_kdma(hdev);
5977 
5978 	rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC);
5979 	if (rc) {
5980 		dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc);
5981 		return rc;
5982 	}
5983 
5984 	rc = gaudi2->cpucp_info_get(hdev);
5985 	if (rc) {
5986 		dev_err(hdev->dev, "Failed to get cpucp info\n");
5987 		return rc;
5988 	}
5989 
5990 	rc = gaudi2_mmu_init(hdev);
5991 	if (rc)
5992 		return rc;
5993 
5994 	gaudi2_init_pdma(hdev);
5995 	gaudi2_init_edma(hdev);
5996 	gaudi2_init_sm(hdev);
5997 	gaudi2_init_tpc(hdev);
5998 	gaudi2_init_mme(hdev);
5999 	gaudi2_init_rotator(hdev);
6000 	gaudi2_init_dec(hdev);
6001 	gaudi2_enable_timestamp(hdev);
6002 
6003 	rc = gaudi2_coresight_init(hdev);
6004 	if (rc)
6005 		goto disable_queues;
6006 
6007 	rc = gaudi2_enable_msix(hdev);
6008 	if (rc)
6009 		goto disable_queues;
6010 
6011 	/* Perform read from the device to flush all configuration */
6012 	RREG32(mmHW_STATE);
6013 
6014 	return 0;
6015 
6016 disable_queues:
6017 	gaudi2_disable_dma_qmans(hdev);
6018 	gaudi2_disable_mme_qmans(hdev);
6019 	gaudi2_disable_tpc_qmans(hdev);
6020 	gaudi2_disable_rot_qmans(hdev);
6021 	gaudi2_disable_nic_qmans(hdev);
6022 
6023 	gaudi2_disable_timestamp(hdev);
6024 
6025 	return rc;
6026 }
6027 
6028 /**
6029  * gaudi2_send_hard_reset_cmd - common function to handle reset
6030  *
6031  * @hdev: pointer to the habanalabs device structure
6032  *
6033  * This function handles the various possible scenarios for reset.
6034  * It considers if reset is handled by driver\FW and what FW components are loaded
6035  */
6036 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev)
6037 {
6038 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6039 	bool heartbeat_reset, preboot_only, cpu_initialized = false;
6040 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6041 	u32 cpu_boot_status;
6042 
6043 	preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU);
6044 	heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT);
6045 
6046 	/*
6047 	 * Handle corner case where failure was at cpu management app load,
6048 	 * and driver didn't detect any failure while loading the FW,
6049 	 * then at such scenario driver will send only HALT_MACHINE
6050 	 * and no one will respond to this request since FW already back to preboot
6051 	 * and it cannot handle such cmd.
6052 	 * In this case next time the management app loads it'll check on events register
6053 	 * which will still have the halt indication, and will reboot the device.
6054 	 * The solution is to let preboot clear all relevant registers before next boot
6055 	 * once driver send COMMS_RST_DEV.
6056 	 */
6057 	cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS);
6058 
6059 	if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) &&
6060 			(cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL))
6061 		cpu_initialized = true;
6062 
6063 	/*
6064 	 * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways:
6065 	 * 1. FW reset: FW initiate the reset sequence
6066 	 * 2. driver reset: FW will start HALT sequence (the preparations for the
6067 	 *                  reset but not the reset itself as it is not implemented
6068 	 *                  on their part) and LKD will wait to let FW complete the
6069 	 *                  sequence before issuing the reset
6070 	 */
6071 	if (!preboot_only && cpu_initialized) {
6072 		WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq),
6073 			gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id);
6074 
6075 		msleep(GAUDI2_CPU_RESET_WAIT_MSEC);
6076 	}
6077 
6078 	/*
6079 	 * When working with preboot (without Linux/Boot fit) we can
6080 	 * communicate only using the COMMS commands to issue halt/reset.
6081 	 *
6082 	 * For the case in which we are working with Linux/Bootfit this is a hail-mary
6083 	 * attempt to revive the card in the small chance that the f/w has
6084 	 * experienced a watchdog event, which caused it to return back to preboot.
6085 	 * In that case, triggering reset through GIC won't help. We need to
6086 	 * trigger the reset as if Linux wasn't loaded.
6087 	 *
6088 	 * We do it only if the reset cause was HB, because that would be the
6089 	 * indication of such an event.
6090 	 *
6091 	 * In case watchdog hasn't expired but we still got HB, then this won't
6092 	 * do any damage.
6093 	 */
6094 
6095 	if (heartbeat_reset || preboot_only || !cpu_initialized) {
6096 		if (hdev->asic_prop.hard_reset_done_by_fw)
6097 			hl_fw_ask_hard_reset_without_linux(hdev);
6098 		else
6099 			hl_fw_ask_halt_machine_without_linux(hdev);
6100 	}
6101 }
6102 
6103 /**
6104  * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6105  *
6106  * @hdev: pointer to the habanalabs device structure
6107  *
6108  * This function executes hard reset based on if driver/FW should do the reset
6109  */
6110 static void gaudi2_execute_hard_reset(struct hl_device *hdev)
6111 {
6112 	if (hdev->asic_prop.hard_reset_done_by_fw) {
6113 		gaudi2_send_hard_reset_cmd(hdev);
6114 		return;
6115 	}
6116 
6117 	/* Set device to handle FLR by H/W as we will put the device
6118 	 * CPU to halt mode
6119 	 */
6120 	WREG32(mmPCIE_AUX_FLR_CTRL,
6121 			(PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK));
6122 
6123 	gaudi2_send_hard_reset_cmd(hdev);
6124 
6125 	WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1);
6126 }
6127 
6128 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us)
6129 {
6130 	int i, rc = 0;
6131 	u32 reg_val;
6132 
6133 	for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
6134 		rc = hl_poll_timeout(
6135 			hdev,
6136 			mmCPU_RST_STATUS_TO_HOST,
6137 			reg_val,
6138 			reg_val == CPU_RST_STATUS_SOFT_RST_DONE,
6139 			1000,
6140 			poll_timeout_us);
6141 
6142 	if (rc)
6143 		dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n",
6144 				reg_val);
6145 	return rc;
6146 }
6147 
6148 /**
6149  * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6150  *
6151  * @hdev: pointer to the habanalabs device structure
6152  * @driver_performs_reset: true if driver should perform reset instead of f/w.
6153  * @poll_timeout_us: time to wait for response from f/w.
6154  *
6155  * This function executes soft reset based on if driver/FW should do the reset
6156  */
6157 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset,
6158 						u32 poll_timeout_us)
6159 {
6160 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6161 	int rc = 0;
6162 
6163 	if (!driver_performs_reset) {
6164 		if (hl_is_fw_sw_ver_below(hdev, 1, 10)) {
6165 			/* set SP to indicate reset request sent to FW */
6166 			if (dyn_regs->cpu_rst_status)
6167 				WREG32(le32_to_cpu(dyn_regs->cpu_rst_status), CPU_RST_STATUS_NA);
6168 			else
6169 				WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA);
6170 			WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq),
6171 				gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id);
6172 
6173 			/* wait for f/w response */
6174 			rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us);
6175 		} else {
6176 			rc = hl_fw_send_soft_reset(hdev);
6177 		}
6178 		return rc;
6179 	}
6180 
6181 	/* Block access to engines, QMANs and SM during reset, these
6182 	 * RRs will be reconfigured after soft reset.
6183 	 * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset.
6184 	 */
6185 	gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1,
6186 					mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE);
6187 
6188 	gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2,
6189 				mmPCIE_MSIX_BASE + HL_BLOCK_SIZE,
6190 				mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE);
6191 
6192 	WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1);
6193 	return 0;
6194 }
6195 
6196 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us)
6197 {
6198 	int i, rc = 0;
6199 	u32 reg_val;
6200 
6201 	/* We poll the BTM done indication multiple times after reset due to
6202 	 * a HW errata 'GAUDI2_0300'
6203 	 */
6204 	for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
6205 		rc = hl_poll_timeout(
6206 			hdev,
6207 			mmPSOC_GLOBAL_CONF_BTM_FSM,
6208 			reg_val,
6209 			reg_val == 0,
6210 			1000,
6211 			poll_timeout_us);
6212 
6213 	if (rc)
6214 		dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val);
6215 }
6216 
6217 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
6218 {
6219 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6220 	u32 poll_timeout_us, reset_sleep_ms;
6221 	bool driver_performs_reset = false;
6222 	int rc;
6223 
6224 	if (hdev->pldm) {
6225 		reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC :
6226 						GAUDI2_PLDM_SRESET_TIMEOUT_MSEC;
6227 		poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC;
6228 	} else {
6229 		reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC;
6230 		poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC;
6231 	}
6232 
6233 	if (fw_reset)
6234 		goto skip_reset;
6235 
6236 	gaudi2_reset_arcs(hdev);
6237 
6238 	if (hard_reset) {
6239 		driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw;
6240 		gaudi2_execute_hard_reset(hdev);
6241 	} else {
6242 		/*
6243 		 * As we have to support also work with preboot only (which does not supports
6244 		 * soft reset) we have to make sure that security is disabled before letting driver
6245 		 * do the reset. user shall control the BFE flags to avoid asking soft reset in
6246 		 * secured device with preboot only.
6247 		 */
6248 		driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU &&
6249 							!hdev->asic_prop.fw_security_enabled);
6250 		rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us);
6251 		if (rc)
6252 			return rc;
6253 	}
6254 
6255 skip_reset:
6256 	if (driver_performs_reset || hard_reset) {
6257 		/*
6258 		 * Instead of waiting for BTM indication we should wait for preboot ready:
6259 		 * Consider the below scenario:
6260 		 * 1. FW update is being triggered
6261 		 *        - setting the dirty bit
6262 		 * 2. hard reset will be triggered due to the dirty bit
6263 		 * 3. FW initiates the reset:
6264 		 *        - dirty bit cleared
6265 		 *        - BTM indication cleared
6266 		 *        - preboot ready indication cleared
6267 		 * 4. during hard reset:
6268 		 *        - BTM indication will be set
6269 		 *        - BIST test performed and another reset triggered
6270 		 * 5. only after this reset the preboot will set the preboot ready
6271 		 *
6272 		 * when polling on BTM indication alone we can lose sync with FW while trying to
6273 		 * communicate with FW that is during reset.
6274 		 * to overcome this we will always wait to preboot ready indication
6275 		 */
6276 
6277 		/* without this sleep reset will not work */
6278 		msleep(reset_sleep_ms);
6279 
6280 		if (hdev->fw_components & FW_TYPE_PREBOOT_CPU)
6281 			hl_fw_wait_preboot_ready(hdev);
6282 		else
6283 			gaudi2_poll_btm_indication(hdev, poll_timeout_us);
6284 	}
6285 
6286 	if (!gaudi2)
6287 		return 0;
6288 
6289 	gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK);
6290 	gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK);
6291 
6292 	/*
6293 	 * Clear NIC capability mask in order for driver to re-configure
6294 	 * NIC QMANs. NIC ports will not be re-configured during soft
6295 	 * reset as we call gaudi2_nic_init only during hard reset
6296 	 */
6297 	gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK);
6298 
6299 	if (hard_reset) {
6300 		gaudi2->hw_cap_initialized &=
6301 			~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK |
6302 			HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q |
6303 			HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK |
6304 			HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA |
6305 			HW_CAP_MME_MASK | HW_CAP_ROT_MASK);
6306 
6307 		memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat));
6308 	} else {
6309 		gaudi2->hw_cap_initialized &=
6310 			~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET |
6311 			HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK |
6312 			HW_CAP_ROT_MASK);
6313 	}
6314 	return 0;
6315 }
6316 
6317 static int gaudi2_suspend(struct hl_device *hdev)
6318 {
6319 	int rc;
6320 
6321 	rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
6322 	if (rc)
6323 		dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
6324 
6325 	return rc;
6326 }
6327 
6328 static int gaudi2_resume(struct hl_device *hdev)
6329 {
6330 	return gaudi2_init_iatu(hdev);
6331 }
6332 
6333 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
6334 		void *cpu_addr, dma_addr_t dma_addr, size_t size)
6335 {
6336 	int rc;
6337 
6338 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
6339 			VM_DONTCOPY | VM_NORESERVE);
6340 
6341 #ifdef _HAS_DMA_MMAP_COHERENT
6342 
6343 	rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size);
6344 	if (rc)
6345 		dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
6346 
6347 #else
6348 
6349 	rc = remap_pfn_range(vma, vma->vm_start,
6350 				virt_to_phys(cpu_addr) >> PAGE_SHIFT,
6351 				size, vma->vm_page_prot);
6352 	if (rc)
6353 		dev_err(hdev->dev, "remap_pfn_range error %d", rc);
6354 
6355 #endif
6356 
6357 	return rc;
6358 }
6359 
6360 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id)
6361 {
6362 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6363 	u64 hw_cap_mask = 0;
6364 	u64 hw_tpc_cap_bit = 0;
6365 	u64 hw_nic_cap_bit = 0;
6366 	u64 hw_test_cap_bit = 0;
6367 
6368 	switch (hw_queue_id) {
6369 	case GAUDI2_QUEUE_ID_PDMA_0_0:
6370 	case GAUDI2_QUEUE_ID_PDMA_0_1:
6371 	case GAUDI2_QUEUE_ID_PDMA_1_0:
6372 		hw_cap_mask = HW_CAP_PDMA_MASK;
6373 		break;
6374 	case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3:
6375 		hw_test_cap_bit = HW_CAP_EDMA_SHIFT +
6376 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2);
6377 		break;
6378 	case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3:
6379 		hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE +
6380 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2);
6381 		break;
6382 	case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3:
6383 		hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE +
6384 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2);
6385 		break;
6386 	case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3:
6387 		hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE +
6388 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2);
6389 		break;
6390 
6391 	case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3:
6392 		hw_test_cap_bit = HW_CAP_MME_SHIFT;
6393 		break;
6394 
6395 	case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3:
6396 		hw_test_cap_bit = HW_CAP_MME_SHIFT + 1;
6397 		break;
6398 
6399 	case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3:
6400 		hw_test_cap_bit = HW_CAP_MME_SHIFT + 2;
6401 		break;
6402 
6403 	case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3:
6404 		hw_test_cap_bit = HW_CAP_MME_SHIFT + 3;
6405 		break;
6406 
6407 	case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3:
6408 		hw_tpc_cap_bit = HW_CAP_TPC_SHIFT +
6409 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2);
6410 
6411 		/* special case where cap bit refers to the first queue id */
6412 		if (!hw_tpc_cap_bit)
6413 			return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0));
6414 		break;
6415 
6416 	case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3:
6417 		hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE +
6418 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2);
6419 		break;
6420 
6421 	case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3:
6422 		hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) +
6423 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2);
6424 		break;
6425 
6426 	case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3:
6427 		hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) +
6428 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2);
6429 		break;
6430 
6431 	case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3:
6432 		hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE);
6433 		break;
6434 
6435 	case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3:
6436 		hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2);
6437 		break;
6438 
6439 	case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3:
6440 		hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2);
6441 
6442 		/* special case where cap bit refers to the first queue id */
6443 		if (!hw_nic_cap_bit)
6444 			return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0));
6445 		break;
6446 
6447 	case GAUDI2_QUEUE_ID_CPU_PQ:
6448 		return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q);
6449 
6450 	default:
6451 		return false;
6452 	}
6453 
6454 	if (hw_tpc_cap_bit)
6455 		return  !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit));
6456 
6457 	if (hw_nic_cap_bit)
6458 		return  !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit));
6459 
6460 	if (hw_test_cap_bit)
6461 		hw_cap_mask = BIT_ULL(hw_test_cap_bit);
6462 
6463 	return !!(gaudi2->hw_cap_initialized & hw_cap_mask);
6464 }
6465 
6466 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id)
6467 {
6468 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6469 
6470 	switch (arc_id) {
6471 	case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
6472 	case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
6473 		return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id));
6474 
6475 	case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
6476 		return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));
6477 
6478 	case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
6479 		return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));
6480 
6481 	default:
6482 		return false;
6483 	}
6484 }
6485 
6486 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id)
6487 {
6488 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6489 
6490 	switch (arc_id) {
6491 	case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
6492 	case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
6493 		gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id));
6494 		break;
6495 
6496 	case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
6497 		gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));
6498 		break;
6499 
6500 	case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
6501 		gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));
6502 		break;
6503 
6504 	default:
6505 		return;
6506 	}
6507 }
6508 
6509 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id)
6510 {
6511 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6512 
6513 	switch (arc_id) {
6514 	case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
6515 	case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
6516 		gaudi2->active_hw_arc |= BIT_ULL(arc_id);
6517 		break;
6518 
6519 	case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
6520 		gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0);
6521 		break;
6522 
6523 	case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
6524 		gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0);
6525 		break;
6526 
6527 	default:
6528 		return;
6529 	}
6530 }
6531 
6532 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
6533 {
6534 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
6535 	u32 pq_offset, reg_base, db_reg_offset, db_value;
6536 
6537 	if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) {
6538 		/*
6539 		 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs.
6540 		 * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ
6541 		 * number.
6542 		 */
6543 		pq_offset = (hw_queue_id & 0x3) * 4;
6544 		reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
6545 		db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset;
6546 	} else {
6547 		db_reg_offset = mmCPU_IF_PF_PQ_PI;
6548 	}
6549 
6550 	db_value = pi;
6551 
6552 	/* ring the doorbell */
6553 	WREG32(db_reg_offset, db_value);
6554 
6555 	if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) {
6556 		/* make sure device CPU will read latest data from host */
6557 		mb();
6558 		WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),
6559 			gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id);
6560 	}
6561 }
6562 
6563 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
6564 {
6565 	__le64 *pbd = (__le64 *) bd;
6566 
6567 	/* The QMANs are on the host memory so a simple copy suffice */
6568 	pqe[0] = pbd[0];
6569 	pqe[1] = pbd[1];
6570 }
6571 
6572 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size,
6573 				dma_addr_t *dma_handle, gfp_t flags)
6574 {
6575 	return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags);
6576 }
6577 
6578 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size,
6579 				void *cpu_addr, dma_addr_t dma_handle)
6580 {
6581 	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
6582 }
6583 
6584 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
6585 				u32 timeout, u64 *result)
6586 {
6587 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6588 
6589 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) {
6590 		if (result)
6591 			*result = 0;
6592 		return 0;
6593 	}
6594 
6595 	if (!timeout)
6596 		timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC;
6597 
6598 	return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result);
6599 }
6600 
6601 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size,
6602 				gfp_t mem_flags, dma_addr_t *dma_handle)
6603 {
6604 	if (size > GAUDI2_DMA_POOL_BLK_SIZE)
6605 		return NULL;
6606 
6607 	return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
6608 }
6609 
6610 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr)
6611 {
6612 	dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
6613 }
6614 
6615 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
6616 						dma_addr_t *dma_handle)
6617 {
6618 	return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
6619 }
6620 
6621 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
6622 {
6623 	hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
6624 }
6625 
6626 static dma_addr_t gaudi2_dma_map_single(struct hl_device *hdev, void *addr, int len,
6627 					enum dma_data_direction dir)
6628 {
6629 	dma_addr_t dma_addr;
6630 
6631 	dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir);
6632 	if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr)))
6633 		return 0;
6634 
6635 	return dma_addr;
6636 }
6637 
6638 static void gaudi2_dma_unmap_single(struct hl_device *hdev, dma_addr_t addr, int len,
6639 					enum dma_data_direction dir)
6640 {
6641 	dma_unmap_single(&hdev->pdev->dev, addr, len, dir);
6642 }
6643 
6644 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser)
6645 {
6646 	struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
6647 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6648 
6649 	if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) {
6650 		dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);
6651 		return -EINVAL;
6652 	}
6653 
6654 	/* Just check if CB address is valid */
6655 
6656 	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6657 					parser->user_cb_size,
6658 					asic_prop->sram_user_base_address,
6659 					asic_prop->sram_end_address))
6660 		return 0;
6661 
6662 	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6663 					parser->user_cb_size,
6664 					asic_prop->dram_user_base_address,
6665 					asic_prop->dram_end_address))
6666 		return 0;
6667 
6668 	if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) &&
6669 		hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6670 						parser->user_cb_size,
6671 						asic_prop->dmmu.start_addr,
6672 						asic_prop->dmmu.end_addr))
6673 		return 0;
6674 
6675 	if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) {
6676 		if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
6677 					parser->user_cb_size,
6678 					asic_prop->pmmu.start_addr,
6679 					asic_prop->pmmu.end_addr) ||
6680 			hl_mem_area_inside_range(
6681 					(u64) (uintptr_t) parser->user_cb,
6682 					parser->user_cb_size,
6683 					asic_prop->pmmu_huge.start_addr,
6684 					asic_prop->pmmu_huge.end_addr))
6685 			return 0;
6686 
6687 	} else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) {
6688 		if (!hdev->pdev)
6689 			return 0;
6690 
6691 		if (!device_iommu_mapped(&hdev->pdev->dev))
6692 			return 0;
6693 	}
6694 
6695 	dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n",
6696 		parser->user_cb, parser->user_cb_size);
6697 
6698 	return -EFAULT;
6699 }
6700 
6701 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
6702 {
6703 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6704 
6705 	if (!parser->is_kernel_allocated_cb)
6706 		return gaudi2_validate_cb_address(hdev, parser);
6707 
6708 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
6709 		dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n");
6710 		return -EINVAL;
6711 	}
6712 
6713 	return 0;
6714 }
6715 
6716 static int gaudi2_send_heartbeat(struct hl_device *hdev)
6717 {
6718 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6719 
6720 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
6721 		return 0;
6722 
6723 	return hl_fw_send_heartbeat(hdev);
6724 }
6725 
6726 /* This is an internal helper function, used to update the KDMA mmu props.
6727  * Should be called with a proper kdma lock.
6728  */
6729 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev,
6730 					   bool mmu_bypass, u32 asid)
6731 {
6732 	u32 rw_asid, rw_mmu_bp;
6733 
6734 	rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
6735 		      (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
6736 
6737 	rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) |
6738 			(!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT);
6739 
6740 	WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid);
6741 	WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp);
6742 }
6743 
6744 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id,
6745 						u32 mon_payload, u32 sync_value)
6746 {
6747 	u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm;
6748 	u8 mask;
6749 
6750 	sob_offset = sob_id * 4;
6751 	mon_offset = mon_id * 4;
6752 
6753 	/* Reset the SOB value */
6754 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0);
6755 
6756 	/* Configure this address with CQ_ID 0 because CQ_EN is set */
6757 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id);
6758 
6759 	/* Configure this address with CS index because CQ_EN is set */
6760 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload);
6761 
6762 	sync_group_id = sob_id / 8;
6763 	mask = ~(1 << (sob_id & 0x7));
6764 	mode = 1; /* comparison mode is "equal to" */
6765 
6766 	mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value);
6767 	mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode);
6768 	mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask);
6769 	mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id);
6770 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm);
6771 }
6772 
6773 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */
6774 static int gaudi2_send_job_to_kdma(struct hl_device *hdev,
6775 					u64 src_addr, u64 dst_addr,
6776 					u32 size, bool is_memset)
6777 {
6778 	u32 comp_val, commit_mask, *polling_addr, timeout, status = 0;
6779 	struct hl_cq_entry *cq_base;
6780 	struct hl_cq *cq;
6781 	u64 comp_addr;
6782 	int rc;
6783 
6784 	gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION,
6785 				GAUDI2_RESERVED_MON_KDMA_COMPLETION,
6786 				GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1);
6787 
6788 	comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 +
6789 			(GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32));
6790 
6791 	comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
6792 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
6793 
6794 	WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr));
6795 	WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr));
6796 	WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr));
6797 	WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr));
6798 	WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr));
6799 	WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr));
6800 	WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val);
6801 	WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size);
6802 
6803 	commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) |
6804 				FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1);
6805 
6806 	if (is_memset)
6807 		commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1);
6808 
6809 	WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask);
6810 
6811 	/* Wait for completion */
6812 	cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION];
6813 	cq_base = cq->kernel_address;
6814 	polling_addr = (u32 *)&cq_base[cq->ci];
6815 
6816 	if (hdev->pldm)
6817 		/* for each 1MB 20 second of timeout */
6818 		timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20;
6819 	else
6820 		timeout = KDMA_TIMEOUT_USEC;
6821 
6822 	/* Polling */
6823 	rc = hl_poll_timeout_memory(
6824 			hdev,
6825 			polling_addr,
6826 			status,
6827 			(status == 1),
6828 			1000,
6829 			timeout,
6830 			true);
6831 
6832 	*polling_addr = 0;
6833 
6834 	if (rc) {
6835 		dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n");
6836 		WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT);
6837 		return rc;
6838 	}
6839 
6840 	cq->ci = hl_cq_inc_ptr(cq->ci);
6841 
6842 	return 0;
6843 }
6844 
6845 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val)
6846 {
6847 	u32 i;
6848 
6849 	for (i = 0 ; i < size ; i += sizeof(u32))
6850 		WREG32(addr + i, val);
6851 }
6852 
6853 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable)
6854 {
6855 	u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
6856 
6857 	if (enable) {
6858 		WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE);
6859 		WREG32(reg_base + QM_PQC_CFG_OFFSET, 0);
6860 	} else {
6861 		WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED);
6862 		WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);
6863 	}
6864 }
6865 
6866 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id)
6867 {
6868 	return hdev->asic_prop.first_available_user_sob[0] +
6869 				hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0;
6870 }
6871 
6872 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id)
6873 {
6874 	u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6875 	u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
6876 
6877 	/* Reset the SOB value */
6878 	WREG32(sob_addr, 0);
6879 }
6880 
6881 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val,
6882 					    struct gaudi2_queues_test_info *msg_info)
6883 {
6884 	u32 sob_offset =  gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6885 	u32 tmp, sob_base = 1;
6886 	struct packet_msg_short *msg_short_pkt = msg_info->kern_addr;
6887 	size_t pkt_size = sizeof(struct packet_msg_short);
6888 	int rc;
6889 
6890 	tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) |
6891 		(1 << GAUDI2_PKT_CTL_EB_SHIFT) |
6892 		(1 << GAUDI2_PKT_CTL_MB_SHIFT) |
6893 		(sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) |
6894 		(sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT);
6895 
6896 	msg_short_pkt->value = cpu_to_le32(sob_val);
6897 	msg_short_pkt->ctl = cpu_to_le32(tmp);
6898 
6899 	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr);
6900 	if (rc)
6901 		dev_err(hdev->dev,
6902 			"Failed to send msg_short packet to H/W queue %d\n", hw_queue_id);
6903 
6904 	return rc;
6905 }
6906 
6907 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val)
6908 {
6909 	u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4;
6910 	u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
6911 	u32 timeout_usec, tmp;
6912 	int rc;
6913 
6914 	if (hdev->pldm)
6915 		timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC;
6916 	else
6917 		timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC;
6918 
6919 	rc = hl_poll_timeout(
6920 			hdev,
6921 			sob_addr,
6922 			tmp,
6923 			(tmp == sob_val),
6924 			1000,
6925 			timeout_usec);
6926 
6927 	if (rc == -ETIMEDOUT) {
6928 		dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n",
6929 			hw_queue_id, tmp);
6930 		rc = -EIO;
6931 	}
6932 
6933 	return rc;
6934 }
6935 
6936 static int gaudi2_test_cpu_queue(struct hl_device *hdev)
6937 {
6938 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6939 
6940 	/*
6941 	 * check capability here as send_cpu_message() won't update the result
6942 	 * value if no capability
6943 	 */
6944 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
6945 		return 0;
6946 
6947 	return hl_fw_test_cpu_queue(hdev);
6948 }
6949 
6950 static int gaudi2_test_queues(struct hl_device *hdev)
6951 {
6952 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6953 	struct gaudi2_queues_test_info *msg_info;
6954 	u32 sob_val = 0x5a5a;
6955 	int i, rc;
6956 
6957 	/* send test message on all enabled Qs */
6958 	for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
6959 		if (!gaudi2_is_queue_enabled(hdev, i))
6960 			continue;
6961 
6962 		msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0];
6963 		gaudi2_qman_set_test_mode(hdev, i, true);
6964 		gaudi2_test_queue_clear(hdev, i);
6965 		rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info);
6966 		if (rc)
6967 			goto done;
6968 	}
6969 
6970 	rc = gaudi2_test_cpu_queue(hdev);
6971 	if (rc)
6972 		goto done;
6973 
6974 	/* verify that all messages were processed */
6975 	for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
6976 		if (!gaudi2_is_queue_enabled(hdev, i))
6977 			continue;
6978 
6979 		rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val);
6980 		if (rc)
6981 			/* chip is not usable, no need for cleanups, just bail-out with error */
6982 			goto done;
6983 
6984 		gaudi2_test_queue_clear(hdev, i);
6985 		gaudi2_qman_set_test_mode(hdev, i, false);
6986 	}
6987 
6988 done:
6989 	return rc;
6990 }
6991 
6992 static int gaudi2_compute_reset_late_init(struct hl_device *hdev)
6993 {
6994 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
6995 	size_t irq_arr_size;
6996 	int rc;
6997 
6998 	gaudi2_init_arcs(hdev);
6999 
7000 	rc = gaudi2_scrub_arcs_dccm(hdev);
7001 	if (rc) {
7002 		dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");
7003 		return rc;
7004 	}
7005 
7006 	gaudi2_init_security(hdev);
7007 
7008 	/* Unmask all IRQs since some could have been received during the soft reset */
7009 	irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]);
7010 	return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size);
7011 }
7012 
7013 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7014 		struct engines_data *e)
7015 {
7016 	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1;
7017 	struct asic_fixed_properties *prop = &hdev->asic_prop;
7018 	unsigned long *mask = (unsigned long *) mask_arr;
7019 	const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n";
7020 	bool is_idle = true, is_eng_idle;
7021 	int engine_idx, i, j;
7022 	u64 offset;
7023 
7024 	if (e)
7025 		hl_engine_data_sprintf(e,
7026 			"\nCORE  EDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0  DMA_CORE_STS1\n"
7027 			"----  ----  -------  ------------  -------------  -------------\n");
7028 
7029 	for (i = 0; i < NUM_OF_DCORES; i++) {
7030 		for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) {
7031 			int seq = i * NUM_OF_EDMA_PER_DCORE + j;
7032 
7033 			if (!(prop->edma_enabled_mask & BIT(seq)))
7034 				continue;
7035 
7036 			engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 +
7037 					i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
7038 			offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET;
7039 
7040 			dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset);
7041 			dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset);
7042 
7043 			qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset);
7044 			qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset);
7045 			qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset);
7046 
7047 			is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
7048 					IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1);
7049 			is_idle &= is_eng_idle;
7050 
7051 			if (mask && !is_eng_idle)
7052 				set_bit(engine_idx, mask);
7053 
7054 			if (e)
7055 				hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N",
7056 							qm_glbl_sts0, dma_core_sts0, dma_core_sts1);
7057 		}
7058 	}
7059 
7060 	return is_idle;
7061 }
7062 
7063 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7064 		struct engines_data *e)
7065 {
7066 	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1;
7067 	unsigned long *mask = (unsigned long *) mask_arr;
7068 	const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n";
7069 	bool is_idle = true, is_eng_idle;
7070 	int engine_idx, i;
7071 	u64 offset;
7072 
7073 	if (e)
7074 		hl_engine_data_sprintf(e,
7075 					"\nPDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0  DMA_CORE_STS1\n"
7076 					"----  -------  ------------  -------------  -------------\n");
7077 
7078 	for (i = 0 ; i < NUM_OF_PDMA ; i++) {
7079 		engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i;
7080 		offset = i * PDMA_OFFSET;
7081 		dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset);
7082 		dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset);
7083 
7084 		qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset);
7085 		qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset);
7086 		qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset);
7087 
7088 		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
7089 				IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1);
7090 		is_idle &= is_eng_idle;
7091 
7092 		if (mask && !is_eng_idle)
7093 			set_bit(engine_idx, mask);
7094 
7095 		if (e)
7096 			hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N",
7097 						qm_glbl_sts0, dma_core_sts0, dma_core_sts1);
7098 	}
7099 
7100 	return is_idle;
7101 }
7102 
7103 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7104 		struct engines_data *e)
7105 {
7106 	unsigned long *mask = (unsigned long *) mask_arr;
7107 	const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n";
7108 	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
7109 	bool is_idle = true, is_eng_idle;
7110 	int engine_idx, i;
7111 	u64 offset = 0;
7112 
7113 	/* NIC, twelve macros in Full chip */
7114 	if (e && hdev->nic_ports_mask)
7115 		hl_engine_data_sprintf(e,
7116 					"\nNIC  is_idle  QM_GLBL_STS0  QM_CGM_STS\n"
7117 					"---  -------  ------------  ----------\n");
7118 
7119 	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) {
7120 		if (!(i & 1))
7121 			offset = i / 2 * NIC_OFFSET;
7122 		else
7123 			offset += NIC_QM_OFFSET;
7124 
7125 		if (!(hdev->nic_ports_mask & BIT(i)))
7126 			continue;
7127 
7128 		engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i;
7129 
7130 
7131 		qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset);
7132 		qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset);
7133 		qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset);
7134 
7135 		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7136 		is_idle &= is_eng_idle;
7137 
7138 		if (mask && !is_eng_idle)
7139 			set_bit(engine_idx, mask);
7140 
7141 		if (e)
7142 			hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N",
7143 						qm_glbl_sts0, qm_cgm_sts);
7144 	}
7145 
7146 	return is_idle;
7147 }
7148 
7149 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7150 		struct engines_data *e)
7151 {
7152 	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts;
7153 	unsigned long *mask = (unsigned long *) mask_arr;
7154 	const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n";
7155 	bool is_idle = true, is_eng_idle;
7156 	int engine_idx, i;
7157 	u64 offset;
7158 
7159 	if (e)
7160 		hl_engine_data_sprintf(e,
7161 					"\nMME  Stub  is_idle  QM_GLBL_STS0  MME_ARCH_STATUS\n"
7162 					"---  ----  -------  ------------  ---------------\n");
7163 	/* MME, one per Dcore */
7164 	for (i = 0 ; i < NUM_OF_DCORES ; i++) {
7165 		engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET;
7166 		offset = i * DCORE_OFFSET;
7167 
7168 		qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset);
7169 		qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset);
7170 		qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset);
7171 
7172 		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7173 		is_idle &= is_eng_idle;
7174 
7175 		mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset);
7176 		is_eng_idle &= IS_MME_IDLE(mme_arch_sts);
7177 		is_idle &= is_eng_idle;
7178 
7179 		if (e)
7180 			hl_engine_data_sprintf(e, mme_fmt, i, "N",
7181 				is_eng_idle ? "Y" : "N",
7182 				qm_glbl_sts0,
7183 				mme_arch_sts);
7184 
7185 		if (mask && !is_eng_idle)
7186 			set_bit(engine_idx, mask);
7187 	}
7188 
7189 	return is_idle;
7190 }
7191 
7192 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset,
7193 					struct iterate_module_ctx *ctx)
7194 {
7195 	struct gaudi2_tpc_idle_data *idle_data = ctx->data;
7196 	u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
7197 	bool is_eng_idle;
7198 	int engine_idx;
7199 
7200 	if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1)))
7201 		engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
7202 	else
7203 		engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 +
7204 				dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst;
7205 
7206 	tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset);
7207 	qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset);
7208 	qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset);
7209 	qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset);
7210 
7211 	is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
7212 						IS_TPC_IDLE(tpc_cfg_sts);
7213 	*(idle_data->is_idle) &= is_eng_idle;
7214 
7215 	if (idle_data->mask && !is_eng_idle)
7216 		set_bit(engine_idx, idle_data->mask);
7217 
7218 	if (idle_data->e)
7219 		hl_engine_data_sprintf(idle_data->e,
7220 					idle_data->tpc_fmt, dcore, inst,
7221 					is_eng_idle ? "Y" : "N",
7222 					qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
7223 }
7224 
7225 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7226 		struct engines_data *e)
7227 {
7228 	struct asic_fixed_properties *prop = &hdev->asic_prop;
7229 	unsigned long *mask = (unsigned long *) mask_arr;
7230 	bool is_idle = true;
7231 
7232 	struct gaudi2_tpc_idle_data tpc_idle_data = {
7233 		.tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n",
7234 		.e = e,
7235 		.mask = mask,
7236 		.is_idle = &is_idle,
7237 	};
7238 	struct iterate_module_ctx tpc_iter = {
7239 		.fn = &gaudi2_is_tpc_engine_idle,
7240 		.data = &tpc_idle_data,
7241 	};
7242 
7243 	if (e && prop->tpc_enabled_mask)
7244 		hl_engine_data_sprintf(e,
7245 			"\nCORE  TPC  is_idle  QM_GLBL_STS0  QM_CGM_STS  STATUS\n"
7246 			"----  ---  -------  ------------  ----------  ------\n");
7247 
7248 	gaudi2_iterate_tpcs(hdev, &tpc_iter);
7249 
7250 	return *tpc_idle_data.is_idle;
7251 }
7252 
7253 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7254 		struct engines_data *e)
7255 {
7256 	struct asic_fixed_properties *prop = &hdev->asic_prop;
7257 	unsigned long *mask = (unsigned long *) mask_arr;
7258 	const char *pcie_dec_fmt = "%-10d%-9s%#x\n";
7259 	const char *dec_fmt = "%-6d%-5d%-9s%#x\n";
7260 	bool is_idle = true, is_eng_idle;
7261 	u32 dec_swreg15, dec_enabled_bit;
7262 	int engine_idx, i, j;
7263 	u64 offset;
7264 
7265 	/* Decoders, two each Dcore and two shared PCIe decoders */
7266 	if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK)))
7267 		hl_engine_data_sprintf(e,
7268 			"\nCORE  DEC  is_idle  VSI_CMD_SWREG15\n"
7269 			"----  ---  -------  ---------------\n");
7270 
7271 	for (i = 0 ; i < NUM_OF_DCORES ; i++) {
7272 		for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) {
7273 			dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j);
7274 			if (!(prop->decoder_enabled_mask & dec_enabled_bit))
7275 				continue;
7276 
7277 			engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 +
7278 					i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
7279 			offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET;
7280 
7281 			dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset);
7282 			is_eng_idle = IS_DEC_IDLE(dec_swreg15);
7283 			is_idle &= is_eng_idle;
7284 
7285 			if (mask && !is_eng_idle)
7286 				set_bit(engine_idx, mask);
7287 
7288 			if (e)
7289 				hl_engine_data_sprintf(e, dec_fmt, i, j,
7290 							is_eng_idle ? "Y" : "N", dec_swreg15);
7291 		}
7292 	}
7293 
7294 	if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK))
7295 		hl_engine_data_sprintf(e,
7296 			"\nPCIe DEC  is_idle  VSI_CMD_SWREG15\n"
7297 			"--------  -------  ---------------\n");
7298 
7299 	/* Check shared(PCIe) decoders */
7300 	for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) {
7301 		dec_enabled_bit = PCIE_DEC_SHIFT + i;
7302 		if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit)))
7303 			continue;
7304 
7305 		engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i;
7306 		offset = i * DCORE_DEC_OFFSET;
7307 		dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset);
7308 		is_eng_idle = IS_DEC_IDLE(dec_swreg15);
7309 		is_idle &= is_eng_idle;
7310 
7311 		if (mask && !is_eng_idle)
7312 			set_bit(engine_idx, mask);
7313 
7314 		if (e)
7315 			hl_engine_data_sprintf(e, pcie_dec_fmt, i,
7316 						is_eng_idle ? "Y" : "N", dec_swreg15);
7317 	}
7318 
7319 	return is_idle;
7320 }
7321 
7322 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7323 		struct engines_data *e)
7324 {
7325 	const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n";
7326 	unsigned long *mask = (unsigned long *) mask_arr;
7327 	u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
7328 	bool is_idle = true, is_eng_idle;
7329 	int engine_idx, i;
7330 	u64 offset;
7331 
7332 	if (e)
7333 		hl_engine_data_sprintf(e,
7334 			"\nCORE  ROT  is_idle  QM_GLBL_STS0  QM_GLBL_STS1  QM_CGM_STS\n"
7335 			"----  ---  -------  ------------  ------------  ----------\n");
7336 
7337 	for (i = 0 ; i < NUM_OF_ROT ; i++) {
7338 		engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i;
7339 
7340 		offset = i * ROT_OFFSET;
7341 
7342 		qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset);
7343 		qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset);
7344 		qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset);
7345 
7346 		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7347 		is_idle &= is_eng_idle;
7348 
7349 		if (mask && !is_eng_idle)
7350 			set_bit(engine_idx, mask);
7351 
7352 		if (e)
7353 			hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N",
7354 						qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
7355 	}
7356 
7357 	return is_idle;
7358 }
7359 
7360 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
7361 					struct engines_data *e)
7362 {
7363 	bool is_idle = true;
7364 
7365 	is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e);
7366 	is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e);
7367 	is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e);
7368 	is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e);
7369 	is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e);
7370 	is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e);
7371 	is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e);
7372 
7373 	return is_idle;
7374 }
7375 
7376 static void gaudi2_hw_queues_lock(struct hl_device *hdev)
7377 	__acquires(&gaudi2->hw_queues_lock)
7378 {
7379 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
7380 
7381 	spin_lock(&gaudi2->hw_queues_lock);
7382 }
7383 
7384 static void gaudi2_hw_queues_unlock(struct hl_device *hdev)
7385 	__releases(&gaudi2->hw_queues_lock)
7386 {
7387 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
7388 
7389 	spin_unlock(&gaudi2->hw_queues_lock);
7390 }
7391 
7392 static u32 gaudi2_get_pci_id(struct hl_device *hdev)
7393 {
7394 	return hdev->pdev->device;
7395 }
7396 
7397 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
7398 {
7399 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
7400 
7401 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
7402 		return 0;
7403 
7404 	return hl_fw_get_eeprom_data(hdev, data, max_size);
7405 }
7406 
7407 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val)
7408 {
7409 	WREG32(mmCPU_IF_EQ_RD_OFFS, val);
7410 }
7411 
7412 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
7413 {
7414 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
7415 
7416 	if (aggregate) {
7417 		*size = (u32) sizeof(gaudi2->events_stat_aggregate);
7418 		return gaudi2->events_stat_aggregate;
7419 	}
7420 
7421 	*size = (u32) sizeof(gaudi2->events_stat);
7422 	return gaudi2->events_stat;
7423 }
7424 
7425 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id,
7426 				int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp)
7427 {
7428 	u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) *
7429 			dcore_vdec_id + DCORE_OFFSET * dcore_id;
7430 
7431 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp);
7432 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid);
7433 
7434 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp);
7435 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid);
7436 
7437 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp);
7438 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid);
7439 
7440 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp);
7441 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid);
7442 
7443 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp);
7444 	WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid);
7445 }
7446 
7447 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid)
7448 {
7449 	u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
7450 			(asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
7451 	struct asic_fixed_properties *prop = &hdev->asic_prop;
7452 	u32 dcore_offset = dcore_id * DCORE_OFFSET;
7453 	u32 vdec_id, i, ports_offset, reg_val;
7454 	u8 edma_seq_base;
7455 
7456 	/* EDMA */
7457 	edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE;
7458 	if (prop->edma_enabled_mask & BIT(edma_seq_base)) {
7459 		WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
7460 		WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
7461 		WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0);
7462 		WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid);
7463 	}
7464 
7465 	if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) {
7466 		WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
7467 		WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
7468 		WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid);
7469 		WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0);
7470 	}
7471 
7472 	/* Sync Mngr */
7473 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid);
7474 	/*
7475 	 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID
7476 	 * for any access type
7477 	 */
7478 	if (dcore_id > 0) {
7479 		reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) |
7480 			  (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT);
7481 		WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val);
7482 		WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0);
7483 	}
7484 
7485 	WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0);
7486 	WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid);
7487 
7488 	for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) {
7489 		ports_offset = i * DCORE_MME_SBTE_OFFSET;
7490 		WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP +
7491 				dcore_offset + ports_offset, 0);
7492 		WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID +
7493 				dcore_offset + ports_offset, rw_asid);
7494 	}
7495 
7496 	for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) {
7497 		ports_offset = i * DCORE_MME_WB_OFFSET;
7498 		WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP +
7499 				dcore_offset + ports_offset, 0);
7500 		WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID +
7501 				dcore_offset + ports_offset, rw_asid);
7502 	}
7503 
7504 	WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
7505 	WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
7506 
7507 	/*
7508 	 * Decoders
7509 	 */
7510 	for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) {
7511 		if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id))
7512 			gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0);
7513 	}
7514 }
7515 
7516 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev,
7517 				int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp)
7518 {
7519 	u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id;
7520 
7521 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp);
7522 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid);
7523 
7524 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp);
7525 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid);
7526 
7527 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp);
7528 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid);
7529 
7530 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp);
7531 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid);
7532 
7533 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp);
7534 	WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid);
7535 }
7536 
7537 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id,
7538 							u32 rw_asid, u32 rw_mmu_bp)
7539 {
7540 	u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id;
7541 
7542 	WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp);
7543 	WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid);
7544 }
7545 
7546 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid)
7547 {
7548 	u32 reg_base, reg_offset, reg_val = 0;
7549 
7550 	reg_base = gaudi2_arc_blocks_bases[cpu_id];
7551 
7552 	/* Enable MMU and configure asid for all relevant ARC regions */
7553 	reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0);
7554 	reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid);
7555 
7556 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL);
7557 	WREG32(reg_base + reg_offset, reg_val);
7558 
7559 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW);
7560 	WREG32(reg_base + reg_offset, reg_val);
7561 
7562 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA);
7563 	WREG32(reg_base + reg_offset, reg_val);
7564 
7565 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA);
7566 	WREG32(reg_base + reg_offset, reg_val);
7567 
7568 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA);
7569 	WREG32(reg_base + reg_offset, reg_val);
7570 
7571 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE);
7572 	WREG32(reg_base + reg_offset, reg_val);
7573 
7574 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL);
7575 	WREG32(reg_base + reg_offset, reg_val);
7576 
7577 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL);
7578 	WREG32(reg_base + reg_offset, reg_val);
7579 
7580 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL);
7581 	WREG32(reg_base + reg_offset, reg_val);
7582 
7583 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL);
7584 	WREG32(reg_base + reg_offset, reg_val);
7585 
7586 	reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL);
7587 	WREG32(reg_base + reg_offset, reg_val);
7588 }
7589 
7590 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid)
7591 {
7592 	int i;
7593 
7594 	if (hdev->fw_components & FW_TYPE_BOOT_CPU)
7595 		return hl_fw_cpucp_engine_core_asid_set(hdev, asid);
7596 
7597 	for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++)
7598 		gaudi2_arc_mmu_prepare(hdev, i, asid);
7599 
7600 	for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) {
7601 		if (!gaudi2_is_queue_enabled(hdev, i))
7602 			continue;
7603 
7604 		gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid);
7605 	}
7606 
7607 	return 0;
7608 }
7609 
7610 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid)
7611 {
7612 	struct asic_fixed_properties *prop = &hdev->asic_prop;
7613 	u32 rw_asid, offset;
7614 	int rc, i;
7615 
7616 	rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) |
7617 			FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid);
7618 
7619 	WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid);
7620 	WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0);
7621 	WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid);
7622 	WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0);
7623 
7624 	WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid);
7625 	WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0);
7626 	WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid);
7627 	WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0);
7628 
7629 	/* ROT */
7630 	for (i = 0 ; i < NUM_OF_ROT ; i++) {
7631 		offset = i * ROT_OFFSET;
7632 		WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid);
7633 		WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0);
7634 		RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK);
7635 		RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK);
7636 		RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK);
7637 	}
7638 
7639 	/* Shared Decoders are the last bits in the decoders mask */
7640 	if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0))
7641 		gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0);
7642 
7643 	if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1))
7644 		gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0);
7645 
7646 	/* arc farm arc dup eng */
7647 	for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++)
7648 		gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0);
7649 
7650 	rc = gaudi2_arc_mmu_prepare_all(hdev, asid);
7651 	if (rc)
7652 		return rc;
7653 
7654 	return 0;
7655 }
7656 
7657 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst,	u32 offset,
7658 					struct iterate_module_ctx *ctx)
7659 {
7660 	struct gaudi2_tpc_mmu_data *mmu_data = ctx->data;
7661 
7662 	WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0);
7663 	WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid);
7664 	WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0);
7665 	WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid);
7666 }
7667 
7668 /* zero the MMUBP and set the ASID */
7669 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid)
7670 {
7671 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
7672 	struct gaudi2_tpc_mmu_data tpc_mmu_data;
7673 	struct iterate_module_ctx tpc_iter = {
7674 		.fn = &gaudi2_tpc_mmu_prepare,
7675 		.data = &tpc_mmu_data,
7676 	};
7677 	int rc, i;
7678 
7679 	if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) {
7680 		dev_crit(hdev->dev, "asid %u is too big\n", asid);
7681 		return -EINVAL;
7682 	}
7683 
7684 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK))
7685 		return 0;
7686 
7687 	rc = gaudi2_mmu_shared_prepare(hdev, asid);
7688 	if (rc)
7689 		return rc;
7690 
7691 	/* configure DCORE MMUs */
7692 	tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
7693 				(asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
7694 	gaudi2_iterate_tpcs(hdev, &tpc_iter);
7695 	for (i = 0 ; i < NUM_OF_DCORES ; i++)
7696 		gaudi2_mmu_dcore_prepare(hdev, i, asid);
7697 
7698 	return 0;
7699 }
7700 
7701 static inline bool is_info_event(u32 event)
7702 {
7703 	switch (event) {
7704 	case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
7705 	case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
7706 
7707 	/* return in case of NIC status event - these events are received periodically and not as
7708 	 * an indication to an error.
7709 	 */
7710 	case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1:
7711 		return true;
7712 	default:
7713 		return false;
7714 	}
7715 }
7716 
7717 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type,
7718 			bool ratelimited, const char *fmt, ...)
7719 {
7720 	struct va_format vaf;
7721 	va_list args;
7722 
7723 	va_start(args, fmt);
7724 	vaf.fmt = fmt;
7725 	vaf.va = &args;
7726 
7727 	if (ratelimited)
7728 		dev_err_ratelimited(hdev->dev, "%s: %pV\n",
7729 			gaudi2_irq_map_table[event_type].valid ?
7730 			gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf);
7731 	else
7732 		dev_err(hdev->dev, "%s: %pV\n",
7733 			gaudi2_irq_map_table[event_type].valid ?
7734 			gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf);
7735 
7736 	va_end(args);
7737 }
7738 
7739 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
7740 		struct hl_eq_ecc_data *ecc_data)
7741 {
7742 	u64 ecc_address = 0, ecc_syndrom = 0;
7743 	u8 memory_wrapper_idx = 0;
7744 
7745 	ecc_address = le64_to_cpu(ecc_data->ecc_address);
7746 	ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);
7747 	memory_wrapper_idx = ecc_data->memory_wrapper_idx;
7748 
7749 	gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
7750 		"ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.",
7751 		ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical);
7752 
7753 	return !!ecc_data->is_critical;
7754 }
7755 
7756 static void print_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base)
7757 {
7758 	u32 lo, hi, cq_ptr_size, arc_cq_ptr_size;
7759 	u64 cq_ptr, arc_cq_ptr, cp_current_inst;
7760 
7761 	lo = RREG32(qman_base + QM_CQ_PTR_LO_4_OFFSET);
7762 	hi = RREG32(qman_base + QM_CQ_PTR_HI_4_OFFSET);
7763 	cq_ptr = ((u64) hi) << 32 | lo;
7764 	cq_ptr_size = RREG32(qman_base + QM_CQ_TSIZE_4_OFFSET);
7765 
7766 	lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_OFFSET);
7767 	hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_OFFSET);
7768 	arc_cq_ptr = ((u64) hi) << 32 | lo;
7769 	arc_cq_ptr_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_OFFSET);
7770 
7771 	lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET);
7772 	hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET);
7773 	cp_current_inst = ((u64) hi) << 32 | lo;
7774 
7775 	dev_info(hdev->dev,
7776 		"LowerQM. CQ: {ptr %#llx, size %u}, ARC_CQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n",
7777 		cq_ptr, cq_ptr_size, arc_cq_ptr, arc_cq_ptr_size, cp_current_inst);
7778 }
7779 
7780 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type,
7781 							u64 qman_base, u32 qid_base)
7782 {
7783 	u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0;
7784 	u64 glbl_sts_addr, arb_err_addr;
7785 	char reg_desc[32];
7786 
7787 	glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE);
7788 	arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE);
7789 
7790 	/* Iterate through all stream GLBL_ERR_STS registers + Lower CP */
7791 	for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {
7792 		glbl_sts_val = RREG32(glbl_sts_addr + 4 * i);
7793 
7794 		if (!glbl_sts_val)
7795 			continue;
7796 
7797 		if (i == QMAN_STREAMS) {
7798 			snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM");
7799 			num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE;
7800 		} else {
7801 			snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i);
7802 			num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE;
7803 		}
7804 
7805 		for (j = 0 ; j < num_error_causes ; j++)
7806 			if (glbl_sts_val & BIT(j)) {
7807 				gaudi2_print_event(hdev, event_type, true,
7808 					"%s. err cause: %s", reg_desc,
7809 					i == QMAN_STREAMS ?
7810 					gaudi2_lower_qman_error_cause[j] :
7811 					gaudi2_qman_error_cause[j]);
7812 				error_count++;
7813 			}
7814 
7815 		if (i == QMAN_STREAMS)
7816 			print_lower_qman_data_on_err(hdev, qman_base);
7817 	}
7818 
7819 	arb_err_val = RREG32(arb_err_addr);
7820 
7821 	if (!arb_err_val)
7822 		goto out;
7823 
7824 	for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) {
7825 		if (arb_err_val & BIT(j)) {
7826 			gaudi2_print_event(hdev, event_type, true,
7827 				"ARB_ERR. err cause: %s",
7828 				gaudi2_qman_arb_error_cause[j]);
7829 			error_count++;
7830 		}
7831 	}
7832 
7833 out:
7834 	return error_count;
7835 }
7836 
7837 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
7838 			u64 rtr_mstr_if_base_addr, bool is_write, char *name,
7839 			enum gaudi2_engine_id id, u64 *event_mask)
7840 {
7841 	u32 razwi_hi, razwi_lo, razwi_xy;
7842 	u16 eng_id = id;
7843 	u8 rd_wr_flag;
7844 
7845 	if (is_write) {
7846 		razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI);
7847 		razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO);
7848 		razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY);
7849 		rd_wr_flag = HL_RAZWI_WRITE;
7850 	} else {
7851 		razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI);
7852 		razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO);
7853 		razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY);
7854 		rd_wr_flag = HL_RAZWI_READ;
7855 	}
7856 
7857 	hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1,
7858 				rd_wr_flag | HL_RAZWI_HBW, event_mask);
7859 
7860 	dev_err_ratelimited(hdev->dev,
7861 		"%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n",
7862 		name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy);
7863 }
7864 
7865 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
7866 			u64 rtr_mstr_if_base_addr, bool is_write, char *name,
7867 			enum gaudi2_engine_id id, u64 *event_mask)
7868 {
7869 	u64 razwi_addr = CFG_BASE;
7870 	u32 razwi_xy;
7871 	u16 eng_id = id;
7872 	u8 rd_wr_flag;
7873 
7874 	if (is_write) {
7875 		razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI);
7876 		razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY);
7877 		rd_wr_flag = HL_RAZWI_WRITE;
7878 	} else {
7879 		razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI);
7880 		razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY);
7881 		rd_wr_flag = HL_RAZWI_READ;
7882 	}
7883 
7884 	hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask);
7885 	dev_err_ratelimited(hdev->dev,
7886 				"%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n",
7887 				name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr,
7888 						razwi_xy);
7889 }
7890 
7891 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev,
7892 						enum razwi_event_sources module, u8 module_idx)
7893 {
7894 	switch (module) {
7895 	case RAZWI_TPC:
7896 		if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES))
7897 			return GAUDI2_DCORE0_ENGINE_ID_TPC_6;
7898 		return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
7899 				(module_idx % NUM_OF_TPC_PER_DCORE) +
7900 				(GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));
7901 
7902 	case RAZWI_MME:
7903 		return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) +
7904 			(module_idx * ENGINE_ID_DCORE_OFFSET));
7905 
7906 	case RAZWI_EDMA:
7907 		return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
7908 			(module_idx % NUM_OF_EDMA_PER_DCORE));
7909 
7910 	case RAZWI_PDMA:
7911 		return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx);
7912 
7913 	case RAZWI_NIC:
7914 		return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx));
7915 
7916 	case RAZWI_DEC:
7917 		if (module_idx == 8)
7918 			return GAUDI2_PCIE_ENGINE_ID_DEC_0;
7919 
7920 		if (module_idx == 9)
7921 			return GAUDI2_PCIE_ENGINE_ID_DEC_1;
7922 					;
7923 		return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
7924 				(module_idx % NUM_OF_DEC_PER_DCORE) +
7925 				(GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));
7926 
7927 	case RAZWI_ROT:
7928 		return GAUDI2_ENGINE_ID_ROT_0 + module_idx;
7929 
7930 	default:
7931 		return GAUDI2_ENGINE_ID_SIZE;
7932 	}
7933 }
7934 
7935 /*
7936  * This function handles RR(Range register) hit events.
7937  * raised be initiators not PSOC RAZWI.
7938  */
7939 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
7940 				enum razwi_event_sources module, u8 module_idx,
7941 				u8 module_sub_idx, u64 *event_mask)
7942 {
7943 	bool via_sft = false;
7944 	u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx;
7945 	u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr;
7946 	u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0;
7947 	u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0;
7948 	char initiator_name[64];
7949 
7950 	switch (module) {
7951 	case RAZWI_TPC:
7952 		sprintf(initiator_name, "TPC_%u", module_idx);
7953 		if (hdev->tpc_binning) {
7954 			binned_idx = __ffs(hdev->tpc_binning);
7955 			if (binned_idx == module_idx)
7956 				module_idx = TPC_ID_DCORE0_TPC6;
7957 		}
7958 
7959 		hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx];
7960 
7961 		if (hl_is_fw_sw_ver_below(hdev, 1, 9) &&
7962 				!hdev->asic_prop.fw_security_enabled &&
7963 				((module_idx == 0) || (module_idx == 1)))
7964 			lbw_rtr_id = DCORE0_RTR0;
7965 		else
7966 			lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx];
7967 		break;
7968 	case RAZWI_MME:
7969 		sprintf(initiator_name, "MME_%u", module_idx);
7970 		switch (module_sub_idx) {
7971 		case MME_WAP0:
7972 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0;
7973 			break;
7974 		case MME_WAP1:
7975 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1;
7976 			break;
7977 		case MME_WRITE:
7978 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write;
7979 			break;
7980 		case MME_READ:
7981 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read;
7982 			break;
7983 		case MME_SBTE0:
7984 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0;
7985 			break;
7986 		case MME_SBTE1:
7987 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1;
7988 			break;
7989 		case MME_SBTE2:
7990 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2;
7991 			break;
7992 		case MME_SBTE3:
7993 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3;
7994 			break;
7995 		case MME_SBTE4:
7996 			hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4;
7997 			break;
7998 		default:
7999 			return;
8000 		}
8001 		lbw_rtr_id = hbw_rtr_id;
8002 		break;
8003 	case RAZWI_EDMA:
8004 		hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx];
8005 		dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE;
8006 		/* SFT has separate MSTR_IF for LBW, only there we can
8007 		 * read the LBW razwi related registers
8008 		 */
8009 		lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE +
8010 								dcore_id * SFT_DCORE_OFFSET;
8011 		via_sft = true;
8012 		sprintf(initiator_name, "EDMA_%u", module_idx);
8013 		break;
8014 	case RAZWI_PDMA:
8015 		hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx];
8016 		lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx];
8017 		sprintf(initiator_name, "PDMA_%u", module_idx);
8018 		break;
8019 	case RAZWI_NIC:
8020 		hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx];
8021 		lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx];
8022 		sprintf(initiator_name, "NIC_%u", module_idx);
8023 		break;
8024 	case RAZWI_DEC:
8025 		sprintf(initiator_name, "DEC_%u", module_idx);
8026 		if (hdev->decoder_binning) {
8027 			binned_idx = __ffs(hdev->decoder_binning);
8028 			if (binned_idx == module_idx)
8029 				module_idx = DEC_ID_PCIE_VDEC1;
8030 		}
8031 		hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx];
8032 		lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx];
8033 		break;
8034 	case RAZWI_ROT:
8035 		hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx];
8036 		lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx];
8037 		sprintf(initiator_name, "ROT_%u", module_idx);
8038 		break;
8039 	default:
8040 		return;
8041 	}
8042 
8043 	/* Find router mstr_if register base */
8044 	if (!via_sft) {
8045 		dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE;
8046 		dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE;
8047 		hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE +
8048 				dcore_id * DCORE_OFFSET +
8049 				dcore_rtr_id * DCORE_RTR_OFFSET +
8050 				RTR_MSTR_IF_OFFSET;
8051 		lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr +
8052 				(((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET);
8053 	}
8054 
8055 	/* Find out event cause by reading "RAZWI_HAPPENED" registers */
8056 	hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED);
8057 	hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED);
8058 	lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED);
8059 	lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED);
8060 
8061 	eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx);
8062 	if (hbw_shrd_aw) {
8063 		gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true,
8064 						initiator_name, eng_id, event_mask);
8065 
8066 		/* Clear event indication */
8067 		WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw);
8068 	}
8069 
8070 	if (hbw_shrd_ar) {
8071 		gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false,
8072 						initiator_name, eng_id, event_mask);
8073 
8074 		/* Clear event indication */
8075 		WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar);
8076 	}
8077 
8078 	if (lbw_shrd_aw) {
8079 		gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true,
8080 						initiator_name, eng_id, event_mask);
8081 
8082 		/* Clear event indication */
8083 		WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw);
8084 	}
8085 
8086 	if (lbw_shrd_ar) {
8087 		gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false,
8088 						initiator_name, eng_id, event_mask);
8089 
8090 		/* Clear event indication */
8091 		WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar);
8092 	}
8093 }
8094 
8095 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev)
8096 {
8097 	struct asic_fixed_properties *prop = &hdev->asic_prop;
8098 	u8 mod_idx, sub_mod;
8099 
8100 	/* check all TPCs */
8101 	for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) {
8102 		if (prop->tpc_enabled_mask & BIT(mod_idx))
8103 			gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL);
8104 	}
8105 
8106 	/* check all MMEs */
8107 	for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
8108 		for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++)
8109 			gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx,
8110 									sub_mod, NULL);
8111 
8112 	/* check all EDMAs */
8113 	for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
8114 		if (prop->edma_enabled_mask & BIT(mod_idx))
8115 			gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL);
8116 
8117 	/* check all PDMAs */
8118 	for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++)
8119 		gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL);
8120 
8121 	/* check all NICs */
8122 	for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++)
8123 		if (hdev->nic_ports_mask & BIT(mod_idx))
8124 			gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0,
8125 								NULL);
8126 
8127 	/* check all DECs */
8128 	for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++)
8129 		if (prop->decoder_enabled_mask & BIT(mod_idx))
8130 			gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL);
8131 
8132 	/* check all ROTs */
8133 	for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++)
8134 		gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL);
8135 }
8136 
8137 static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size,
8138 						u32 axuser_xy, u32 *base, u16 *eng_id,
8139 						char *eng_name)
8140 {
8141 
8142 	int i, num_of_eng = 0;
8143 	u16 str_size = 0;
8144 
8145 	for (i = 0 ; i < array_size ; i++) {
8146 		if (axuser_xy != razwi_info[i].axuser_xy)
8147 			continue;
8148 
8149 		eng_id[num_of_eng] = razwi_info[i].eng_id;
8150 		base[num_of_eng] = razwi_info[i].rtr_ctrl;
8151 		if (!num_of_eng)
8152 			str_size += scnprintf(eng_name + str_size,
8153 						PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s",
8154 						razwi_info[i].eng_name);
8155 		else
8156 			str_size += scnprintf(eng_name + str_size,
8157 						PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s",
8158 						razwi_info[i].eng_name);
8159 		num_of_eng++;
8160 	}
8161 
8162 	return num_of_eng;
8163 }
8164 
8165 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg,
8166 						u64 *event_mask)
8167 {
8168 	u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0;
8169 	u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR];
8170 	u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR];
8171 	char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE];
8172 	bool razwi_happened = false;
8173 	u64 addr;
8174 	int i;
8175 
8176 	num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info),
8177 							axuser_xy, base, eng_id, eng_name_str);
8178 
8179 	/* If no match for XY coordinates, try to find it in MME razwi table */
8180 	if (!num_of_eng) {
8181 		axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg);
8182 		num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info,
8183 								ARRAY_SIZE(mme_razwi_info),
8184 								axuser_xy, base, eng_id,
8185 								eng_name_str);
8186 	}
8187 
8188 	for  (i = 0 ; i < num_of_eng ; i++) {
8189 		if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) {
8190 			addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI);
8191 			addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO);
8192 			addr = ((u64)addr_hi << 32) + addr_lo;
8193 			if (addr) {
8194 				dev_err(hdev->dev,
8195 					"PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n",
8196 					eng_name_str, addr);
8197 				hl_handle_razwi(hdev, addr, &eng_id[0],
8198 					num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask);
8199 				razwi_happened = true;
8200 			}
8201 		}
8202 
8203 		if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) {
8204 			addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI);
8205 			addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO);
8206 			addr = ((u64)addr_hi << 32) + addr_lo;
8207 			if (addr) {
8208 				dev_err(hdev->dev,
8209 					"PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n",
8210 					eng_name_str, addr);
8211 				hl_handle_razwi(hdev, addr, &eng_id[0],
8212 					num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask);
8213 				razwi_happened = true;
8214 			}
8215 		}
8216 
8217 		if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) {
8218 			addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR);
8219 			if (addr_lo) {
8220 				dev_err(hdev->dev,
8221 					"PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n",
8222 					eng_name_str, addr_lo);
8223 				hl_handle_razwi(hdev, addr_lo, &eng_id[0],
8224 					num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask);
8225 				razwi_happened = true;
8226 			}
8227 		}
8228 
8229 		if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) {
8230 			addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR);
8231 			if (addr_lo) {
8232 				dev_err(hdev->dev,
8233 						"PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n",
8234 						eng_name_str, addr_lo);
8235 				hl_handle_razwi(hdev, addr_lo, &eng_id[0],
8236 					num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask);
8237 				razwi_happened = true;
8238 			}
8239 		}
8240 		/* In common case the loop will break, when there is only one engine id, or
8241 		 * several engines with the same router. The exceptional case is with psoc razwi
8242 		 * from EDMA, where it's possible to get axuser id which fits 2 routers (2
8243 		 * interfaces of sft router). In this case, maybe the first router won't hold info
8244 		 * and we will need to iterate on the other router.
8245 		 */
8246 		if (razwi_happened)
8247 			break;
8248 	}
8249 
8250 	return razwi_happened;
8251 }
8252 
8253 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */
8254 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask)
8255 {
8256 	u32 razwi_mask_info, razwi_intr = 0, error_count = 0;
8257 
8258 	if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) {
8259 		razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT);
8260 		if (!razwi_intr)
8261 			return 0;
8262 	}
8263 
8264 	razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO);
8265 
8266 	dev_err_ratelimited(hdev->dev,
8267 		"PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n",
8268 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info),
8269 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info),
8270 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info),
8271 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info),
8272 		FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info));
8273 
8274 	if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask))
8275 		error_count++;
8276 	else
8277 		dev_err_ratelimited(hdev->dev,
8278 				"PSOC RAZWI interrupt: invalid razwi info (0x%x)\n",
8279 				razwi_mask_info);
8280 
8281 	/* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */
8282 	if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX))
8283 		WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr);
8284 
8285 	return error_count;
8286 }
8287 
8288 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type)
8289 {
8290 	u32 i, sts_val, sts_clr_val = 0, error_count = 0;
8291 
8292 	sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET);
8293 
8294 	for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) {
8295 		if (sts_val & BIT(i)) {
8296 			gaudi2_print_event(hdev, event_type, true,
8297 				"err cause: %s", gaudi2_qm_sei_error_cause[i]);
8298 			sts_clr_val |= BIT(i);
8299 			error_count++;
8300 		}
8301 	}
8302 
8303 	WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val);
8304 
8305 	return error_count;
8306 }
8307 
8308 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
8309 					bool extended_err_check, u64 *event_mask)
8310 {
8311 	enum razwi_event_sources module;
8312 	u32 error_count = 0;
8313 	u64 qman_base;
8314 	u8 index;
8315 
8316 	switch (event_type) {
8317 	case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP:
8318 		index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
8319 		qman_base = mmDCORE0_TPC0_QM_BASE +
8320 				(index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET +
8321 				(index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET;
8322 		module = RAZWI_TPC;
8323 		break;
8324 	case GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
8325 		qman_base = mmDCORE0_TPC6_QM_BASE;
8326 		module = RAZWI_TPC;
8327 		break;
8328 	case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
8329 	case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
8330 	case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
8331 	case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
8332 		index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /
8333 				(GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
8334 						GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
8335 		qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET;
8336 		module = RAZWI_MME;
8337 		break;
8338 	case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
8339 	case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
8340 		index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP;
8341 		qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET;
8342 		module = RAZWI_PDMA;
8343 		break;
8344 	case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
8345 	case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
8346 		index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
8347 		qman_base = mmROT0_QM_BASE + index * ROT_OFFSET;
8348 		module = RAZWI_ROT;
8349 		break;
8350 	default:
8351 		return 0;
8352 	}
8353 
8354 	error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type);
8355 
8356 	/* There is a single event per NIC macro, so should check its both QMAN blocks */
8357 	if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE &&
8358 			event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE)
8359 		error_count += _gaudi2_handle_qm_sei_err(hdev,
8360 					qman_base + NIC_QM_OFFSET, event_type);
8361 
8362 	if (extended_err_check) {
8363 		/* check if RAZWI happened */
8364 		gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask);
8365 		hl_check_for_glbl_errors(hdev);
8366 	}
8367 
8368 	return error_count;
8369 }
8370 
8371 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
8372 {
8373 	u32 qid_base, error_count = 0;
8374 	u64 qman_base;
8375 	u8 index = 0;
8376 
8377 	switch (event_type) {
8378 	case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM:
8379 		index = event_type - GAUDI2_EVENT_TPC0_QM;
8380 		qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS;
8381 		qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8382 		break;
8383 	case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM:
8384 		index = event_type - GAUDI2_EVENT_TPC6_QM;
8385 		qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS;
8386 		qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8387 		break;
8388 	case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM:
8389 		index = event_type - GAUDI2_EVENT_TPC12_QM;
8390 		qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS;
8391 		qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8392 		break;
8393 	case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM:
8394 		index = event_type - GAUDI2_EVENT_TPC18_QM;
8395 		qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS;
8396 		qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
8397 		break;
8398 	case GAUDI2_EVENT_TPC24_QM:
8399 		qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0;
8400 		qman_base = mmDCORE0_TPC6_QM_BASE;
8401 		break;
8402 	case GAUDI2_EVENT_MME0_QM:
8403 		qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0;
8404 		qman_base = mmDCORE0_MME_QM_BASE;
8405 		break;
8406 	case GAUDI2_EVENT_MME1_QM:
8407 		qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
8408 		qman_base = mmDCORE1_MME_QM_BASE;
8409 		break;
8410 	case GAUDI2_EVENT_MME2_QM:
8411 		qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
8412 		qman_base = mmDCORE2_MME_QM_BASE;
8413 		break;
8414 	case GAUDI2_EVENT_MME3_QM:
8415 		qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
8416 		qman_base = mmDCORE3_MME_QM_BASE;
8417 		break;
8418 	case GAUDI2_EVENT_HDMA0_QM:
8419 		index = 0;
8420 		qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0;
8421 		qman_base = mmDCORE0_EDMA0_QM_BASE;
8422 		break;
8423 	case GAUDI2_EVENT_HDMA1_QM:
8424 		index = 1;
8425 		qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0;
8426 		qman_base = mmDCORE0_EDMA1_QM_BASE;
8427 		break;
8428 	case GAUDI2_EVENT_HDMA2_QM:
8429 		index = 2;
8430 		qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0;
8431 		qman_base = mmDCORE1_EDMA0_QM_BASE;
8432 		break;
8433 	case GAUDI2_EVENT_HDMA3_QM:
8434 		index = 3;
8435 		qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0;
8436 		qman_base = mmDCORE1_EDMA1_QM_BASE;
8437 		break;
8438 	case GAUDI2_EVENT_HDMA4_QM:
8439 		index = 4;
8440 		qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0;
8441 		qman_base = mmDCORE2_EDMA0_QM_BASE;
8442 		break;
8443 	case GAUDI2_EVENT_HDMA5_QM:
8444 		index = 5;
8445 		qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0;
8446 		qman_base = mmDCORE2_EDMA1_QM_BASE;
8447 		break;
8448 	case GAUDI2_EVENT_HDMA6_QM:
8449 		index = 6;
8450 		qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0;
8451 		qman_base = mmDCORE3_EDMA0_QM_BASE;
8452 		break;
8453 	case GAUDI2_EVENT_HDMA7_QM:
8454 		index = 7;
8455 		qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0;
8456 		qman_base = mmDCORE3_EDMA1_QM_BASE;
8457 		break;
8458 	case GAUDI2_EVENT_PDMA0_QM:
8459 		qid_base = GAUDI2_QUEUE_ID_PDMA_0_0;
8460 		qman_base = mmPDMA0_QM_BASE;
8461 		break;
8462 	case GAUDI2_EVENT_PDMA1_QM:
8463 		qid_base = GAUDI2_QUEUE_ID_PDMA_1_0;
8464 		qman_base = mmPDMA1_QM_BASE;
8465 		break;
8466 	case GAUDI2_EVENT_ROTATOR0_ROT0_QM:
8467 		qid_base = GAUDI2_QUEUE_ID_ROT_0_0;
8468 		qman_base = mmROT0_QM_BASE;
8469 		break;
8470 	case GAUDI2_EVENT_ROTATOR1_ROT1_QM:
8471 		qid_base = GAUDI2_QUEUE_ID_ROT_1_0;
8472 		qman_base = mmROT1_QM_BASE;
8473 		break;
8474 	default:
8475 		return 0;
8476 	}
8477 
8478 	error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, qid_base);
8479 
8480 	/* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */
8481 	if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) {
8482 		error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type);
8483 		gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask);
8484 	}
8485 
8486 	hl_check_for_glbl_errors(hdev);
8487 
8488 	return error_count;
8489 }
8490 
8491 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type)
8492 {
8493 	u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm;
8494 
8495 	for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) {
8496 		sts_clr_val = 0;
8497 		sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS +
8498 				(arc_farm * ARC_FARM_OFFSET));
8499 
8500 		for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) {
8501 			if (sts_val & BIT(i)) {
8502 				gaudi2_print_event(hdev, event_type, true,
8503 						"ARC FARM ARC %u err cause: %s",
8504 						arc_farm, gaudi2_arc_sei_error_cause[i]);
8505 				sts_clr_val |= BIT(i);
8506 				error_count++;
8507 			}
8508 		}
8509 		WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET),
8510 				sts_clr_val);
8511 	}
8512 
8513 	hl_check_for_glbl_errors(hdev);
8514 
8515 	return error_count;
8516 }
8517 
8518 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type)
8519 {
8520 	u32 i, sts_val, sts_clr_val = 0, error_count = 0;
8521 
8522 	sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS);
8523 
8524 	for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) {
8525 		if (sts_val & BIT(i)) {
8526 			gaudi2_print_event(hdev, event_type, true,
8527 				"err cause: %s", gaudi2_cpu_sei_error_cause[i]);
8528 			sts_clr_val |= BIT(i);
8529 			error_count++;
8530 		}
8531 	}
8532 
8533 	hl_check_for_glbl_errors(hdev);
8534 
8535 	WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val);
8536 
8537 	return error_count;
8538 }
8539 
8540 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type,
8541 					struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause,
8542 					u64 *event_mask)
8543 {
8544 	u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
8545 	u32 error_count = 0;
8546 	int i;
8547 
8548 	for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++)
8549 		if (intr_cause_data & BIT(i)) {
8550 			gaudi2_print_event(hdev, event_type, true,
8551 				"err cause: %s", guadi2_rot_error_cause[i]);
8552 			error_count++;
8553 		}
8554 
8555 	/* check if RAZWI happened */
8556 	gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask);
8557 	hl_check_for_glbl_errors(hdev);
8558 
8559 	return error_count;
8560 }
8561 
8562 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev,  u8 tpc_index, u16 event_type,
8563 					struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause,
8564 					u64 *event_mask)
8565 {
8566 	u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
8567 	u32 error_count = 0;
8568 	int i;
8569 
8570 	for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++)
8571 		if (intr_cause_data & BIT(i)) {
8572 			gaudi2_print_event(hdev, event_type, true,
8573 				"interrupt cause: %s",  gaudi2_tpc_interrupts_cause[i]);
8574 			error_count++;
8575 		}
8576 
8577 	/* check if RAZWI happened */
8578 	gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask);
8579 	hl_check_for_glbl_errors(hdev);
8580 
8581 	return error_count;
8582 }
8583 
8584 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type,
8585 					u64 *event_mask)
8586 {
8587 	u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0;
8588 	int i;
8589 
8590 	if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES)
8591 		/* DCORE DEC */
8592 		sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR +
8593 				DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) +
8594 				DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE);
8595 	else
8596 		/* PCIE DEC */
8597 		sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET *
8598 				(dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES);
8599 
8600 	sts_val = RREG32(sts_addr);
8601 
8602 	for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) {
8603 		if (sts_val & BIT(i)) {
8604 			gaudi2_print_event(hdev, event_type, true,
8605 				"err cause: %s", gaudi2_dec_error_cause[i]);
8606 			sts_clr_val |= BIT(i);
8607 			error_count++;
8608 		}
8609 	}
8610 
8611 	/* check if RAZWI happened */
8612 	gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask);
8613 	hl_check_for_glbl_errors(hdev);
8614 
8615 	/* Write 1 clear errors */
8616 	WREG32(sts_addr, sts_clr_val);
8617 
8618 	return error_count;
8619 }
8620 
8621 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
8622 					u64 *event_mask)
8623 {
8624 	u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0;
8625 	int i;
8626 
8627 	sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index;
8628 	sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index;
8629 
8630 	sts_val = RREG32(sts_addr);
8631 
8632 	for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) {
8633 		if (sts_val & BIT(i)) {
8634 			gaudi2_print_event(hdev, event_type, true,
8635 				"err cause: %s", guadi2_mme_error_cause[i]);
8636 			sts_clr_val |= BIT(i);
8637 			error_count++;
8638 		}
8639 	}
8640 
8641 	/* check if RAZWI happened */
8642 	for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++)
8643 		gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask);
8644 
8645 	hl_check_for_glbl_errors(hdev);
8646 
8647 	WREG32(sts_clr_addr, sts_clr_val);
8648 
8649 	return error_count;
8650 }
8651 
8652 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type,
8653 					u64 intr_cause_data)
8654 {
8655 	int i, error_count = 0;
8656 
8657 	for (i = 0 ; i < GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE ; i++)
8658 		if (intr_cause_data & BIT(i)) {
8659 			gaudi2_print_event(hdev, event_type, true,
8660 				"err cause: %s", guadi2_mme_sbte_error_cause[i]);
8661 			error_count++;
8662 		}
8663 
8664 	hl_check_for_glbl_errors(hdev);
8665 
8666 	return error_count;
8667 }
8668 
8669 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
8670 					u64 *event_mask)
8671 {
8672 	u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0;
8673 	int i;
8674 
8675 	sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index;
8676 	sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index;
8677 
8678 	sts_val = RREG32(sts_addr);
8679 
8680 	for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) {
8681 		if (sts_val & BIT(i)) {
8682 			gaudi2_print_event(hdev, event_type, true,
8683 				"err cause: %s", guadi2_mme_wap_error_cause[i]);
8684 			sts_clr_val |= BIT(i);
8685 			error_count++;
8686 		}
8687 	}
8688 
8689 	/* check if RAZWI happened on WAP0/1 */
8690 	gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask);
8691 	gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask);
8692 	hl_check_for_glbl_errors(hdev);
8693 
8694 	WREG32(sts_clr_addr, sts_clr_val);
8695 
8696 	return error_count;
8697 }
8698 
8699 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type,
8700 					u64 intr_cause_data)
8701 {
8702 	u32 error_count = 0;
8703 	int i;
8704 
8705 	/* If an AXI read or write error is received, an error is reported and
8706 	 * interrupt message is sent. Due to an HW errata, when reading the cause
8707 	 * register of the KDMA engine, the reported error is always HBW even if
8708 	 * the actual error caused by a LBW KDMA transaction.
8709 	 */
8710 	for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
8711 		if (intr_cause_data & BIT(i)) {
8712 			gaudi2_print_event(hdev, event_type, true,
8713 				"err cause: %s", gaudi2_kdma_core_interrupts_cause[i]);
8714 			error_count++;
8715 		}
8716 
8717 	hl_check_for_glbl_errors(hdev);
8718 
8719 	return error_count;
8720 }
8721 
8722 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause)
8723 {
8724 	u32 error_count = 0;
8725 	int i;
8726 
8727 	for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
8728 		if (intr_cause & BIT(i)) {
8729 			gaudi2_print_event(hdev, event_type, true,
8730 				"err cause: %s", gaudi2_dma_core_interrupts_cause[i]);
8731 			error_count++;
8732 		}
8733 
8734 	hl_check_for_glbl_errors(hdev);
8735 
8736 	return error_count;
8737 }
8738 
8739 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask)
8740 {
8741 	u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr;
8742 
8743 	razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED;
8744 	if (RREG32(razwi_happened_addr)) {
8745 		gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE",
8746 							GAUDI2_ENGINE_ID_PCIE, event_mask);
8747 		WREG32(razwi_happened_addr, 0x1);
8748 	}
8749 
8750 	razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED;
8751 	if (RREG32(razwi_happened_addr)) {
8752 		gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE",
8753 							GAUDI2_ENGINE_ID_PCIE, event_mask);
8754 		WREG32(razwi_happened_addr, 0x1);
8755 	}
8756 
8757 	razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED;
8758 	if (RREG32(razwi_happened_addr)) {
8759 		gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE",
8760 							GAUDI2_ENGINE_ID_PCIE, event_mask);
8761 		WREG32(razwi_happened_addr, 0x1);
8762 	}
8763 
8764 	razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED;
8765 	if (RREG32(razwi_happened_addr)) {
8766 		gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE",
8767 							GAUDI2_ENGINE_ID_PCIE, event_mask);
8768 		WREG32(razwi_happened_addr, 0x1);
8769 	}
8770 }
8771 
8772 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type,
8773 					u64 intr_cause_data, u64 *event_mask)
8774 {
8775 	u32 error_count = 0;
8776 	int i;
8777 
8778 	gaudi2_print_event(hdev, event_type, true,
8779 		"intr_cause_data: %#llx", intr_cause_data);
8780 
8781 	for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) {
8782 		if (!(intr_cause_data & BIT_ULL(i)))
8783 			continue;
8784 
8785 		gaudi2_print_event(hdev, event_type, true,
8786 			"err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]);
8787 		error_count++;
8788 
8789 		/*
8790 		 * Always check for LBW and HBW additional info as the indication itself is
8791 		 * sometimes missing
8792 		 */
8793 	}
8794 
8795 	hl_check_for_glbl_errors(hdev);
8796 	gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask);
8797 
8798 	return error_count;
8799 }
8800 
8801 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type,
8802 				u64 intr_cause_data)
8803 
8804 {
8805 	u32 error_count = 0;
8806 	int i;
8807 
8808 	for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) {
8809 		if (intr_cause_data & BIT_ULL(i)) {
8810 			gaudi2_print_event(hdev, event_type, true,
8811 				"err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]);
8812 			error_count++;
8813 		}
8814 	}
8815 
8816 	return error_count;
8817 }
8818 
8819 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data)
8820 {
8821 	u32 error_count = 0;
8822 	int i;
8823 
8824 	for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) {
8825 		if (intr_cause_data & BIT_ULL(i)) {
8826 			gaudi2_print_event(hdev, event_type, true,
8827 				"err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]);
8828 			error_count++;
8829 		}
8830 	}
8831 
8832 	return error_count;
8833 }
8834 
8835 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu,
8836 					u64 *event_mask)
8837 {
8838 	u32 valid, val;
8839 	u64 addr;
8840 
8841 	valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
8842 
8843 	if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK))
8844 		return;
8845 
8846 	val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE));
8847 	addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK;
8848 	addr <<= 32;
8849 	addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA));
8850 
8851 	if (is_pmmu) {
8852 		dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr);
8853 	} else {
8854 
8855 		addr = gaudi2_mmu_descramble_addr(hdev, addr);
8856 		addr &= HW_UNSCRAMBLED_BITS_MASK;
8857 		dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n",
8858 				addr, addr + ~HW_UNSCRAMBLED_BITS_MASK);
8859 	}
8860 
8861 	hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask);
8862 
8863 	WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0);
8864 }
8865 
8866 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
8867 {
8868 	u32 valid, val;
8869 	u64 addr;
8870 
8871 	valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
8872 
8873 	if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK))
8874 		return;
8875 
8876 	val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE));
8877 	addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK;
8878 	addr <<= 32;
8879 	addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA));
8880 
8881 	if (!is_pmmu)
8882 		addr = gaudi2_mmu_descramble_addr(hdev, addr);
8883 
8884 	dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n",
8885 				is_pmmu ? "PMMU" : "HMMU", addr);
8886 	WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0);
8887 }
8888 
8889 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type,
8890 						u64 mmu_base, bool is_pmmu, u64 *event_mask)
8891 {
8892 	u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0;
8893 	int i;
8894 
8895 	spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET);
8896 
8897 	for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) {
8898 		if (spi_sei_cause & BIT(i)) {
8899 			gaudi2_print_event(hdev, event_type, true,
8900 				"err cause: %s", gaudi2_mmu_spi_sei[i].cause);
8901 
8902 			if (i == 0)
8903 				gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask);
8904 			else if (i == 1)
8905 				gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
8906 
8907 			if (gaudi2_mmu_spi_sei[i].clear_bit >= 0)
8908 				interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit);
8909 
8910 			error_count++;
8911 		}
8912 	}
8913 
8914 	/* Clear cause */
8915 	WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause);
8916 
8917 	/* Clear interrupt */
8918 	WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr);
8919 
8920 	return error_count;
8921 }
8922 
8923 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index)
8924 {
8925 	u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log,
8926 		cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0;
8927 	int i;
8928 
8929 	sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index;
8930 	cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index;
8931 
8932 	sei_cause_val = RREG32(sei_cause_addr);
8933 	sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val);
8934 	cq_intr_val = RREG32(cq_intr_addr);
8935 
8936 	/* SEI interrupt */
8937 	if (sei_cause_cause) {
8938 		/* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */
8939 		sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK,
8940 					sei_cause_val);
8941 
8942 		for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) {
8943 			if (!(sei_cause_cause & BIT(i)))
8944 				continue;
8945 
8946 			gaudi2_print_event(hdev, event_type, true,
8947 				"err cause: %s. %s: 0x%X",
8948 				gaudi2_sm_sei_cause[i].cause_name,
8949 				gaudi2_sm_sei_cause[i].log_name,
8950 				sei_cause_log);
8951 			error_count++;
8952 			break;
8953 		}
8954 
8955 		/* Clear SM_SEI_CAUSE */
8956 		WREG32(sei_cause_addr, 0);
8957 	}
8958 
8959 	/* CQ interrupt */
8960 	if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) {
8961 		cq_intr_queue_index =
8962 				FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK,
8963 					cq_intr_val);
8964 
8965 		dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n",
8966 				sm_index, cq_intr_queue_index);
8967 		error_count++;
8968 
8969 		/* Clear CQ_INTR */
8970 		WREG32(cq_intr_addr, 0);
8971 	}
8972 
8973 	hl_check_for_glbl_errors(hdev);
8974 
8975 	return error_count;
8976 }
8977 
8978 static u64 get_hmmu_base(u16 event_type)
8979 {
8980 	u8 dcore, index_in_dcore;
8981 
8982 	switch (event_type) {
8983 	case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP:
8984 	case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR:
8985 		dcore = 0;
8986 		index_in_dcore = 0;
8987 	break;
8988 	case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP:
8989 	case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR:
8990 		dcore = 1;
8991 		index_in_dcore = 0;
8992 	break;
8993 	case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP:
8994 	case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR:
8995 		dcore = 0;
8996 		index_in_dcore = 1;
8997 	break;
8998 	case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP:
8999 	case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR:
9000 		dcore = 1;
9001 		index_in_dcore = 1;
9002 	break;
9003 	case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP:
9004 	case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR:
9005 		dcore = 3;
9006 		index_in_dcore = 2;
9007 	break;
9008 	case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP:
9009 	case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR:
9010 		dcore = 2;
9011 		index_in_dcore = 2;
9012 	break;
9013 	case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP:
9014 	case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR:
9015 		dcore = 3;
9016 		index_in_dcore = 3;
9017 	break;
9018 	case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP:
9019 	case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR:
9020 		dcore = 2;
9021 		index_in_dcore = 3;
9022 	break;
9023 	case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP:
9024 	case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR:
9025 		dcore = 0;
9026 		index_in_dcore = 2;
9027 	break;
9028 	case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP:
9029 	case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR:
9030 		dcore = 1;
9031 		index_in_dcore = 2;
9032 	break;
9033 	case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP:
9034 	case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR:
9035 		dcore = 0;
9036 		index_in_dcore = 3;
9037 	break;
9038 	case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP:
9039 	case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR:
9040 		dcore = 1;
9041 		index_in_dcore = 3;
9042 	break;
9043 	case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
9044 	case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
9045 		dcore = 3;
9046 		index_in_dcore = 0;
9047 	break;
9048 	case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP:
9049 	case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR:
9050 		dcore = 2;
9051 		index_in_dcore = 0;
9052 	break;
9053 	case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP:
9054 	case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR:
9055 		dcore = 3;
9056 		index_in_dcore = 1;
9057 	break;
9058 	case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP:
9059 	case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR:
9060 		dcore = 2;
9061 		index_in_dcore = 1;
9062 	break;
9063 	default:
9064 		return ULONG_MAX;
9065 	}
9066 
9067 	return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET;
9068 }
9069 
9070 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
9071 {
9072 	bool is_pmmu = false;
9073 	u32 error_count = 0;
9074 	u64 mmu_base;
9075 
9076 	switch (event_type) {
9077 	case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
9078 	case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
9079 		mmu_base = get_hmmu_base(event_type);
9080 		break;
9081 
9082 	case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
9083 	case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
9084 		is_pmmu = true;
9085 		mmu_base = mmPMMU_HBW_MMU_BASE;
9086 		break;
9087 	default:
9088 		return 0;
9089 	}
9090 
9091 	if (mmu_base == ULONG_MAX)
9092 		return 0;
9093 
9094 	error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base,
9095 							is_pmmu, event_mask);
9096 	hl_check_for_glbl_errors(hdev);
9097 
9098 	return error_count;
9099 }
9100 
9101 
9102 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */
9103 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev,
9104 			struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt)
9105 {
9106 	u32 addr, beat, beat_shift;
9107 	bool rc = false;
9108 
9109 	dev_err_ratelimited(hdev->dev,
9110 			"READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n",
9111 			FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt),
9112 			FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt),
9113 			FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt));
9114 
9115 	addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val);
9116 	dev_err_ratelimited(hdev->dev,
9117 			"READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n",
9118 			FIELD_GET(HBM_RD_ADDR_SID_MASK, addr),
9119 			FIELD_GET(HBM_RD_ADDR_BG_MASK, addr),
9120 			FIELD_GET(HBM_RD_ADDR_BA_MASK, addr),
9121 			FIELD_GET(HBM_RD_ADDR_COL_MASK, addr),
9122 			FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr));
9123 
9124 	/* For each beat (RDQS edge), look for possible errors and print relevant info */
9125 	for (beat = 0 ; beat < 4 ; beat++) {
9126 		if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9127 			(HBM_RD_ERR_SERR_BEAT0_MASK << beat))
9128 			dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n",
9129 						beat,
9130 						le32_to_cpu(rd_err_data->dbg_rd_err_dm),
9131 						le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));
9132 
9133 		if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9134 			(HBM_RD_ERR_DERR_BEAT0_MASK << beat)) {
9135 			dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n",
9136 						beat,
9137 						le32_to_cpu(rd_err_data->dbg_rd_err_dm),
9138 						le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));
9139 			rc |= true;
9140 		}
9141 
9142 		beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT;
9143 		if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9144 			(HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) {
9145 			dev_err_ratelimited(hdev->dev,
9146 					"Beat%d read PARITY: DM: %#x, PAR data: %#x\n",
9147 					beat,
9148 					le32_to_cpu(rd_err_data->dbg_rd_err_dm),
9149 					(le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
9150 						(HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >>
9151 						(HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift));
9152 			rc |= true;
9153 		}
9154 
9155 		dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat);
9156 		dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
9157 					le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2]));
9158 		dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
9159 					le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1]));
9160 	}
9161 
9162 	return rc;
9163 }
9164 
9165 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev,
9166 			struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt)
9167 {
9168 	struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds;
9169 	u32 i, curr_addr, derr = wr_par_err_data->dbg_derr;
9170 
9171 	dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt);
9172 
9173 	dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n",
9174 				derr & 0x3, derr & 0xc);
9175 
9176 	/* JIRA H6-3286 - the following prints may not be valid */
9177 	dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n");
9178 	for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) {
9179 		curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr);
9180 		dev_err_ratelimited(hdev->dev,
9181 				"\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n",
9182 				i,
9183 				FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr),
9184 				FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr),
9185 				FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr),
9186 				FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr));
9187 	}
9188 }
9189 
9190 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev,
9191 		struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt)
9192 {
9193 	__le32 *col_cmd = ca_par_err_data->dbg_col;
9194 	__le16 *row_cmd = ca_par_err_data->dbg_row;
9195 	u32 i;
9196 
9197 	dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt);
9198 
9199 	dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n");
9200 	for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++)
9201 		dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i,
9202 			le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0),
9203 			le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0));
9204 }
9205 
9206 /* Returns true if hard reset is needed or false otherwise */
9207 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type,
9208 					struct hl_eq_hbm_sei_data *sei_data)
9209 {
9210 	bool require_hard_reset = false;
9211 	u32 hbm_id, mc_id, cause_idx;
9212 
9213 	hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4;
9214 	mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2;
9215 
9216 	cause_idx = sei_data->hdr.sei_cause;
9217 	if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) {
9218 		gaudi2_print_event(hdev, event_type, true,
9219 			"err cause: %s",
9220 			"Invalid HBM SEI event cause (%d) provided by FW", cause_idx);
9221 		return true;
9222 	}
9223 
9224 	gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical,
9225 		"System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s",
9226 		sei_data->hdr.is_critical ? "Critical" : "Non-critical",
9227 		hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel,
9228 		hbm_mc_sei_cause[cause_idx]);
9229 
9230 	/* Print error-specific info */
9231 	switch (cause_idx) {
9232 	case HBM_SEI_CATTRIP:
9233 		require_hard_reset = true;
9234 		break;
9235 
9236 	case  HBM_SEI_CMD_PARITY_EVEN:
9237 		gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info,
9238 						le32_to_cpu(sei_data->hdr.cnt));
9239 		require_hard_reset = true;
9240 		break;
9241 
9242 	case  HBM_SEI_CMD_PARITY_ODD:
9243 		gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info,
9244 						le32_to_cpu(sei_data->hdr.cnt));
9245 		require_hard_reset = true;
9246 		break;
9247 
9248 	case HBM_SEI_WRITE_DATA_PARITY_ERR:
9249 		gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info,
9250 						le32_to_cpu(sei_data->hdr.cnt));
9251 		require_hard_reset = true;
9252 		break;
9253 
9254 	case HBM_SEI_READ_ERR:
9255 		/* Unlike other SEI events, read error requires further processing of the
9256 		 * raw data in order to determine the root cause.
9257 		 */
9258 		require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev,
9259 								&sei_data->read_err_info,
9260 								le32_to_cpu(sei_data->hdr.cnt));
9261 		break;
9262 
9263 	default:
9264 		break;
9265 	}
9266 
9267 	require_hard_reset |= !!sei_data->hdr.is_critical;
9268 
9269 	return require_hard_reset;
9270 }
9271 
9272 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type,
9273 				u64 intr_cause_data)
9274 {
9275 	if (intr_cause_data) {
9276 		gaudi2_print_event(hdev, event_type, true,
9277 			"temperature error cause: %#llx", intr_cause_data);
9278 		return 1;
9279 	}
9280 
9281 	return 0;
9282 }
9283 
9284 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data)
9285 {
9286 	u32 i, error_count = 0;
9287 
9288 	for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++)
9289 		if (intr_cause_data & hbm_mc_spi[i].mask) {
9290 			dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n",
9291 				hbm_mc_spi[i].cause);
9292 			error_count++;
9293 		}
9294 
9295 	return error_count;
9296 }
9297 
9298 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask)
9299 {
9300 	ktime_t zero_time = ktime_set(0, 0);
9301 
9302 	mutex_lock(&hdev->clk_throttling.lock);
9303 
9304 	switch (event_type) {
9305 	case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S:
9306 		hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
9307 		hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
9308 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
9309 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
9310 		dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n");
9311 		break;
9312 
9313 	case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E:
9314 		hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
9315 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
9316 		dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n");
9317 		break;
9318 
9319 	case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S:
9320 		hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
9321 		hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
9322 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
9323 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
9324 		*event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9325 		dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n");
9326 		break;
9327 
9328 	case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
9329 		hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
9330 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
9331 		*event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9332 		dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n");
9333 		break;
9334 
9335 	default:
9336 		dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type);
9337 		break;
9338 	}
9339 
9340 	mutex_unlock(&hdev->clk_throttling.lock);
9341 }
9342 
9343 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type,
9344 					struct cpucp_pkt_sync_err *sync_err)
9345 {
9346 	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
9347 
9348 	gaudi2_print_event(hdev, event_type, false,
9349 		"FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d",
9350 		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci),
9351 		q->pi, atomic_read(&q->ci));
9352 }
9353 
9354 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type)
9355 {
9356 	u32 p2p_intr, msix_gw_intr, error_count = 0;
9357 
9358 	p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR);
9359 	msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR);
9360 
9361 	if (p2p_intr) {
9362 		gaudi2_print_event(hdev, event_type, true,
9363 			"pcie p2p transaction terminated due to security, req_id(0x%x)",
9364 			RREG32(mmPCIE_WRAP_P2P_REQ_ID));
9365 
9366 		WREG32(mmPCIE_WRAP_P2P_INTR, 0x1);
9367 		error_count++;
9368 	}
9369 
9370 	if (msix_gw_intr) {
9371 		gaudi2_print_event(hdev, event_type, true,
9372 			"pcie msi-x gen denied due to vector num check failure, vec(0x%X)",
9373 			RREG32(mmPCIE_WRAP_MSIX_GW_VEC));
9374 
9375 		WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1);
9376 		error_count++;
9377 	}
9378 
9379 	return error_count;
9380 }
9381 
9382 static int gaudi2_handle_pcie_drain(struct hl_device *hdev,
9383 			struct hl_eq_pcie_drain_ind_data *drain_data)
9384 {
9385 	u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause, error_count = 0;
9386 
9387 	cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data);
9388 	lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw);
9389 	lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw);
9390 	hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw);
9391 	hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw);
9392 
9393 	if (cause & BIT_ULL(0)) {
9394 		dev_err_ratelimited(hdev->dev,
9395 			"PCIE AXI drain LBW completed, read_err %u, write_err %u\n",
9396 			!!lbw_rd, !!lbw_wr);
9397 		error_count++;
9398 	}
9399 
9400 	if (cause & BIT_ULL(1)) {
9401 		dev_err_ratelimited(hdev->dev,
9402 			"PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n",
9403 			hbw_rd, hbw_wr);
9404 		error_count++;
9405 	}
9406 
9407 	return error_count;
9408 }
9409 
9410 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data)
9411 {
9412 	u32 error_count = 0;
9413 	int i;
9414 
9415 	for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) {
9416 		if (intr_cause_data & BIT_ULL(i)) {
9417 			dev_err_ratelimited(hdev->dev, "PSOC %s completed\n",
9418 				gaudi2_psoc_axi_drain_interrupts_cause[i]);
9419 			error_count++;
9420 		}
9421 	}
9422 
9423 	hl_check_for_glbl_errors(hdev);
9424 
9425 	return error_count;
9426 }
9427 
9428 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type,
9429 					struct cpucp_pkt_sync_err *sync_err)
9430 {
9431 	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
9432 
9433 	gaudi2_print_event(hdev, event_type, false,
9434 		"FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d",
9435 		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));
9436 }
9437 
9438 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type,
9439 					struct hl_eq_engine_arc_intr_data *data)
9440 {
9441 	struct hl_engine_arc_dccm_queue_full_irq *q;
9442 	u32 intr_type, engine_id;
9443 	u64 payload;
9444 
9445 	intr_type = le32_to_cpu(data->intr_type);
9446 	engine_id = le32_to_cpu(data->engine_id);
9447 	payload = le64_to_cpu(data->payload);
9448 
9449 	switch (intr_type) {
9450 	case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ:
9451 		q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload;
9452 
9453 		gaudi2_print_event(hdev, event_type, true,
9454 				"ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u",
9455 				engine_id, intr_type, q->queue_index);
9456 		return 1;
9457 	default:
9458 		gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type");
9459 		return 0;
9460 	}
9461 }
9462 
9463 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
9464 {
9465 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
9466 	bool reset_required = false, is_critical = false;
9467 	u32 index, ctl, reset_flags = 0, error_count = 0;
9468 	u64 event_mask = 0;
9469 	u16 event_type;
9470 
9471 	ctl = le32_to_cpu(eq_entry->hdr.ctl);
9472 	event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
9473 
9474 	if (event_type >= GAUDI2_EVENT_SIZE) {
9475 		dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
9476 				event_type, GAUDI2_EVENT_SIZE - 1);
9477 		return;
9478 	}
9479 
9480 	gaudi2->events_stat[event_type]++;
9481 	gaudi2->events_stat_aggregate[event_type]++;
9482 
9483 	switch (event_type) {
9484 	case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR:
9485 		fallthrough;
9486 	case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR:
9487 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9488 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9489 		reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
9490 		is_critical = eq_entry->ecc_data.is_critical;
9491 		error_count++;
9492 		break;
9493 
9494 	case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM:
9495 		fallthrough;
9496 	case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM:
9497 		fallthrough;
9498 	case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1:
9499 		error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask);
9500 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9501 		break;
9502 
9503 	case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0:
9504 		error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type);
9505 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9506 		break;
9507 
9508 	case GAUDI2_EVENT_CPU_AXI_ERR_RSP:
9509 		error_count = gaudi2_handle_cpu_sei_err(hdev, event_type);
9510 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9511 		event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
9512 		break;
9513 
9514 	case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
9515 	case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
9516 		error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask);
9517 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9518 		break;
9519 
9520 	case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
9521 	case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
9522 		index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
9523 		error_count = gaudi2_handle_rot_err(hdev, index, event_type,
9524 					&eq_entry->razwi_with_intr_cause, &event_mask);
9525 		error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9526 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9527 		break;
9528 
9529 	case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
9530 		index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
9531 		error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type,
9532 						&eq_entry->razwi_with_intr_cause, &event_mask);
9533 		error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9534 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9535 		break;
9536 
9537 	case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE:
9538 		index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;
9539 		error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask);
9540 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9541 		break;
9542 
9543 	case GAUDI2_EVENT_TPC0_KERNEL_ERR:
9544 	case GAUDI2_EVENT_TPC1_KERNEL_ERR:
9545 	case GAUDI2_EVENT_TPC2_KERNEL_ERR:
9546 	case GAUDI2_EVENT_TPC3_KERNEL_ERR:
9547 	case GAUDI2_EVENT_TPC4_KERNEL_ERR:
9548 	case GAUDI2_EVENT_TPC5_KERNEL_ERR:
9549 	case GAUDI2_EVENT_TPC6_KERNEL_ERR:
9550 	case GAUDI2_EVENT_TPC7_KERNEL_ERR:
9551 	case GAUDI2_EVENT_TPC8_KERNEL_ERR:
9552 	case GAUDI2_EVENT_TPC9_KERNEL_ERR:
9553 	case GAUDI2_EVENT_TPC10_KERNEL_ERR:
9554 	case GAUDI2_EVENT_TPC11_KERNEL_ERR:
9555 	case GAUDI2_EVENT_TPC12_KERNEL_ERR:
9556 	case GAUDI2_EVENT_TPC13_KERNEL_ERR:
9557 	case GAUDI2_EVENT_TPC14_KERNEL_ERR:
9558 	case GAUDI2_EVENT_TPC15_KERNEL_ERR:
9559 	case GAUDI2_EVENT_TPC16_KERNEL_ERR:
9560 	case GAUDI2_EVENT_TPC17_KERNEL_ERR:
9561 	case GAUDI2_EVENT_TPC18_KERNEL_ERR:
9562 	case GAUDI2_EVENT_TPC19_KERNEL_ERR:
9563 	case GAUDI2_EVENT_TPC20_KERNEL_ERR:
9564 	case GAUDI2_EVENT_TPC21_KERNEL_ERR:
9565 	case GAUDI2_EVENT_TPC22_KERNEL_ERR:
9566 	case GAUDI2_EVENT_TPC23_KERNEL_ERR:
9567 	case GAUDI2_EVENT_TPC24_KERNEL_ERR:
9568 		index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) /
9569 			(GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR);
9570 		error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type,
9571 					&eq_entry->razwi_with_intr_cause, &event_mask);
9572 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9573 		break;
9574 
9575 	case GAUDI2_EVENT_DEC0_SPI:
9576 	case GAUDI2_EVENT_DEC1_SPI:
9577 	case GAUDI2_EVENT_DEC2_SPI:
9578 	case GAUDI2_EVENT_DEC3_SPI:
9579 	case GAUDI2_EVENT_DEC4_SPI:
9580 	case GAUDI2_EVENT_DEC5_SPI:
9581 	case GAUDI2_EVENT_DEC6_SPI:
9582 	case GAUDI2_EVENT_DEC7_SPI:
9583 	case GAUDI2_EVENT_DEC8_SPI:
9584 	case GAUDI2_EVENT_DEC9_SPI:
9585 		index = (event_type - GAUDI2_EVENT_DEC0_SPI) /
9586 				(GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI);
9587 		error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask);
9588 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9589 		break;
9590 
9591 	case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
9592 	case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
9593 	case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
9594 	case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
9595 		index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /
9596 				(GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
9597 						GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
9598 		error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask);
9599 		error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask);
9600 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9601 		break;
9602 
9603 	case GAUDI2_EVENT_MME0_QMAN_SW_ERROR:
9604 	case GAUDI2_EVENT_MME1_QMAN_SW_ERROR:
9605 	case GAUDI2_EVENT_MME2_QMAN_SW_ERROR:
9606 	case GAUDI2_EVENT_MME3_QMAN_SW_ERROR:
9607 		index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) /
9608 				(GAUDI2_EVENT_MME1_QMAN_SW_ERROR -
9609 					GAUDI2_EVENT_MME0_QMAN_SW_ERROR);
9610 		error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask);
9611 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9612 		break;
9613 
9614 	case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID:
9615 	case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID:
9616 	case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID:
9617 	case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID:
9618 		index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) /
9619 				(GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID -
9620 					GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID);
9621 		error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask);
9622 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9623 		break;
9624 
9625 	case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP:
9626 	case GAUDI2_EVENT_KDMA0_CORE:
9627 		error_count = gaudi2_handle_kdma_core_event(hdev, event_type,
9628 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9629 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9630 		break;
9631 
9632 	case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE:
9633 		error_count = gaudi2_handle_dma_core_event(hdev, event_type,
9634 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9635 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9636 		break;
9637 
9638 	case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE:
9639 		error_count = gaudi2_handle_dma_core_event(hdev, event_type,
9640 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9641 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9642 		break;
9643 
9644 	case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR:
9645 		error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type,
9646 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask);
9647 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9648 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9649 		break;
9650 
9651 	case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
9652 	case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
9653 	case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
9654 	case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
9655 		error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask);
9656 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9657 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9658 		break;
9659 
9660 	case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL:
9661 		error_count = gaudi2_handle_hif_fatal(hdev, event_type,
9662 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9663 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9664 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9665 		break;
9666 
9667 	case GAUDI2_EVENT_PMMU_FATAL_0:
9668 		error_count = gaudi2_handle_pif_fatal(hdev, event_type,
9669 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9670 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9671 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9672 		break;
9673 
9674 	case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT:
9675 		error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask);
9676 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9677 		break;
9678 
9679 	case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE:
9680 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9681 		if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) {
9682 			reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9683 			reset_required = true;
9684 		}
9685 		error_count++;
9686 		break;
9687 
9688 	case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5:
9689 		error_count = gaudi2_handle_hbm_cattrip(hdev, event_type,
9690 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9691 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9692 		break;
9693 
9694 	case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI:
9695 		error_count = gaudi2_handle_hbm_mc_spi(hdev,
9696 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9697 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9698 		break;
9699 
9700 	case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE:
9701 		error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);
9702 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9703 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9704 		break;
9705 
9706 	case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN:
9707 		error_count = gaudi2_handle_psoc_drain(hdev,
9708 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9709 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9710 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9711 		break;
9712 
9713 	case GAUDI2_EVENT_CPU_AXI_ECC:
9714 		error_count = GAUDI2_NA_EVENT_CAUSE;
9715 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9716 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9717 		break;
9718 	case GAUDI2_EVENT_CPU_L2_RAM_ECC:
9719 		error_count = GAUDI2_NA_EVENT_CAUSE;
9720 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9721 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9722 		break;
9723 	case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP:
9724 	case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP:
9725 	case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP:
9726 	case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP:
9727 		error_count = gaudi2_handle_mme_sbte_err(hdev, event_type,
9728 						le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
9729 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9730 		break;
9731 	case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B:
9732 		error_count = GAUDI2_NA_EVENT_CAUSE;
9733 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9734 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9735 		break;
9736 	case GAUDI2_EVENT_PSOC_AXI_ERR_RSP:
9737 		error_count = GAUDI2_NA_EVENT_CAUSE;
9738 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9739 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9740 		break;
9741 	case GAUDI2_EVENT_PSOC_PRSTN_FALL:
9742 		error_count = GAUDI2_NA_EVENT_CAUSE;
9743 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9744 		break;
9745 	case GAUDI2_EVENT_PCIE_APB_TIMEOUT:
9746 		error_count = GAUDI2_NA_EVENT_CAUSE;
9747 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9748 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9749 		break;
9750 	case GAUDI2_EVENT_PCIE_FATAL_ERR:
9751 		error_count = GAUDI2_NA_EVENT_CAUSE;
9752 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9753 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9754 		break;
9755 	case GAUDI2_EVENT_TPC0_BMON_SPMU:
9756 	case GAUDI2_EVENT_TPC1_BMON_SPMU:
9757 	case GAUDI2_EVENT_TPC2_BMON_SPMU:
9758 	case GAUDI2_EVENT_TPC3_BMON_SPMU:
9759 	case GAUDI2_EVENT_TPC4_BMON_SPMU:
9760 	case GAUDI2_EVENT_TPC5_BMON_SPMU:
9761 	case GAUDI2_EVENT_TPC6_BMON_SPMU:
9762 	case GAUDI2_EVENT_TPC7_BMON_SPMU:
9763 	case GAUDI2_EVENT_TPC8_BMON_SPMU:
9764 	case GAUDI2_EVENT_TPC9_BMON_SPMU:
9765 	case GAUDI2_EVENT_TPC10_BMON_SPMU:
9766 	case GAUDI2_EVENT_TPC11_BMON_SPMU:
9767 	case GAUDI2_EVENT_TPC12_BMON_SPMU:
9768 	case GAUDI2_EVENT_TPC13_BMON_SPMU:
9769 	case GAUDI2_EVENT_TPC14_BMON_SPMU:
9770 	case GAUDI2_EVENT_TPC15_BMON_SPMU:
9771 	case GAUDI2_EVENT_TPC16_BMON_SPMU:
9772 	case GAUDI2_EVENT_TPC17_BMON_SPMU:
9773 	case GAUDI2_EVENT_TPC18_BMON_SPMU:
9774 	case GAUDI2_EVENT_TPC19_BMON_SPMU:
9775 	case GAUDI2_EVENT_TPC20_BMON_SPMU:
9776 	case GAUDI2_EVENT_TPC21_BMON_SPMU:
9777 	case GAUDI2_EVENT_TPC22_BMON_SPMU:
9778 	case GAUDI2_EVENT_TPC23_BMON_SPMU:
9779 	case GAUDI2_EVENT_TPC24_BMON_SPMU:
9780 	case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU:
9781 	case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU:
9782 	case GAUDI2_EVENT_MME0_WAP_BMON_SPMU:
9783 	case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU:
9784 	case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU:
9785 	case GAUDI2_EVENT_MME1_WAP_BMON_SPMU:
9786 	case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU:
9787 	case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU:
9788 	case GAUDI2_EVENT_MME2_WAP_BMON_SPMU:
9789 	case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU:
9790 	case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU:
9791 	case GAUDI2_EVENT_MME3_WAP_BMON_SPMU:
9792 	case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU:
9793 		fallthrough;
9794 	case GAUDI2_EVENT_DEC0_BMON_SPMU:
9795 	case GAUDI2_EVENT_DEC1_BMON_SPMU:
9796 	case GAUDI2_EVENT_DEC2_BMON_SPMU:
9797 	case GAUDI2_EVENT_DEC3_BMON_SPMU:
9798 	case GAUDI2_EVENT_DEC4_BMON_SPMU:
9799 	case GAUDI2_EVENT_DEC5_BMON_SPMU:
9800 	case GAUDI2_EVENT_DEC6_BMON_SPMU:
9801 	case GAUDI2_EVENT_DEC7_BMON_SPMU:
9802 	case GAUDI2_EVENT_DEC8_BMON_SPMU:
9803 	case GAUDI2_EVENT_DEC9_BMON_SPMU:
9804 	case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU:
9805 		error_count = GAUDI2_NA_EVENT_CAUSE;
9806 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9807 		break;
9808 
9809 	case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S:
9810 	case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E:
9811 	case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S:
9812 	case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
9813 		gaudi2_print_clk_change_info(hdev, event_type, &event_mask);
9814 		error_count = GAUDI2_NA_EVENT_CAUSE;
9815 		break;
9816 
9817 	case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC:
9818 		gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err);
9819 		error_count = GAUDI2_NA_EVENT_CAUSE;
9820 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9821 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9822 		break;
9823 
9824 	case GAUDI2_EVENT_PCIE_FLR_REQUESTED:
9825 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9826 		error_count = GAUDI2_NA_EVENT_CAUSE;
9827 		/* Do nothing- FW will handle it */
9828 		break;
9829 
9830 	case GAUDI2_EVENT_PCIE_P2P_MSIX:
9831 		error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type);
9832 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9833 		break;
9834 
9835 	case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE:
9836 		index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE;
9837 		error_count = gaudi2_handle_sm_err(hdev, event_type, index);
9838 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9839 		break;
9840 
9841 	case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR:
9842 		error_count = GAUDI2_NA_EVENT_CAUSE;
9843 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9844 		break;
9845 
9846 	case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
9847 		dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n",
9848 						le64_to_cpu(eq_entry->data[0]));
9849 		error_count = GAUDI2_NA_EVENT_CAUSE;
9850 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9851 		break;
9852 	case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT:
9853 		dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n",
9854 						le64_to_cpu(eq_entry->data[0]));
9855 		error_count = GAUDI2_NA_EVENT_CAUSE;
9856 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9857 		break;
9858 
9859 	case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED:
9860 		gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err);
9861 		error_count = GAUDI2_NA_EVENT_CAUSE;
9862 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
9863 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9864 		break;
9865 
9866 	case GAUDI2_EVENT_ARC_DCCM_FULL:
9867 		error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data);
9868 		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
9869 		break;
9870 
9871 	case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED:
9872 	case GAUDI2_EVENT_CPU_DEV_RESET_REQ:
9873 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
9874 		error_count = GAUDI2_NA_EVENT_CAUSE;
9875 		is_critical = true;
9876 		break;
9877 
9878 	default:
9879 		if (gaudi2_irq_map_table[event_type].valid) {
9880 			dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
9881 						event_type);
9882 			error_count = GAUDI2_NA_EVENT_CAUSE;
9883 		}
9884 	}
9885 
9886 	/* Make sure to dump an error in case no error cause was printed so far.
9887 	 * Note that although we have counted the errors, we use this number as
9888 	 * a boolean.
9889 	 */
9890 	if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type))
9891 		gaudi2_print_event(hdev, event_type, true, "%d", event_type);
9892 	else if (error_count == 0)
9893 		gaudi2_print_event(hdev, event_type, true,
9894 				"No error cause for H/W event %u", event_type);
9895 
9896 	if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) ||
9897 				reset_required) {
9898 		if (reset_required ||
9899 				(gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD))
9900 			reset_flags |= HL_DRV_RESET_HARD;
9901 
9902 		if (hdev->hard_reset_on_fw_events ||
9903 				(hdev->asic_prop.fw_security_enabled && is_critical))
9904 			goto reset_device;
9905 	}
9906 
9907 	/* Send unmask irq only for interrupts not classified as MSG */
9908 	if (!gaudi2_irq_map_table[event_type].msg)
9909 		hl_fw_unmask_irq(hdev, event_type);
9910 
9911 	if (event_mask)
9912 		hl_notifier_event_send_all(hdev, event_mask);
9913 
9914 	return;
9915 
9916 reset_device:
9917 	if (hdev->asic_prop.fw_security_enabled && is_critical) {
9918 		reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW;
9919 		event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
9920 	} else {
9921 		reset_flags |= HL_DRV_RESET_DELAY;
9922 	}
9923 	/* escalate general hw errors to critical/fatal error */
9924 	if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR)
9925 		hl_handle_critical_hw_err(hdev, event_type, &event_mask);
9926 
9927 	event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;
9928 	hl_device_cond_reset(hdev, reset_flags, event_mask);
9929 }
9930 
9931 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev,
9932 			struct packet_lin_dma *lin_dma_pkt, dma_addr_t pkt_dma_addr,
9933 			u32 hw_queue_id, u32 size, u64 addr, u32 val)
9934 {
9935 	u32 ctl, pkt_size;
9936 	int rc = 0;
9937 
9938 	ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA);
9939 	ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1);
9940 	ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1);
9941 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1);
9942 
9943 	lin_dma_pkt->ctl = cpu_to_le32(ctl);
9944 	lin_dma_pkt->src_addr = cpu_to_le64(val);
9945 	lin_dma_pkt->dst_addr = cpu_to_le64(addr);
9946 	lin_dma_pkt->tsize = cpu_to_le32(size);
9947 
9948 	pkt_size = sizeof(struct packet_lin_dma);
9949 
9950 	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr);
9951 	if (rc)
9952 		dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n",
9953 				hw_queue_id);
9954 
9955 	return rc;
9956 }
9957 
9958 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val)
9959 {
9960 	u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
9961 					GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0,
9962 					GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0,
9963 					GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0};
9964 	u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val,
9965 		old_mmubp, mmubp, num_of_pkts, busy, pkt_size;
9966 	u64 comp_addr, cur_addr = addr, end_addr = addr + size;
9967 	struct asic_fixed_properties *prop = &hdev->asic_prop;
9968 	void *lin_dma_pkts_arr;
9969 	dma_addr_t pkt_dma_addr;
9970 	int rc = 0, dma_num = 0;
9971 
9972 	if (prop->edma_enabled_mask == 0) {
9973 		dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");
9974 		return -EIO;
9975 	}
9976 
9977 	sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
9978 	sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
9979 	comp_addr = CFG_BASE + sob_addr;
9980 	comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
9981 		FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
9982 	mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) |
9983 		FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1);
9984 
9985 	/* Calculate how many lin dma pkts we'll need */
9986 	num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G);
9987 	pkt_size = sizeof(struct packet_lin_dma);
9988 
9989 	lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts,
9990 					&pkt_dma_addr, GFP_KERNEL);
9991 	if (!lin_dma_pkts_arr)
9992 		return -ENOMEM;
9993 
9994 	/*
9995 	 * set mmu bypass for the scrubbing - all ddmas are configured the same so save
9996 	 * only the first one to restore later
9997 	 * also set the sob addr for all edma cores for completion.
9998 	 * set QM as trusted to allow it to access physical address with MMU bp.
9999 	 */
10000 	old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP);
10001 	for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
10002 		for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
10003 			u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET;
10004 			u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
10005 
10006 			if (!(prop->edma_enabled_mask & BIT(edma_bit)))
10007 				continue;
10008 
10009 			WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP +
10010 					edma_offset, mmubp);
10011 			WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset,
10012 					lower_32_bits(comp_addr));
10013 			WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset,
10014 					upper_32_bits(comp_addr));
10015 			WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset,
10016 					comp_val);
10017 			gaudi2_qman_set_test_mode(hdev,
10018 					edma_queues_id[dcore] + 4 * edma_idx, true);
10019 		}
10020 	}
10021 
10022 	WREG32(sob_addr, 0);
10023 
10024 	while (cur_addr < end_addr) {
10025 		for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
10026 			for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
10027 				u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
10028 
10029 				if (!(prop->edma_enabled_mask & BIT(edma_bit)))
10030 					continue;
10031 
10032 				chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr);
10033 
10034 				rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev,
10035 					(struct packet_lin_dma *)lin_dma_pkts_arr + dma_num,
10036 					pkt_dma_addr + dma_num * pkt_size,
10037 					edma_queues_id[dcore] + edma_idx * 4,
10038 					chunk_size, cur_addr, val);
10039 				if (rc)
10040 					goto end;
10041 
10042 				dma_num++;
10043 				cur_addr += chunk_size;
10044 				if (cur_addr == end_addr)
10045 					break;
10046 			}
10047 		}
10048 	}
10049 
10050 	rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000);
10051 	if (rc) {
10052 		dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n");
10053 		goto end;
10054 	}
10055 end:
10056 	for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
10057 		for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
10058 			u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET;
10059 			u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
10060 
10061 			if (!(prop->edma_enabled_mask & BIT(edma_bit)))
10062 				continue;
10063 
10064 			WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp);
10065 			WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0);
10066 			WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0);
10067 			WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0);
10068 			gaudi2_qman_set_test_mode(hdev,
10069 					edma_queues_id[dcore] + 4 * edma_idx, false);
10070 		}
10071 	}
10072 
10073 	WREG32(sob_addr, 0);
10074 	hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr);
10075 
10076 	return rc;
10077 }
10078 
10079 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val)
10080 {
10081 	int rc;
10082 	struct asic_fixed_properties *prop = &hdev->asic_prop;
10083 	u64 size = prop->dram_end_address - prop->dram_user_base_address;
10084 
10085 	rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val);
10086 
10087 	if (rc)
10088 		dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n",
10089 				prop->dram_user_base_address, size);
10090 	return rc;
10091 }
10092 
10093 static int gaudi2_scrub_device_mem(struct hl_device *hdev)
10094 {
10095 	int rc;
10096 	struct asic_fixed_properties *prop = &hdev->asic_prop;
10097 	u64 val = hdev->memory_scrub_val;
10098 	u64 addr, size;
10099 
10100 	if (!hdev->memory_scrub)
10101 		return 0;
10102 
10103 	/* scrub SRAM */
10104 	addr = prop->sram_user_base_address;
10105 	size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET);
10106 	dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n",
10107 			addr, addr + size, val);
10108 	rc = gaudi2_memset_device_memory(hdev, addr, size, val);
10109 	if (rc) {
10110 		dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc);
10111 		return rc;
10112 	}
10113 
10114 	/* scrub DRAM */
10115 	rc = gaudi2_scrub_device_dram(hdev, val);
10116 	if (rc) {
10117 		dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc);
10118 		return rc;
10119 	}
10120 	return 0;
10121 }
10122 
10123 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev)
10124 {
10125 	u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr,
10126 		cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr;
10127 	u32 val, size, offset;
10128 	int dcore_id;
10129 
10130 	offset = hdev->asic_prop.first_available_cq[0] * 4;
10131 	cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset;
10132 	cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset;
10133 	cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset;
10134 	cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset;
10135 	cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset;
10136 	cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset;
10137 	size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 -
10138 			(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset);
10139 
10140 	/* memset dcore0 CQ registers */
10141 	gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
10142 	gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
10143 	gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
10144 	gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
10145 	gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
10146 	gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
10147 
10148 	cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET;
10149 	cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET;
10150 	cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET;
10151 	cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET;
10152 	cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET;
10153 	cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET;
10154 	size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0;
10155 
10156 	for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
10157 		gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
10158 		gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
10159 		gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
10160 		gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
10161 		gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
10162 		gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
10163 
10164 		cq_lbw_l_addr += DCORE_OFFSET;
10165 		cq_lbw_h_addr += DCORE_OFFSET;
10166 		cq_lbw_data_addr += DCORE_OFFSET;
10167 		cq_base_l_addr += DCORE_OFFSET;
10168 		cq_base_h_addr += DCORE_OFFSET;
10169 		cq_size_addr += DCORE_OFFSET;
10170 	}
10171 
10172 	offset = hdev->asic_prop.first_available_user_mon[0] * 4;
10173 	addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset;
10174 	val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT;
10175 	size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset);
10176 
10177 	/* memset dcore0 monitors */
10178 	gaudi2_memset_device_lbw(hdev, addr, size, val);
10179 
10180 	addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset;
10181 	gaudi2_memset_device_lbw(hdev, addr, size, 0);
10182 
10183 	mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET;
10184 	mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET;
10185 	size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0;
10186 
10187 	for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
10188 		gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val);
10189 		gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0);
10190 		mon_sts_addr += DCORE_OFFSET;
10191 		mon_cfg_addr += DCORE_OFFSET;
10192 	}
10193 
10194 	offset = hdev->asic_prop.first_available_user_sob[0] * 4;
10195 	addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset;
10196 	val = 0;
10197 	size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 -
10198 			(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset);
10199 
10200 	/* memset dcore0 sobs */
10201 	gaudi2_memset_device_lbw(hdev, addr, size, val);
10202 
10203 	addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET;
10204 	size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0;
10205 
10206 	for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
10207 		gaudi2_memset_device_lbw(hdev, addr, size, val);
10208 		addr += DCORE_OFFSET;
10209 	}
10210 
10211 	/* Flush all WREG to prevent race */
10212 	val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset);
10213 }
10214 
10215 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev)
10216 {
10217 	u32 reg_base, hw_queue_id;
10218 
10219 	for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0;
10220 							hw_queue_id += NUM_OF_PQ_PER_QMAN) {
10221 		if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
10222 			continue;
10223 
10224 		gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
10225 
10226 		reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
10227 		WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0);
10228 	}
10229 
10230 	/* Flush all WREG to prevent race */
10231 	RREG32(mmPDMA0_QM_ARB_CFG_0);
10232 }
10233 
10234 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev)
10235 {
10236 	u32 reg_base, hw_queue_id;
10237 
10238 	for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3;
10239 							hw_queue_id += NUM_OF_PQ_PER_QMAN) {
10240 		if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
10241 			continue;
10242 
10243 		gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
10244 
10245 		reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
10246 		WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0);
10247 	}
10248 
10249 	/* Flush all WREG to prevent race */
10250 	RREG32(mmPDMA0_QM_ARB_CFG_0);
10251 }
10252 
10253 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid)
10254 {
10255 	return 0;
10256 }
10257 
10258 static void gaudi2_restore_phase_topology(struct hl_device *hdev)
10259 {
10260 }
10261 
10262 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx,
10263 						struct dup_block_ctx *cfg_ctx)
10264 {
10265 	u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off;
10266 	u8 seq;
10267 	int i;
10268 
10269 	for (i = 0 ; i < cfg_ctx->instances ; i++) {
10270 		seq = block_idx * cfg_ctx->instances + i;
10271 
10272 		/* skip disabled instance */
10273 		if (!(cfg_ctx->enabled_mask & BIT_ULL(seq)))
10274 			continue;
10275 
10276 		cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off,
10277 					cfg_ctx->data);
10278 	}
10279 }
10280 
10281 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx,
10282 						u64 mask)
10283 {
10284 	int i;
10285 
10286 	cfg_ctx->enabled_mask = mask;
10287 
10288 	for (i = 0 ; i < cfg_ctx->blocks ; i++)
10289 		gaudi2_init_block_instances(hdev, i, cfg_ctx);
10290 }
10291 
10292 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx)
10293 {
10294 	gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX);
10295 }
10296 
10297 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr)
10298 {
10299 	void *host_mem_virtual_addr;
10300 	dma_addr_t host_mem_dma_addr;
10301 	u64 reserved_va_base;
10302 	u32 pos, size_left, size_to_dma;
10303 	struct hl_ctx *ctx;
10304 	int rc = 0;
10305 
10306 	/* Fetch the ctx */
10307 	ctx = hl_get_compute_ctx(hdev);
10308 	if (!ctx) {
10309 		dev_err(hdev->dev, "No ctx available\n");
10310 		return -EINVAL;
10311 	}
10312 
10313 	/* Allocate buffers for read and for poll */
10314 	host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr,
10315 								GFP_KERNEL | __GFP_ZERO);
10316 	if (host_mem_virtual_addr == NULL) {
10317 		dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n");
10318 		rc = -ENOMEM;
10319 		goto put_ctx;
10320 	}
10321 
10322 	/* Reserve VM region on asic side */
10323 	reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M,
10324 						HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
10325 	if (!reserved_va_base) {
10326 		dev_err(hdev->dev, "Failed to reserve vmem on asic\n");
10327 		rc = -ENOMEM;
10328 		goto free_data_buffer;
10329 	}
10330 
10331 	/* Create mapping on asic side */
10332 	mutex_lock(&hdev->mmu_lock);
10333 
10334 	rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M);
10335 	if (rc) {
10336 		dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
10337 		goto unreserve_va;
10338 	}
10339 
10340 	rc = hl_mmu_invalidate_cache_range(hdev, false,
10341 				      MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV,
10342 				      ctx->asid, reserved_va_base, SZ_2M);
10343 	if (rc) {
10344 		hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
10345 		goto unreserve_va;
10346 	}
10347 
10348 	mutex_unlock(&hdev->mmu_lock);
10349 
10350 	/* Enable MMU on KDMA */
10351 	gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid);
10352 
10353 	pos = 0;
10354 	size_left = size;
10355 	size_to_dma = SZ_2M;
10356 
10357 	while (size_left > 0) {
10358 		if (size_left < SZ_2M)
10359 			size_to_dma = size_left;
10360 
10361 		rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false);
10362 		if (rc)
10363 			break;
10364 
10365 		memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma);
10366 
10367 		if (size_left <= SZ_2M)
10368 			break;
10369 
10370 		pos += SZ_2M;
10371 		addr += SZ_2M;
10372 		size_left -= SZ_2M;
10373 	}
10374 
10375 	gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID);
10376 
10377 	mutex_lock(&hdev->mmu_lock);
10378 
10379 	rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
10380 	if (rc)
10381 		goto unreserve_va;
10382 
10383 	rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
10384 				      ctx->asid, reserved_va_base, SZ_2M);
10385 
10386 unreserve_va:
10387 	mutex_unlock(&hdev->mmu_lock);
10388 	hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M);
10389 free_data_buffer:
10390 	hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr);
10391 put_ctx:
10392 	hl_ctx_put(ctx);
10393 
10394 	return rc;
10395 }
10396 
10397 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx)
10398 {
10399 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10400 	int min_alloc_order, rc;
10401 
10402 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
10403 		return 0;
10404 
10405 	hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,
10406 								HOST_SPACE_INTERNAL_CB_SZ,
10407 								&hdev->internal_cb_pool_dma_addr,
10408 								GFP_KERNEL | __GFP_ZERO);
10409 
10410 	if (!hdev->internal_cb_pool_virt_addr)
10411 		return -ENOMEM;
10412 
10413 	min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev),
10414 					gaudi2_get_wait_cb_size(hdev)));
10415 
10416 	hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);
10417 	if (!hdev->internal_cb_pool) {
10418 		dev_err(hdev->dev, "Failed to create internal CB pool\n");
10419 		rc = -ENOMEM;
10420 		goto free_internal_cb_pool;
10421 	}
10422 
10423 	rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr,
10424 				HOST_SPACE_INTERNAL_CB_SZ, -1);
10425 	if (rc) {
10426 		dev_err(hdev->dev, "Failed to add memory to internal CB pool\n");
10427 		rc = -EFAULT;
10428 		goto destroy_internal_cb_pool;
10429 	}
10430 
10431 	hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST,
10432 					HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
10433 
10434 	if (!hdev->internal_cb_va_base) {
10435 		rc = -ENOMEM;
10436 		goto destroy_internal_cb_pool;
10437 	}
10438 
10439 	mutex_lock(&hdev->mmu_lock);
10440 
10441 	rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,
10442 					HOST_SPACE_INTERNAL_CB_SZ);
10443 	if (rc)
10444 		goto unreserve_internal_cb_pool;
10445 
10446 	rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
10447 	if (rc)
10448 		goto unmap_internal_cb_pool;
10449 
10450 	mutex_unlock(&hdev->mmu_lock);
10451 
10452 	return 0;
10453 
10454 unmap_internal_cb_pool:
10455 	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10456 unreserve_internal_cb_pool:
10457 	mutex_unlock(&hdev->mmu_lock);
10458 	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10459 destroy_internal_cb_pool:
10460 	gen_pool_destroy(hdev->internal_cb_pool);
10461 free_internal_cb_pool:
10462 	hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
10463 					hdev->internal_cb_pool_dma_addr);
10464 
10465 	return rc;
10466 }
10467 
10468 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx)
10469 {
10470 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10471 
10472 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
10473 		return;
10474 
10475 	mutex_lock(&hdev->mmu_lock);
10476 	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10477 	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
10478 	hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
10479 	mutex_unlock(&hdev->mmu_lock);
10480 
10481 	gen_pool_destroy(hdev->internal_cb_pool);
10482 
10483 	hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
10484 					hdev->internal_cb_pool_dma_addr);
10485 }
10486 
10487 static void gaudi2_restore_user_registers(struct hl_device *hdev)
10488 {
10489 	gaudi2_restore_user_sm_registers(hdev);
10490 	gaudi2_restore_user_qm_registers(hdev);
10491 }
10492 
10493 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx)
10494 {
10495 	struct hl_device *hdev = ctx->hdev;
10496 	struct asic_fixed_properties *prop = &hdev->asic_prop;
10497 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10498 	int rc;
10499 
10500 	rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START,
10501 				gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true);
10502 	if (rc)
10503 		dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n",
10504 			RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START);
10505 
10506 	return rc;
10507 }
10508 
10509 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx)
10510 {
10511 	struct hl_device *hdev = ctx->hdev;
10512 	struct asic_fixed_properties *prop = &hdev->asic_prop;
10513 	int rc;
10514 
10515 	rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START,
10516 				prop->pmmu.page_size, true);
10517 	if (rc)
10518 		dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n",
10519 			RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START);
10520 }
10521 
10522 static int gaudi2_ctx_init(struct hl_ctx *ctx)
10523 {
10524 	int rc;
10525 
10526 	rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid);
10527 	if (rc)
10528 		return rc;
10529 
10530 	/* No need to clear user registers if the device has just
10531 	 * performed reset, we restore only nic qm registers
10532 	 */
10533 	if (ctx->hdev->reset_upon_device_release)
10534 		gaudi2_restore_nic_qm_registers(ctx->hdev);
10535 	else
10536 		gaudi2_restore_user_registers(ctx->hdev);
10537 
10538 	rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx);
10539 	if (rc)
10540 		return rc;
10541 
10542 	rc = gaudi2_map_virtual_msix_doorbell_memory(ctx);
10543 	if (rc)
10544 		gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
10545 
10546 	return rc;
10547 }
10548 
10549 static void gaudi2_ctx_fini(struct hl_ctx *ctx)
10550 {
10551 	if (ctx->asid == HL_KERNEL_ASID_ID)
10552 		return;
10553 
10554 	gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
10555 
10556 	gaudi2_unmap_virtual_msix_doorbell_memory(ctx);
10557 }
10558 
10559 static int gaudi2_pre_schedule_cs(struct hl_cs *cs)
10560 {
10561 	struct hl_device *hdev = cs->ctx->hdev;
10562 	int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
10563 	u32 mon_payload, sob_id, mon_id;
10564 
10565 	if (!cs_needs_completion(cs))
10566 		return 0;
10567 
10568 	/*
10569 	 * First 64 SOB/MON are reserved for driver for QMAN auto completion
10570 	 * mechanism. Each SOB/MON pair are used for a pending CS with the same
10571 	 * cyclic index. The SOB value is increased when each of the CS jobs is
10572 	 * completed. When the SOB reaches the number of CS jobs, the monitor
10573 	 * generates MSI-X interrupt.
10574 	 */
10575 
10576 	sob_id = mon_id = index;
10577 	mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) |
10578 				(1 << CQ_ENTRY_READY_SHIFT) | index;
10579 
10580 	gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload,
10581 				cs->jobs_cnt);
10582 
10583 	return 0;
10584 }
10585 
10586 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
10587 {
10588 	return HL_INVALID_QUEUE;
10589 }
10590 
10591 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb)
10592 {
10593 	struct hl_cb *cb = data;
10594 	struct packet_msg_short *pkt;
10595 	u32 value, ctl, pkt_size = sizeof(*pkt);
10596 
10597 	pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size);
10598 	memset(pkt, 0, pkt_size);
10599 
10600 	/* Inc by 1, Mode ADD */
10601 	value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1);
10602 	value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1);
10603 
10604 	ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4);
10605 	ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */
10606 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
10607 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb);
10608 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
10609 
10610 	pkt->value = cpu_to_le32(value);
10611 	pkt->ctl = cpu_to_le32(ctl);
10612 
10613 	return size + pkt_size;
10614 }
10615 
10616 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr)
10617 {
10618 	u32 ctl, pkt_size = sizeof(*pkt);
10619 
10620 	memset(pkt, 0, pkt_size);
10621 
10622 	ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr);
10623 	ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0);  /* MON base */
10624 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
10625 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
10626 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0);
10627 
10628 	pkt->value = cpu_to_le32(value);
10629 	pkt->ctl = cpu_to_le32(ctl);
10630 
10631 	return pkt_size;
10632 }
10633 
10634 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt,
10635 					u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr)
10636 {
10637 	u32 ctl, value, pkt_size = sizeof(*pkt);
10638 	u8 mask;
10639 
10640 	if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) {
10641 		dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask);
10642 		return 0;
10643 	}
10644 
10645 	memset(pkt, 0, pkt_size);
10646 
10647 	value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8);
10648 	value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val);
10649 	value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/
10650 	value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask);
10651 
10652 	ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr);
10653 	ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */
10654 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
10655 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
10656 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
10657 
10658 	pkt->value = cpu_to_le32(value);
10659 	pkt->ctl = cpu_to_le32(ctl);
10660 
10661 	return pkt_size;
10662 }
10663 
10664 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt)
10665 {
10666 	u32 ctl, cfg, pkt_size = sizeof(*pkt);
10667 
10668 	memset(pkt, 0, pkt_size);
10669 
10670 	cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1);
10671 	cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1);
10672 	cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2);
10673 
10674 	ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE);
10675 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
10676 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
10677 
10678 	pkt->cfg = cpu_to_le32(cfg);
10679 	pkt->ctl = cpu_to_le32(ctl);
10680 
10681 	return pkt_size;
10682 }
10683 
10684 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop)
10685 {
10686 	struct hl_cb *cb = prop->data;
10687 	void *buf = (void *) (uintptr_t) (cb->kernel_address);
10688 
10689 	u64 monitor_base, fence_addr = 0;
10690 	u32 stream_index, size = prop->size;
10691 	u16 msg_addr_offset;
10692 
10693 	stream_index = prop->q_idx % 4;
10694 	fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] +
10695 			QM_FENCE2_OFFSET + stream_index * 4;
10696 
10697 	/*
10698 	 * monitor_base should be the content of the base0 address registers,
10699 	 * so it will be added to the msg short offsets
10700 	 */
10701 	monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0;
10702 
10703 	/* First monitor config packet: low address of the sync */
10704 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) -
10705 				monitor_base;
10706 
10707 	size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset);
10708 
10709 	/* Second monitor config packet: high address of the sync */
10710 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) -
10711 				monitor_base;
10712 
10713 	size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset);
10714 
10715 	/*
10716 	 * Third monitor config packet: the payload, i.e. what to write when the
10717 	 * sync triggers
10718 	 */
10719 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) -
10720 				monitor_base;
10721 
10722 	size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset);
10723 
10724 	/* Fourth monitor config packet: bind the monitor to a sync object */
10725 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base;
10726 
10727 	size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask,
10728 						prop->sob_val, msg_addr_offset);
10729 
10730 	/* Fence packet */
10731 	size += gaudi2_add_fence_pkt(buf + size);
10732 
10733 	return size;
10734 }
10735 
10736 static void gaudi2_reset_sob(struct hl_device *hdev, void *data)
10737 {
10738 	struct hl_hw_sob *hw_sob = data;
10739 
10740 	dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id);
10741 
10742 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0);
10743 
10744 	kref_init(&hw_sob->kref);
10745 }
10746 
10747 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group)
10748 {
10749 }
10750 
10751 static u64 gaudi2_get_device_time(struct hl_device *hdev)
10752 {
10753 	u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;
10754 
10755 	return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
10756 }
10757 
10758 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs)
10759 {
10760 	return 0;
10761 }
10762 
10763 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx,
10764 					struct hl_cs *cs, u32 wait_queue_id,
10765 					u32 collective_engine_id, u32 encaps_signal_offset)
10766 {
10767 	return -EINVAL;
10768 }
10769 
10770 /*
10771  * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
10772  *                   to DMMU page-size address (64MB) before mapping it in
10773  *                   the MMU.
10774  * The operation is performed on both the virtual and physical addresses.
10775  * for device with 6 HBMs the scramble is:
10776  * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48]
10777  *
10778  * Example:
10779  * =============================================================================
10780  * Allocated DRAM  Reserved VA      scrambled VA for MMU mapping    Scrambled PA
10781  * Phys address                                                     in MMU last
10782  *                                                                    HOP
10783  * =============================================================================
10784  * PA1 0x3000000  VA1 0x9C000000  SVA1= (VA1/48M)*64M 0xD0000000  <- PA1/48M 0x1
10785  * PA2 0x9000000  VA2 0x9F000000  SVA2= (VA2/48M)*64M 0xD4000000  <- PA2/48M 0x3
10786  * =============================================================================
10787  */
10788 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr)
10789 {
10790 	struct asic_fixed_properties *prop = &hdev->asic_prop;
10791 	u32 divisor, mod_va;
10792 	u64 div_va;
10793 
10794 	/* accept any address in the DRAM address space */
10795 	if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE,
10796 									VA_HBM_SPACE_END)) {
10797 
10798 		divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;
10799 		div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va);
10800 		return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) |
10801 			(div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) |
10802 			(mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT);
10803 	}
10804 
10805 	return raw_addr;
10806 }
10807 
10808 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr)
10809 {
10810 	struct asic_fixed_properties *prop = &hdev->asic_prop;
10811 	u32 divisor, mod_va;
10812 	u64 div_va;
10813 
10814 	/* accept any address in the DRAM address space */
10815 	if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE,
10816 									VA_HBM_SPACE_END)) {
10817 
10818 		divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;
10819 		div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK,
10820 					PAGE_SIZE_64MB, &mod_va);
10821 
10822 		return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) +
10823 					(div_va * divisor + mod_va));
10824 	}
10825 
10826 	return scrambled_addr;
10827 }
10828 
10829 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id)
10830 {
10831 	u32 base = 0, dcore_id, dec_id;
10832 
10833 	if (core_id >= NUMBER_OF_DEC) {
10834 		dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id);
10835 		goto out;
10836 	}
10837 
10838 	if (core_id < 8) {
10839 		dcore_id = core_id / NUM_OF_DEC_PER_DCORE;
10840 		dec_id = core_id % NUM_OF_DEC_PER_DCORE;
10841 
10842 		base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET +
10843 				dec_id * DCORE_VDEC_OFFSET;
10844 	} else {
10845 		/* PCIe Shared Decoder */
10846 		base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET);
10847 	}
10848 out:
10849 	return base;
10850 }
10851 
10852 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr,
10853 				u32 *block_size, u32 *block_id)
10854 {
10855 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10856 	int i;
10857 
10858 	for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) {
10859 		if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) {
10860 			*block_id = i;
10861 			if (block_size)
10862 				*block_size = gaudi2->mapped_blocks[i].size;
10863 			return 0;
10864 		}
10865 	}
10866 
10867 	dev_err(hdev->dev, "Invalid block address %#llx", block_addr);
10868 
10869 	return -EINVAL;
10870 }
10871 
10872 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
10873 			u32 block_id, u32 block_size)
10874 {
10875 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10876 	u64 offset_in_bar;
10877 	u64 address;
10878 	int rc;
10879 
10880 	if (block_id >= NUM_USER_MAPPED_BLOCKS) {
10881 		dev_err(hdev->dev, "Invalid block id %u", block_id);
10882 		return -EINVAL;
10883 	}
10884 
10885 	/* we allow mapping only an entire block */
10886 	if (block_size != gaudi2->mapped_blocks[block_id].size) {
10887 		dev_err(hdev->dev, "Invalid block size %u", block_size);
10888 		return -EINVAL;
10889 	}
10890 
10891 	offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR;
10892 
10893 	address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar;
10894 
10895 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
10896 			VM_DONTCOPY | VM_NORESERVE);
10897 
10898 	rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT,
10899 			block_size, vma->vm_page_prot);
10900 	if (rc)
10901 		dev_err(hdev->dev, "remap_pfn_range error %d", rc);
10902 
10903 	return rc;
10904 }
10905 
10906 static void gaudi2_enable_events_from_fw(struct hl_device *hdev)
10907 {
10908 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10909 
10910 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
10911 	u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq);
10912 
10913 	if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)
10914 		WREG32(irq_handler_offset,
10915 			gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id);
10916 }
10917 
10918 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base)
10919 {
10920 	switch (mmu_id) {
10921 	case HW_CAP_DCORE0_DMMU0:
10922 		*mmu_base = mmDCORE0_HMMU0_MMU_BASE;
10923 		break;
10924 	case HW_CAP_DCORE0_DMMU1:
10925 		*mmu_base = mmDCORE0_HMMU1_MMU_BASE;
10926 		break;
10927 	case HW_CAP_DCORE0_DMMU2:
10928 		*mmu_base = mmDCORE0_HMMU2_MMU_BASE;
10929 		break;
10930 	case HW_CAP_DCORE0_DMMU3:
10931 		*mmu_base = mmDCORE0_HMMU3_MMU_BASE;
10932 		break;
10933 	case HW_CAP_DCORE1_DMMU0:
10934 		*mmu_base = mmDCORE1_HMMU0_MMU_BASE;
10935 		break;
10936 	case HW_CAP_DCORE1_DMMU1:
10937 		*mmu_base = mmDCORE1_HMMU1_MMU_BASE;
10938 		break;
10939 	case HW_CAP_DCORE1_DMMU2:
10940 		*mmu_base = mmDCORE1_HMMU2_MMU_BASE;
10941 		break;
10942 	case HW_CAP_DCORE1_DMMU3:
10943 		*mmu_base = mmDCORE1_HMMU3_MMU_BASE;
10944 		break;
10945 	case HW_CAP_DCORE2_DMMU0:
10946 		*mmu_base = mmDCORE2_HMMU0_MMU_BASE;
10947 		break;
10948 	case HW_CAP_DCORE2_DMMU1:
10949 		*mmu_base = mmDCORE2_HMMU1_MMU_BASE;
10950 		break;
10951 	case HW_CAP_DCORE2_DMMU2:
10952 		*mmu_base = mmDCORE2_HMMU2_MMU_BASE;
10953 		break;
10954 	case HW_CAP_DCORE2_DMMU3:
10955 		*mmu_base = mmDCORE2_HMMU3_MMU_BASE;
10956 		break;
10957 	case HW_CAP_DCORE3_DMMU0:
10958 		*mmu_base = mmDCORE3_HMMU0_MMU_BASE;
10959 		break;
10960 	case HW_CAP_DCORE3_DMMU1:
10961 		*mmu_base = mmDCORE3_HMMU1_MMU_BASE;
10962 		break;
10963 	case HW_CAP_DCORE3_DMMU2:
10964 		*mmu_base = mmDCORE3_HMMU2_MMU_BASE;
10965 		break;
10966 	case HW_CAP_DCORE3_DMMU3:
10967 		*mmu_base = mmDCORE3_HMMU3_MMU_BASE;
10968 		break;
10969 	case HW_CAP_PMMU:
10970 		*mmu_base = mmPMMU_HBW_MMU_BASE;
10971 		break;
10972 	default:
10973 		return -EINVAL;
10974 	}
10975 
10976 	return 0;
10977 }
10978 
10979 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id)
10980 {
10981 	bool is_pmmu = (mmu_id == HW_CAP_PMMU);
10982 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
10983 	u32 mmu_base;
10984 
10985 	if (!(gaudi2->hw_cap_initialized & mmu_id))
10986 		return;
10987 
10988 	if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base))
10989 		return;
10990 
10991 	gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL);
10992 	gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
10993 }
10994 
10995 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask)
10996 {
10997 	u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES;
10998 
10999 	/* check all HMMUs */
11000 	for (i = 0 ; i < num_of_hmmus ; i++) {
11001 		mmu_id = HW_CAP_DCORE0_DMMU0 << i;
11002 
11003 		if (mmu_cap_mask & mmu_id)
11004 			gaudi2_ack_mmu_error(hdev, mmu_id);
11005 	}
11006 
11007 	/* check PMMU */
11008 	if (mmu_cap_mask & HW_CAP_PMMU)
11009 		gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU);
11010 
11011 	return 0;
11012 }
11013 
11014 static void gaudi2_get_msi_info(__le32 *table)
11015 {
11016 	table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX);
11017 }
11018 
11019 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx)
11020 {
11021 	switch (pll_idx) {
11022 	case HL_GAUDI2_CPU_PLL: return CPU_PLL;
11023 	case HL_GAUDI2_PCI_PLL: return PCI_PLL;
11024 	case HL_GAUDI2_NIC_PLL: return NIC_PLL;
11025 	case HL_GAUDI2_DMA_PLL: return DMA_PLL;
11026 	case HL_GAUDI2_MESH_PLL: return MESH_PLL;
11027 	case HL_GAUDI2_MME_PLL: return MME_PLL;
11028 	case HL_GAUDI2_TPC_PLL: return TPC_PLL;
11029 	case HL_GAUDI2_IF_PLL: return IF_PLL;
11030 	case HL_GAUDI2_SRAM_PLL: return SRAM_PLL;
11031 	case HL_GAUDI2_HBM_PLL: return HBM_PLL;
11032 	case HL_GAUDI2_VID_PLL: return VID_PLL;
11033 	case HL_GAUDI2_MSS_PLL: return MSS_PLL;
11034 	default: return -EINVAL;
11035 	}
11036 }
11037 
11038 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map)
11039 {
11040 	/* Not implemented */
11041 	return 0;
11042 }
11043 
11044 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon)
11045 {
11046 	/* Not implemented */
11047 	return 0;
11048 }
11049 
11050 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset,
11051 				struct hl_device *hdev, struct hl_mon_state_dump *mon)
11052 {
11053 	/* Not implemented */
11054 	return 0;
11055 }
11056 
11057 
11058 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset,
11059 				u64 status_base_offset, enum hl_sync_engine_type engine_type,
11060 				u32 engine_id, char **buf, size_t *size, size_t *offset)
11061 {
11062 	/* Not implemented */
11063 	return 0;
11064 }
11065 
11066 
11067 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = {
11068 	.monitor_valid = gaudi2_monitor_valid,
11069 	.print_single_monitor = gaudi2_print_single_monitor,
11070 	.gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map,
11071 	.print_fences_single_engine = gaudi2_print_fences_single_engine,
11072 };
11073 
11074 static void gaudi2_state_dump_init(struct hl_device *hdev)
11075 {
11076 	/* Not implemented */
11077 	hdev->state_dump_specs.props = gaudi2_state_dump_specs_props;
11078 	hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs;
11079 }
11080 
11081 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id)
11082 {
11083 	return 0;
11084 }
11085 
11086 static u32 *gaudi2_get_stream_master_qid_arr(void)
11087 {
11088 	return NULL;
11089 }
11090 
11091 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,
11092 				struct attribute_group *dev_vrm_attr_grp)
11093 {
11094 	hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp);
11095 	hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp);
11096 }
11097 
11098 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
11099 					u32 page_size, u32 *real_page_size, bool is_dram_addr)
11100 {
11101 	struct asic_fixed_properties *prop = &hdev->asic_prop;
11102 
11103 	/* for host pages the page size must be  */
11104 	if (!is_dram_addr) {
11105 		if (page_size % mmu_prop->page_size)
11106 			goto page_size_err;
11107 
11108 		*real_page_size = mmu_prop->page_size;
11109 		return 0;
11110 	}
11111 
11112 	if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size))
11113 		goto page_size_err;
11114 
11115 	/*
11116 	 * MMU page size is different from DRAM page size (more precisely, DMMU page is greater
11117 	 * than DRAM page size).
11118 	 * for this reason work with the DRAM page size and let the MMU scrambling routine handle
11119 	 * this mismatch when calculating the address to place in the MMU page table.
11120 	 * (in that case also make sure that the dram_page_size is not greater than the
11121 	 * mmu page size)
11122 	 */
11123 	*real_page_size = prop->dram_page_size;
11124 
11125 	return 0;
11126 
11127 page_size_err:
11128 	dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n",
11129 							page_size, mmu_prop->page_size >> 10);
11130 	return -EFAULT;
11131 }
11132 
11133 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data)
11134 {
11135 	return -EOPNOTSUPP;
11136 }
11137 
11138 int gaudi2_send_device_activity(struct hl_device *hdev, bool open)
11139 {
11140 	struct gaudi2_device *gaudi2 = hdev->asic_specific;
11141 
11142 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
11143 		return 0;
11144 
11145 	return hl_fw_send_device_activity(hdev, open);
11146 }
11147 
11148 static const struct hl_asic_funcs gaudi2_funcs = {
11149 	.early_init = gaudi2_early_init,
11150 	.early_fini = gaudi2_early_fini,
11151 	.late_init = gaudi2_late_init,
11152 	.late_fini = gaudi2_late_fini,
11153 	.sw_init = gaudi2_sw_init,
11154 	.sw_fini = gaudi2_sw_fini,
11155 	.hw_init = gaudi2_hw_init,
11156 	.hw_fini = gaudi2_hw_fini,
11157 	.halt_engines = gaudi2_halt_engines,
11158 	.suspend = gaudi2_suspend,
11159 	.resume = gaudi2_resume,
11160 	.mmap = gaudi2_mmap,
11161 	.ring_doorbell = gaudi2_ring_doorbell,
11162 	.pqe_write = gaudi2_pqe_write,
11163 	.asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent,
11164 	.asic_dma_free_coherent = gaudi2_dma_free_coherent,
11165 	.scrub_device_mem = gaudi2_scrub_device_mem,
11166 	.scrub_device_dram = gaudi2_scrub_device_dram,
11167 	.get_int_queue_base = NULL,
11168 	.test_queues = gaudi2_test_queues,
11169 	.asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc,
11170 	.asic_dma_pool_free = gaudi2_dma_pool_free,
11171 	.cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc,
11172 	.cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free,
11173 	.asic_dma_unmap_single = gaudi2_dma_unmap_single,
11174 	.asic_dma_map_single = gaudi2_dma_map_single,
11175 	.hl_dma_unmap_sgtable = hl_dma_unmap_sgtable,
11176 	.cs_parser = gaudi2_cs_parser,
11177 	.asic_dma_map_sgtable = hl_dma_map_sgtable,
11178 	.add_end_of_cb_packets = NULL,
11179 	.update_eq_ci = gaudi2_update_eq_ci,
11180 	.context_switch = gaudi2_context_switch,
11181 	.restore_phase_topology = gaudi2_restore_phase_topology,
11182 	.debugfs_read_dma = gaudi2_debugfs_read_dma,
11183 	.add_device_attr = gaudi2_add_device_attr,
11184 	.handle_eqe = gaudi2_handle_eqe,
11185 	.get_events_stat = gaudi2_get_events_stat,
11186 	.read_pte = NULL,
11187 	.write_pte = NULL,
11188 	.mmu_invalidate_cache = gaudi2_mmu_invalidate_cache,
11189 	.mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range,
11190 	.mmu_prefetch_cache_range = NULL,
11191 	.send_heartbeat = gaudi2_send_heartbeat,
11192 	.debug_coresight = gaudi2_debug_coresight,
11193 	.is_device_idle = gaudi2_is_device_idle,
11194 	.compute_reset_late_init = gaudi2_compute_reset_late_init,
11195 	.hw_queues_lock = gaudi2_hw_queues_lock,
11196 	.hw_queues_unlock = gaudi2_hw_queues_unlock,
11197 	.get_pci_id = gaudi2_get_pci_id,
11198 	.get_eeprom_data = gaudi2_get_eeprom_data,
11199 	.get_monitor_dump = gaudi2_get_monitor_dump,
11200 	.send_cpu_message = gaudi2_send_cpu_message,
11201 	.pci_bars_map = gaudi2_pci_bars_map,
11202 	.init_iatu = gaudi2_init_iatu,
11203 	.rreg = hl_rreg,
11204 	.wreg = hl_wreg,
11205 	.halt_coresight = gaudi2_halt_coresight,
11206 	.ctx_init = gaudi2_ctx_init,
11207 	.ctx_fini = gaudi2_ctx_fini,
11208 	.pre_schedule_cs = gaudi2_pre_schedule_cs,
11209 	.get_queue_id_for_cq = gaudi2_get_queue_id_for_cq,
11210 	.load_firmware_to_device = NULL,
11211 	.load_boot_fit_to_device = NULL,
11212 	.get_signal_cb_size = gaudi2_get_signal_cb_size,
11213 	.get_wait_cb_size = gaudi2_get_wait_cb_size,
11214 	.gen_signal_cb = gaudi2_gen_signal_cb,
11215 	.gen_wait_cb = gaudi2_gen_wait_cb,
11216 	.reset_sob = gaudi2_reset_sob,
11217 	.reset_sob_group = gaudi2_reset_sob_group,
11218 	.get_device_time = gaudi2_get_device_time,
11219 	.pb_print_security_errors = gaudi2_pb_print_security_errors,
11220 	.collective_wait_init_cs = gaudi2_collective_wait_init_cs,
11221 	.collective_wait_create_jobs = gaudi2_collective_wait_create_jobs,
11222 	.get_dec_base_addr = gaudi2_get_dec_base_addr,
11223 	.scramble_addr = gaudi2_mmu_scramble_addr,
11224 	.descramble_addr = gaudi2_mmu_descramble_addr,
11225 	.ack_protection_bits_errors = gaudi2_ack_protection_bits_errors,
11226 	.get_hw_block_id = gaudi2_get_hw_block_id,
11227 	.hw_block_mmap = gaudi2_block_mmap,
11228 	.enable_events_from_fw = gaudi2_enable_events_from_fw,
11229 	.ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error,
11230 	.get_msi_info = gaudi2_get_msi_info,
11231 	.map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx,
11232 	.init_firmware_preload_params = gaudi2_init_firmware_preload_params,
11233 	.init_firmware_loader = gaudi2_init_firmware_loader,
11234 	.init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm,
11235 	.state_dump_init = gaudi2_state_dump_init,
11236 	.get_sob_addr = &gaudi2_get_sob_addr,
11237 	.set_pci_memory_regions = gaudi2_set_pci_memory_regions,
11238 	.get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr,
11239 	.check_if_razwi_happened = gaudi2_check_if_razwi_happened,
11240 	.mmu_get_real_page_size = gaudi2_mmu_get_real_page_size,
11241 	.access_dev_mem = hl_access_dev_mem,
11242 	.set_dram_bar_base = gaudi2_set_hbm_bar_base,
11243 	.set_engine_cores = gaudi2_set_engine_cores,
11244 	.set_engines = gaudi2_set_engines,
11245 	.send_device_activity = gaudi2_send_device_activity,
11246 	.set_dram_properties = gaudi2_set_dram_properties,
11247 	.set_binning_masks = gaudi2_set_binning_masks,
11248 };
11249 
11250 void gaudi2_set_asic_funcs(struct hl_device *hdev)
11251 {
11252 	hdev->asic_funcs = &gaudi2_funcs;
11253 }
11254