1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include "ivpu_drv.h"
7 #include "ivpu_fw.h"
8 #include "ivpu_hw.h"
9 #include "ivpu_hw_40xx_reg.h"
10 #include "ivpu_hw_reg_io.h"
11 #include "ivpu_ipc.h"
12 #include "ivpu_mmu.h"
13 #include "ivpu_pm.h"
14 
15 #include <linux/dmi.h>
16 
17 #define TILE_MAX_NUM                 6
18 #define TILE_MAX_MASK                0x3f
19 
20 #define LNL_HW_ID                    0x4040
21 
22 #define SKU_TILE_SHIFT               0u
23 #define SKU_TILE_MASK                0x0000ffffu
24 #define SKU_HW_ID_SHIFT              16u
25 #define SKU_HW_ID_MASK               0xffff0000u
26 
27 #define PLL_CONFIG_DEFAULT           0x0
28 #define PLL_CDYN_DEFAULT             0x80
29 #define PLL_EPP_DEFAULT              0x80
30 #define PLL_REF_CLK_FREQ	     (50 * 1000000)
31 #define PLL_RATIO_TO_FREQ(x)	     ((x) * PLL_REF_CLK_FREQ)
32 
33 #define PLL_PROFILING_FREQ_DEFAULT   38400000
34 #define PLL_PROFILING_FREQ_HIGH      400000000
35 
36 #define TIM_SAFE_ENABLE		     0xf1d0dead
37 #define TIM_WATCHDOG_RESET_VALUE     0xffffffff
38 
39 #define TIMEOUT_US		     (150 * USEC_PER_MSEC)
40 #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
41 #define PLL_TIMEOUT_US		     (1500 * USEC_PER_MSEC)
42 
43 #define WEIGHTS_DEFAULT              0xf711f711u
44 #define WEIGHTS_ATS_DEFAULT          0x0000f711u
45 
46 #define ICB_0_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
47 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
48 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
49 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
50 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
51 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
52 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
53 
54 #define ICB_1_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
55 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
56 			(REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
57 
58 #define ICB_0_1_IRQ_MASK ((((u64)ICB_1_IRQ_MASK) << 32) | ICB_0_IRQ_MASK)
59 
60 #define BUTTRESS_IRQ_MASK ((REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR)) | \
61 			   (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR)) | \
62 			   (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR)) | \
63 			   (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR)) | \
64 			   (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR)) | \
65 			   (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR)))
66 
67 #define BUTTRESS_IRQ_ENABLE_MASK ((u32)~BUTTRESS_IRQ_MASK)
68 #define BUTTRESS_IRQ_DISABLE_MASK ((u32)-1)
69 
70 #define ITF_FIREWALL_VIOLATION_MASK ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
71 				     (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
72 				     (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
73 				     (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
74 				     (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
75 				     (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
76 				     (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
77 
78 static char *ivpu_platform_to_str(u32 platform)
79 {
80 	switch (platform) {
81 	case IVPU_PLATFORM_SILICON:
82 		return "IVPU_PLATFORM_SILICON";
83 	case IVPU_PLATFORM_SIMICS:
84 		return "IVPU_PLATFORM_SIMICS";
85 	case IVPU_PLATFORM_FPGA:
86 		return "IVPU_PLATFORM_FPGA";
87 	default:
88 		return "Invalid platform";
89 	}
90 }
91 
92 static const struct dmi_system_id ivpu_dmi_platform_simulation[] = {
93 	{
94 		.ident = "Intel Simics",
95 		.matches = {
96 			DMI_MATCH(DMI_BOARD_NAME, "lnlrvp"),
97 			DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
98 			DMI_MATCH(DMI_BOARD_SERIAL, "123456789"),
99 		},
100 	},
101 	{
102 		.ident = "Intel Simics",
103 		.matches = {
104 			DMI_MATCH(DMI_BOARD_NAME, "Simics"),
105 		},
106 	},
107 	{ }
108 };
109 
110 static void ivpu_hw_read_platform(struct ivpu_device *vdev)
111 {
112 	if (dmi_check_system(ivpu_dmi_platform_simulation))
113 		vdev->platform = IVPU_PLATFORM_SIMICS;
114 	else
115 		vdev->platform = IVPU_PLATFORM_SILICON;
116 
117 	ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n",
118 		 ivpu_platform_to_str(vdev->platform), vdev->platform);
119 }
120 
121 static void ivpu_hw_wa_init(struct ivpu_device *vdev)
122 {
123 	vdev->wa.punit_disabled = ivpu_is_fpga(vdev);
124 	vdev->wa.clear_runtime_mem = false;
125 
126 	if (ivpu_hw_gen(vdev) == IVPU_HW_40XX)
127 		vdev->wa.disable_clock_relinquish = true;
128 
129 	IVPU_PRINT_WA(punit_disabled);
130 	IVPU_PRINT_WA(clear_runtime_mem);
131 	IVPU_PRINT_WA(disable_clock_relinquish);
132 }
133 
134 static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
135 {
136 	if (ivpu_is_fpga(vdev)) {
137 		vdev->timeout.boot = 100000;
138 		vdev->timeout.jsm = 50000;
139 		vdev->timeout.tdr = 2000000;
140 		vdev->timeout.reschedule_suspend = 1000;
141 	} else if (ivpu_is_simics(vdev)) {
142 		vdev->timeout.boot = 50;
143 		vdev->timeout.jsm = 500;
144 		vdev->timeout.tdr = 10000;
145 		vdev->timeout.reschedule_suspend = 10;
146 	} else {
147 		vdev->timeout.boot = 1000;
148 		vdev->timeout.jsm = 500;
149 		vdev->timeout.tdr = 2000;
150 		vdev->timeout.reschedule_suspend = 10;
151 	}
152 }
153 
154 static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev)
155 {
156 	return REGB_POLL_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US);
157 }
158 
159 static int ivpu_pll_cmd_send(struct ivpu_device *vdev, u16 min_ratio, u16 max_ratio,
160 			     u16 target_ratio, u16 epp, u16 config, u16 cdyn)
161 {
162 	int ret;
163 	u32 val;
164 
165 	ret = ivpu_pll_wait_for_cmd_send(vdev);
166 	if (ret) {
167 		ivpu_err(vdev, "Failed to sync before WP request: %d\n", ret);
168 		return ret;
169 	}
170 
171 	val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0);
172 	val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MIN_RATIO, min_ratio, val);
173 	val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MAX_RATIO, max_ratio, val);
174 	REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, val);
175 
176 	val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1);
177 	val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, TARGET_RATIO, target_ratio, val);
178 	val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, EPP, epp, val);
179 	REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, val);
180 
181 	val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2);
182 	val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CONFIG, config, val);
183 	val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CDYN, cdyn, val);
184 	REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, val);
185 
186 	val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_CMD);
187 	val = REG_SET_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, val);
188 	REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_CMD, val);
189 
190 	ret = ivpu_pll_wait_for_cmd_send(vdev);
191 	if (ret)
192 		ivpu_err(vdev, "Failed to sync after WP request: %d\n", ret);
193 
194 	return ret;
195 }
196 
197 static int ivpu_pll_wait_for_status_ready(struct ivpu_device *vdev)
198 {
199 	return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, 1, PLL_TIMEOUT_US);
200 }
201 
202 static int ivpu_wait_for_clock_own_resource_ack(struct ivpu_device *vdev)
203 {
204 	if (ivpu_is_simics(vdev))
205 		return 0;
206 
207 	return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US);
208 }
209 
210 static void ivpu_pll_init_frequency_ratios(struct ivpu_device *vdev)
211 {
212 	struct ivpu_hw_info *hw = vdev->hw;
213 	u8 fuse_min_ratio, fuse_pn_ratio, fuse_max_ratio;
214 	u32 fmin_fuse, fmax_fuse;
215 
216 	fmin_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMIN_FUSE);
217 	fuse_min_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, MIN_RATIO, fmin_fuse);
218 	fuse_pn_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, PN_RATIO, fmin_fuse);
219 
220 	fmax_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMAX_FUSE);
221 	fuse_max_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMAX_FUSE, MAX_RATIO, fmax_fuse);
222 
223 	hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, fuse_min_ratio, fuse_max_ratio);
224 	hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, fuse_max_ratio);
225 	hw->pll.pn_ratio = clamp_t(u8, fuse_pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
226 }
227 
228 static int ivpu_pll_drive(struct ivpu_device *vdev, bool enable)
229 {
230 	u16 config = enable ? PLL_CONFIG_DEFAULT : 0;
231 	u16 cdyn = enable ? PLL_CDYN_DEFAULT : 0;
232 	u16 epp = enable ? PLL_EPP_DEFAULT : 0;
233 	struct ivpu_hw_info *hw = vdev->hw;
234 	u16 target_ratio = hw->pll.pn_ratio;
235 	int ret;
236 
237 	ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, epp: 0x%x, config: 0x%x, cdyn: 0x%x\n",
238 		 PLL_RATIO_TO_FREQ(target_ratio), epp, config, cdyn);
239 
240 	ret = ivpu_pll_cmd_send(vdev, hw->pll.min_ratio, hw->pll.max_ratio,
241 				target_ratio, epp, config, cdyn);
242 	if (ret) {
243 		ivpu_err(vdev, "Failed to send PLL workpoint request: %d\n", ret);
244 		return ret;
245 	}
246 
247 	if (enable) {
248 		ret = ivpu_pll_wait_for_status_ready(vdev);
249 		if (ret) {
250 			ivpu_err(vdev, "Timed out waiting for PLL ready status\n");
251 			return ret;
252 		}
253 	}
254 
255 	return 0;
256 }
257 
258 static int ivpu_pll_enable(struct ivpu_device *vdev)
259 {
260 	return ivpu_pll_drive(vdev, true);
261 }
262 
263 static int ivpu_pll_disable(struct ivpu_device *vdev)
264 {
265 	return ivpu_pll_drive(vdev, false);
266 }
267 
268 static void ivpu_boot_host_ss_rst_drive(struct ivpu_device *vdev, bool enable)
269 {
270 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN);
271 
272 	if (enable) {
273 		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
274 		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
275 		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
276 	} else {
277 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
278 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
279 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
280 	}
281 
282 	REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val);
283 }
284 
285 static void ivpu_boot_host_ss_clk_drive(struct ivpu_device *vdev, bool enable)
286 {
287 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN);
288 
289 	if (enable) {
290 		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
291 		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
292 		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
293 	} else {
294 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
295 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
296 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
297 	}
298 
299 	REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val);
300 }
301 
302 static int ivpu_boot_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
303 {
304 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
305 
306 	if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
307 		return -EIO;
308 
309 	return 0;
310 }
311 
312 static int ivpu_boot_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
313 {
314 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN);
315 
316 	if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
317 		return -EIO;
318 
319 	return 0;
320 }
321 
322 static int ivpu_boot_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
323 {
324 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY);
325 
326 	if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
327 		return -EIO;
328 
329 	return 0;
330 }
331 
332 static int ivpu_boot_top_noc_qrenqn_check(struct ivpu_device *vdev, u32 exp_val)
333 {
334 	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
335 
336 	if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
337 	    !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
338 		return -EIO;
339 
340 	return 0;
341 }
342 
343 static int ivpu_boot_top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
344 {
345 	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN);
346 
347 	if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
348 	    !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
349 		return -EIO;
350 
351 	return 0;
352 }
353 
354 static int ivpu_boot_top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
355 {
356 	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY);
357 
358 	if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
359 	    !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
360 		return -EIO;
361 
362 	return 0;
363 }
364 
365 static void ivpu_boot_idle_gen_drive(struct ivpu_device *vdev, bool enable)
366 {
367 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN);
368 
369 	if (enable)
370 		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
371 	else
372 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
373 
374 	REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val);
375 }
376 
377 static int ivpu_boot_host_ss_check(struct ivpu_device *vdev)
378 {
379 	int ret;
380 
381 	ret = ivpu_boot_noc_qreqn_check(vdev, 0x0);
382 	if (ret) {
383 		ivpu_err(vdev, "Failed qreqn check: %d\n", ret);
384 		return ret;
385 	}
386 
387 	ret = ivpu_boot_noc_qacceptn_check(vdev, 0x0);
388 	if (ret) {
389 		ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
390 		return ret;
391 	}
392 
393 	ret = ivpu_boot_noc_qdeny_check(vdev, 0x0);
394 	if (ret)
395 		ivpu_err(vdev, "Failed qdeny check %d\n", ret);
396 
397 	return ret;
398 }
399 
400 static int ivpu_boot_host_ss_axi_drive(struct ivpu_device *vdev, bool enable)
401 {
402 	int ret;
403 	u32 val;
404 
405 	val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
406 	if (enable)
407 		val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
408 	else
409 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
410 	REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val);
411 
412 	ret = ivpu_boot_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
413 	if (ret) {
414 		ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
415 		return ret;
416 	}
417 
418 	ret = ivpu_boot_noc_qdeny_check(vdev, 0x0);
419 	if (ret) {
420 		ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
421 		return ret;
422 	}
423 
424 	if (enable) {
425 		REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT);
426 		REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT);
427 	}
428 
429 	return ret;
430 }
431 
432 static int ivpu_boot_host_ss_axi_enable(struct ivpu_device *vdev)
433 {
434 	return ivpu_boot_host_ss_axi_drive(vdev, true);
435 }
436 
437 static int ivpu_boot_host_ss_top_noc_drive(struct ivpu_device *vdev, bool enable)
438 {
439 	int ret;
440 	u32 val;
441 
442 	val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
443 	if (enable) {
444 		val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
445 		val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
446 	} else {
447 		val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
448 		val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
449 	}
450 	REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val);
451 
452 	ret = ivpu_boot_top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
453 	if (ret) {
454 		ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
455 		return ret;
456 	}
457 
458 	ret = ivpu_boot_top_noc_qdeny_check(vdev, 0x0);
459 	if (ret)
460 		ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
461 
462 	return ret;
463 }
464 
465 static int ivpu_boot_host_ss_top_noc_enable(struct ivpu_device *vdev)
466 {
467 	return ivpu_boot_host_ss_top_noc_drive(vdev, true);
468 }
469 
470 static void ivpu_boot_pwr_island_trickle_drive(struct ivpu_device *vdev, bool enable)
471 {
472 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
473 
474 	if (enable)
475 		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
476 	else
477 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
478 
479 	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
480 
481 	if (enable)
482 		ndelay(500);
483 }
484 
485 static void ivpu_boot_pwr_island_drive(struct ivpu_device *vdev, bool enable)
486 {
487 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0);
488 
489 	if (enable)
490 		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
491 	else
492 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
493 
494 	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
495 
496 	if (!enable)
497 		ndelay(500);
498 }
499 
500 static int ivpu_boot_wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val)
501 {
502 	if (ivpu_is_fpga(vdev))
503 		return 0;
504 
505 	return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU,
506 			     exp_val, PWR_ISLAND_STATUS_TIMEOUT_US);
507 }
508 
509 static void ivpu_boot_pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable)
510 {
511 	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0);
512 
513 	if (enable)
514 		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
515 	else
516 		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
517 
518 	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val);
519 }
520 
521 static void ivpu_boot_no_snoop_enable(struct ivpu_device *vdev)
522 {
523 	u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES);
524 
525 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val);
526 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val);
527 	val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val);
528 
529 	REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val);
530 }
531 
532 static void ivpu_boot_tbu_mmu_enable(struct ivpu_device *vdev)
533 {
534 	u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV);
535 
536 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
537 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
538 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val);
539 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val);
540 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
541 	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
542 
543 	REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val);
544 }
545 
546 static int ivpu_boot_cpu_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
547 {
548 	u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN);
549 
550 	if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val))
551 		return -EIO;
552 
553 	return 0;
554 }
555 
556 static int ivpu_boot_cpu_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
557 {
558 	u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY);
559 
560 	if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val))
561 		return -EIO;
562 
563 	return 0;
564 }
565 
566 static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev)
567 {
568 	int ret;
569 
570 	ret = ivpu_wait_for_clock_own_resource_ack(vdev);
571 	if (ret) {
572 		ivpu_err(vdev, "Timed out waiting for clock own resource ACK\n");
573 		return ret;
574 	}
575 
576 	ivpu_boot_pwr_island_trickle_drive(vdev, true);
577 	ivpu_boot_pwr_island_drive(vdev, true);
578 
579 	ret = ivpu_boot_wait_for_pwr_island_status(vdev, 0x1);
580 	if (ret) {
581 		ivpu_err(vdev, "Timed out waiting for power island status\n");
582 		return ret;
583 	}
584 
585 	ret = ivpu_boot_top_noc_qrenqn_check(vdev, 0x0);
586 	if (ret) {
587 		ivpu_err(vdev, "Failed qrenqn check %d\n", ret);
588 		return ret;
589 	}
590 
591 	ivpu_boot_host_ss_clk_drive(vdev, true);
592 	ivpu_boot_host_ss_rst_drive(vdev, true);
593 	ivpu_boot_pwr_island_isolation_drive(vdev, false);
594 
595 	return ret;
596 }
597 
598 static int ivpu_boot_soc_cpu_drive(struct ivpu_device *vdev, bool enable)
599 {
600 	int ret;
601 	u32 val;
602 
603 	val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN);
604 	if (enable)
605 		val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
606 	else
607 		val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
608 	REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val);
609 
610 	ret = ivpu_boot_cpu_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
611 	if (ret) {
612 		ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
613 		return ret;
614 	}
615 
616 	ret = ivpu_boot_cpu_noc_qdeny_check(vdev, 0x0);
617 	if (ret)
618 		ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
619 
620 	return ret;
621 }
622 
623 static int ivpu_boot_soc_cpu_enable(struct ivpu_device *vdev)
624 {
625 	return ivpu_boot_soc_cpu_drive(vdev, true);
626 }
627 
628 static int ivpu_boot_soc_cpu_boot(struct ivpu_device *vdev)
629 {
630 	int ret;
631 	u32 val;
632 	u64 val64;
633 
634 	ret = ivpu_boot_soc_cpu_enable(vdev);
635 	if (ret) {
636 		ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret);
637 		return ret;
638 	}
639 
640 	val64 = vdev->fw->entry_point;
641 	val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1;
642 	REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64);
643 
644 	val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO);
645 	val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val);
646 	REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val);
647 
648 	ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
649 		 ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume");
650 
651 	return 0;
652 }
653 
654 static int ivpu_boot_d0i3_drive(struct ivpu_device *vdev, bool enable)
655 {
656 	int ret;
657 	u32 val;
658 
659 	ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
660 	if (ret) {
661 		ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret);
662 		return ret;
663 	}
664 
665 	val = REGB_RD32(VPU_40XX_BUTTRESS_D0I3_CONTROL);
666 	if (enable)
667 		val = REG_SET_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val);
668 	else
669 		val = REG_CLR_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val);
670 	REGB_WR32(VPU_40XX_BUTTRESS_D0I3_CONTROL, val);
671 
672 	ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
673 	if (ret) {
674 		ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret);
675 		return ret;
676 	}
677 
678 	return 0;
679 }
680 
681 static bool ivpu_tile_disable_check(u32 config)
682 {
683 	/* Allowed values: 0 or one bit from range 0-5 (6 tiles) */
684 	if (config == 0)
685 		return true;
686 
687 	if (config > BIT(TILE_MAX_NUM - 1))
688 		return false;
689 
690 	if ((config & (config - 1)) == 0)
691 		return true;
692 
693 	return false;
694 }
695 
696 static int ivpu_hw_40xx_info_init(struct ivpu_device *vdev)
697 {
698 	struct ivpu_hw_info *hw = vdev->hw;
699 	u32 tile_disable;
700 	u32 fuse;
701 
702 	fuse = REGB_RD32(VPU_40XX_BUTTRESS_TILE_FUSE);
703 	if (!REG_TEST_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, VALID, fuse)) {
704 		ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse);
705 		return -EIO;
706 	}
707 
708 	tile_disable = REG_GET_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, CONFIG, fuse);
709 	if (!ivpu_tile_disable_check(tile_disable)) {
710 		ivpu_err(vdev, "Fuse: Invalid tile disable config (0x%x)\n", tile_disable);
711 		return -EIO;
712 	}
713 
714 	if (tile_disable)
715 		ivpu_dbg(vdev, MISC, "Fuse: %d tiles enabled. Tile number %d disabled\n",
716 			 TILE_MAX_NUM - 1, ffs(tile_disable) - 1);
717 	else
718 		ivpu_dbg(vdev, MISC, "Fuse: All %d tiles enabled\n", TILE_MAX_NUM);
719 
720 	hw->tile_fuse = tile_disable;
721 	hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
722 
723 	ivpu_pll_init_frequency_ratios(vdev);
724 
725 	ivpu_hw_init_range(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
726 	ivpu_hw_init_range(&vdev->hw->ranges.user,   0x80000000, SZ_256M);
727 	ivpu_hw_init_range(&vdev->hw->ranges.shave,  0x80000000 + SZ_256M, SZ_2G - SZ_256M);
728 	ivpu_hw_init_range(&vdev->hw->ranges.dma,   0x200000000, SZ_8G);
729 
730 	ivpu_hw_read_platform(vdev);
731 	ivpu_hw_wa_init(vdev);
732 	ivpu_hw_timeouts_init(vdev);
733 
734 	return 0;
735 }
736 
737 static int ivpu_hw_40xx_reset(struct ivpu_device *vdev)
738 {
739 	int ret;
740 	u32 val;
741 
742 	ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US);
743 	if (ret) {
744 		ivpu_err(vdev, "Wait for *_TRIGGER timed out\n");
745 		return ret;
746 	}
747 
748 	val = REGB_RD32(VPU_40XX_BUTTRESS_IP_RESET);
749 	val = REG_SET_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, val);
750 	REGB_WR32(VPU_40XX_BUTTRESS_IP_RESET, val);
751 
752 	ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US);
753 	if (ret)
754 		ivpu_err(vdev, "Timed out waiting for RESET completion\n");
755 
756 	return ret;
757 }
758 
759 static int ivpu_hw_40xx_d0i3_enable(struct ivpu_device *vdev)
760 {
761 	int ret;
762 
763 	if (IVPU_WA(punit_disabled))
764 		return 0;
765 
766 	ret = ivpu_boot_d0i3_drive(vdev, true);
767 	if (ret)
768 		ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret);
769 
770 	udelay(5); /* VPU requires 5 us to complete the transition */
771 
772 	return ret;
773 }
774 
775 static int ivpu_hw_40xx_d0i3_disable(struct ivpu_device *vdev)
776 {
777 	int ret;
778 
779 	if (IVPU_WA(punit_disabled))
780 		return 0;
781 
782 	ret = ivpu_boot_d0i3_drive(vdev, false);
783 	if (ret)
784 		ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret);
785 
786 	return ret;
787 }
788 
789 static void ivpu_hw_40xx_profiling_freq_reg_set(struct ivpu_device *vdev)
790 {
791 	u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS);
792 
793 	if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT)
794 		val = REG_CLR_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val);
795 	else
796 		val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val);
797 
798 	REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val);
799 }
800 
801 static void ivpu_hw_40xx_ats_print(struct ivpu_device *vdev)
802 {
803 	ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n",
804 		 REGB_RD32(VPU_40XX_BUTTRESS_HM_ATS) ? "Enable" : "Disable");
805 }
806 
807 static void ivpu_hw_40xx_clock_relinquish_disable(struct ivpu_device *vdev)
808 {
809 	u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS);
810 
811 	val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, DISABLE_CLK_RELINQUISH, val);
812 	REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val);
813 }
814 
815 static int ivpu_hw_40xx_power_up(struct ivpu_device *vdev)
816 {
817 	int ret;
818 
819 	ret = ivpu_hw_40xx_reset(vdev);
820 	if (ret) {
821 		ivpu_err(vdev, "Failed to reset HW: %d\n", ret);
822 		return ret;
823 	}
824 
825 	ret = ivpu_hw_40xx_d0i3_disable(vdev);
826 	if (ret)
827 		ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
828 
829 	ret = ivpu_pll_enable(vdev);
830 	if (ret) {
831 		ivpu_err(vdev, "Failed to enable PLL: %d\n", ret);
832 		return ret;
833 	}
834 
835 	if (IVPU_WA(disable_clock_relinquish))
836 		ivpu_hw_40xx_clock_relinquish_disable(vdev);
837 	ivpu_hw_40xx_profiling_freq_reg_set(vdev);
838 	ivpu_hw_40xx_ats_print(vdev);
839 
840 	ret = ivpu_boot_host_ss_check(vdev);
841 	if (ret) {
842 		ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
843 		return ret;
844 	}
845 
846 	ivpu_boot_idle_gen_drive(vdev, false);
847 
848 	ret = ivpu_boot_pwr_domain_enable(vdev);
849 	if (ret) {
850 		ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
851 		return ret;
852 	}
853 
854 	ret = ivpu_boot_host_ss_axi_enable(vdev);
855 	if (ret) {
856 		ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
857 		return ret;
858 	}
859 
860 	ret = ivpu_boot_host_ss_top_noc_enable(vdev);
861 	if (ret)
862 		ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
863 
864 	return ret;
865 }
866 
867 static int ivpu_hw_40xx_boot_fw(struct ivpu_device *vdev)
868 {
869 	int ret;
870 
871 	ivpu_boot_no_snoop_enable(vdev);
872 	ivpu_boot_tbu_mmu_enable(vdev);
873 
874 	ret = ivpu_boot_soc_cpu_boot(vdev);
875 	if (ret)
876 		ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
877 
878 	return ret;
879 }
880 
881 static bool ivpu_hw_40xx_is_idle(struct ivpu_device *vdev)
882 {
883 	u32 val;
884 
885 	if (IVPU_WA(punit_disabled))
886 		return true;
887 
888 	val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS);
889 	return REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, val) &&
890 	       REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, val);
891 }
892 
893 static int ivpu_hw_40xx_power_down(struct ivpu_device *vdev)
894 {
895 	int ret = 0;
896 
897 	if (!ivpu_hw_40xx_is_idle(vdev) && ivpu_hw_40xx_reset(vdev))
898 		ivpu_warn(vdev, "Failed to reset the VPU\n");
899 
900 	if (ivpu_pll_disable(vdev)) {
901 		ivpu_err(vdev, "Failed to disable PLL\n");
902 		ret = -EIO;
903 	}
904 
905 	if (ivpu_hw_40xx_d0i3_enable(vdev)) {
906 		ivpu_err(vdev, "Failed to enter D0I3\n");
907 		ret = -EIO;
908 	}
909 
910 	return ret;
911 }
912 
913 static void ivpu_hw_40xx_wdt_disable(struct ivpu_device *vdev)
914 {
915 	u32 val;
916 
917 	REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
918 	REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
919 
920 	REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
921 	REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0);
922 
923 	val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG);
924 	val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
925 	REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val);
926 }
927 
928 /* Register indirect accesses */
929 static u32 ivpu_hw_40xx_reg_pll_freq_get(struct ivpu_device *vdev)
930 {
931 	u32 pll_curr_ratio;
932 
933 	pll_curr_ratio = REGB_RD32(VPU_40XX_BUTTRESS_PLL_FREQ);
934 	pll_curr_ratio &= VPU_40XX_BUTTRESS_PLL_FREQ_RATIO_MASK;
935 
936 	return PLL_RATIO_TO_FREQ(pll_curr_ratio);
937 }
938 
939 static u32 ivpu_hw_40xx_reg_telemetry_offset_get(struct ivpu_device *vdev)
940 {
941 	return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_OFFSET);
942 }
943 
944 static u32 ivpu_hw_40xx_reg_telemetry_size_get(struct ivpu_device *vdev)
945 {
946 	return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_SIZE);
947 }
948 
949 static u32 ivpu_hw_40xx_reg_telemetry_enable_get(struct ivpu_device *vdev)
950 {
951 	return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_ENABLE);
952 }
953 
954 static void ivpu_hw_40xx_reg_db_set(struct ivpu_device *vdev, u32 db_id)
955 {
956 	u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0;
957 	u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET);
958 
959 	REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
960 }
961 
962 static u32 ivpu_hw_40xx_reg_ipc_rx_addr_get(struct ivpu_device *vdev)
963 {
964 	return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM);
965 }
966 
967 static u32 ivpu_hw_40xx_reg_ipc_rx_count_get(struct ivpu_device *vdev)
968 {
969 	u32 count = REGV_RD32_SILENT(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT);
970 
971 	return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
972 }
973 
974 static void ivpu_hw_40xx_reg_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
975 {
976 	REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr);
977 }
978 
979 static void ivpu_hw_40xx_irq_clear(struct ivpu_device *vdev)
980 {
981 	REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK);
982 }
983 
984 static void ivpu_hw_40xx_irq_enable(struct ivpu_device *vdev)
985 {
986 	REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK);
987 	REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK);
988 	REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_ENABLE_MASK);
989 	REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
990 }
991 
992 static void ivpu_hw_40xx_irq_disable(struct ivpu_device *vdev)
993 {
994 	REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
995 	REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_DISABLE_MASK);
996 	REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
997 	REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul);
998 }
999 
1000 static void ivpu_hw_40xx_irq_wdt_nce_handler(struct ivpu_device *vdev)
1001 {
1002 	/* TODO: For LNN hang consider engine reset instead of full recovery */
1003 	ivpu_pm_schedule_recovery(vdev);
1004 }
1005 
1006 static void ivpu_hw_40xx_irq_wdt_mss_handler(struct ivpu_device *vdev)
1007 {
1008 	ivpu_hw_wdt_disable(vdev);
1009 	ivpu_pm_schedule_recovery(vdev);
1010 }
1011 
1012 static void ivpu_hw_40xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
1013 {
1014 	ivpu_pm_schedule_recovery(vdev);
1015 }
1016 
1017 /* Handler for IRQs from VPU core (irqV) */
1018 static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
1019 {
1020 	u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
1021 	irqreturn_t ret = IRQ_NONE;
1022 
1023 	if (!status)
1024 		return IRQ_NONE;
1025 
1026 	REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status);
1027 
1028 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
1029 		ivpu_mmu_irq_evtq_handler(vdev);
1030 
1031 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
1032 		ret |= ivpu_ipc_irq_handler(vdev);
1033 
1034 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
1035 		ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
1036 
1037 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
1038 		ivpu_mmu_irq_gerr_handler(vdev);
1039 
1040 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
1041 		ivpu_hw_40xx_irq_wdt_mss_handler(vdev);
1042 
1043 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
1044 		ivpu_hw_40xx_irq_wdt_nce_handler(vdev);
1045 
1046 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
1047 		ivpu_hw_40xx_irq_noc_firewall_handler(vdev);
1048 
1049 	return ret;
1050 }
1051 
1052 /* Handler for IRQs from Buttress core (irqB) */
1053 static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
1054 {
1055 	bool schedule_recovery = false;
1056 	u32 status = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
1057 
1058 	if (status == 0)
1059 		return IRQ_NONE;
1060 
1061 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
1062 		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE");
1063 
1064 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, status)) {
1065 		ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
1066 			 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1),
1067 			 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2));
1068 		REGB_WR32(VPU_40XX_BUTTRESS_ATS_ERR_CLEAR, 0x1);
1069 		schedule_recovery = true;
1070 	}
1071 
1072 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, status)) {
1073 		ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG));
1074 		REGB_WR32(VPU_40XX_BUTTRESS_CFI0_ERR_CLEAR, 0x1);
1075 		schedule_recovery = true;
1076 	}
1077 
1078 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, status)) {
1079 		ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG));
1080 		REGB_WR32(VPU_40XX_BUTTRESS_CFI1_ERR_CLEAR, 0x1);
1081 		schedule_recovery = true;
1082 	}
1083 
1084 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, status)) {
1085 		ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x",
1086 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW),
1087 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH));
1088 		REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_CLEAR, 0x1);
1089 		schedule_recovery = true;
1090 	}
1091 
1092 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, status)) {
1093 		ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x",
1094 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW),
1095 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH));
1096 		REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_CLEAR, 0x1);
1097 		schedule_recovery = true;
1098 	}
1099 
1100 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, status)) {
1101 		ivpu_err(vdev, "Survivability error detected\n");
1102 		schedule_recovery = true;
1103 	}
1104 
1105 	/* This must be done after interrupts are cleared at the source. */
1106 	REGB_WR32(VPU_40XX_BUTTRESS_INTERRUPT_STAT, status);
1107 
1108 	if (schedule_recovery)
1109 		ivpu_pm_schedule_recovery(vdev);
1110 
1111 	return IRQ_HANDLED;
1112 }
1113 
1114 static irqreturn_t ivpu_hw_40xx_irq_handler(int irq, void *ptr)
1115 {
1116 	struct ivpu_device *vdev = ptr;
1117 	irqreturn_t ret = IRQ_NONE;
1118 
1119 	REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
1120 
1121 	ret |= ivpu_hw_40xx_irqv_handler(vdev, irq);
1122 	ret |= ivpu_hw_40xx_irqb_handler(vdev, irq);
1123 
1124 	/* Re-enable global interrupts to re-trigger MSI for pending interrupts */
1125 	REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
1126 
1127 	if (ret & IRQ_WAKE_THREAD)
1128 		return IRQ_WAKE_THREAD;
1129 
1130 	return ret;
1131 }
1132 
1133 static void ivpu_hw_40xx_diagnose_failure(struct ivpu_device *vdev)
1134 {
1135 	u32 irqv = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
1136 	u32 irqb = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
1137 
1138 	if (ivpu_hw_40xx_reg_ipc_rx_count_get(vdev))
1139 		ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1140 
1141 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, irqv))
1142 		ivpu_err(vdev, "WDT MSS timeout detected\n");
1143 
1144 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, irqv))
1145 		ivpu_err(vdev, "WDT NCE timeout detected\n");
1146 
1147 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, irqv))
1148 		ivpu_err(vdev, "NOC Firewall irq detected\n");
1149 
1150 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, irqb)) {
1151 		ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
1152 			 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1),
1153 			 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2));
1154 	}
1155 
1156 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, irqb))
1157 		ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG));
1158 
1159 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, irqb))
1160 		ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG));
1161 
1162 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, irqb))
1163 		ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n",
1164 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW),
1165 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH));
1166 
1167 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, irqb))
1168 		ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n",
1169 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW),
1170 			 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH));
1171 
1172 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, irqb))
1173 		ivpu_err(vdev, "Survivability error detected\n");
1174 }
1175 
1176 const struct ivpu_hw_ops ivpu_hw_40xx_ops = {
1177 	.info_init = ivpu_hw_40xx_info_init,
1178 	.power_up = ivpu_hw_40xx_power_up,
1179 	.is_idle = ivpu_hw_40xx_is_idle,
1180 	.power_down = ivpu_hw_40xx_power_down,
1181 	.reset = ivpu_hw_40xx_reset,
1182 	.boot_fw = ivpu_hw_40xx_boot_fw,
1183 	.wdt_disable = ivpu_hw_40xx_wdt_disable,
1184 	.diagnose_failure = ivpu_hw_40xx_diagnose_failure,
1185 	.reg_pll_freq_get = ivpu_hw_40xx_reg_pll_freq_get,
1186 	.reg_telemetry_offset_get = ivpu_hw_40xx_reg_telemetry_offset_get,
1187 	.reg_telemetry_size_get = ivpu_hw_40xx_reg_telemetry_size_get,
1188 	.reg_telemetry_enable_get = ivpu_hw_40xx_reg_telemetry_enable_get,
1189 	.reg_db_set = ivpu_hw_40xx_reg_db_set,
1190 	.reg_ipc_rx_addr_get = ivpu_hw_40xx_reg_ipc_rx_addr_get,
1191 	.reg_ipc_rx_count_get = ivpu_hw_40xx_reg_ipc_rx_count_get,
1192 	.reg_ipc_tx_set = ivpu_hw_40xx_reg_ipc_tx_set,
1193 	.irq_clear = ivpu_hw_40xx_irq_clear,
1194 	.irq_enable = ivpu_hw_40xx_irq_enable,
1195 	.irq_disable = ivpu_hw_40xx_irq_disable,
1196 	.irq_handler = ivpu_hw_40xx_irq_handler,
1197 };
1198