xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c (revision ba8ff971)
1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
28 #include "amdgpu_cs.h"
29 #include "soc15.h"
30 #include "soc15d.h"
31 #include "vcn_v2_0.h"
32 #include "mmsch_v3_0.h"
33 #include "vcn_sw_ring.h"
34 
35 #include "vcn/vcn_3_0_0_offset.h"
36 #include "vcn/vcn_3_0_0_sh_mask.h"
37 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
38 
39 #include <drm/drm_drv.h>
40 
41 #define VCN_VID_SOC_ADDRESS_2_0					0x1fa00
42 #define VCN1_VID_SOC_ADDRESS_3_0				0x48200
43 
44 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET			0x27
45 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET			0x0f
46 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET			0x10
47 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET			0x11
48 #define mmUVD_NO_OP_INTERNAL_OFFSET				0x29
49 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET			0x66
50 #define mmUVD_SCRATCH9_INTERNAL_OFFSET				0xc01d
51 
52 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET			0x431
53 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET		0x3b4
54 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET		0x3b5
55 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET			0x25c
56 
57 #define VCN_INSTANCES_SIENNA_CICHLID				2
58 #define DEC_SW_RING_ENABLED					FALSE
59 
60 #define RDECODE_MSG_CREATE					0x00000000
61 #define RDECODE_MESSAGE_CREATE					0x00000001
62 
63 static int amdgpu_ih_clientid_vcns[] = {
64 	SOC15_IH_CLIENTID_VCN,
65 	SOC15_IH_CLIENTID_VCN1
66 };
67 
68 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
69 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
70 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
71 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
72 static int vcn_v3_0_set_powergating_state(void *handle,
73 			enum amd_powergating_state state);
74 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
75 			int inst_idx, struct dpg_pause_state *new_state);
76 
77 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
78 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
79 
80 /**
81  * vcn_v3_0_early_init - set function pointers and load microcode
82  *
83  * @handle: amdgpu_device pointer
84  *
85  * Set ring and irq function pointers
86  * Load microcode from filesystem
87  */
88 static int vcn_v3_0_early_init(void *handle)
89 {
90 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
91 
92 	if (amdgpu_sriov_vf(adev)) {
93 		adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
94 		adev->vcn.harvest_config = 0;
95 		adev->vcn.num_enc_rings = 1;
96 
97 	} else {
98 		if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
99 						 AMDGPU_VCN_HARVEST_VCN1))
100 			/* both instances are harvested, disable the block */
101 			return -ENOENT;
102 
103 		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 0, 33))
104 			adev->vcn.num_enc_rings = 0;
105 		else
106 			adev->vcn.num_enc_rings = 2;
107 	}
108 
109 	vcn_v3_0_set_dec_ring_funcs(adev);
110 	vcn_v3_0_set_enc_ring_funcs(adev);
111 	vcn_v3_0_set_irq_funcs(adev);
112 
113 	return amdgpu_vcn_early_init(adev);
114 }
115 
116 /**
117  * vcn_v3_0_sw_init - sw init for VCN block
118  *
119  * @handle: amdgpu_device pointer
120  *
121  * Load firmware and sw initialization
122  */
123 static int vcn_v3_0_sw_init(void *handle)
124 {
125 	struct amdgpu_ring *ring;
126 	int i, j, r;
127 	int vcn_doorbell_index = 0;
128 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
129 
130 	r = amdgpu_vcn_sw_init(adev);
131 	if (r)
132 		return r;
133 
134 	amdgpu_vcn_setup_ucode(adev);
135 
136 	r = amdgpu_vcn_resume(adev);
137 	if (r)
138 		return r;
139 
140 	/*
141 	 * Note: doorbell assignment is fixed for SRIOV multiple VCN engines
142 	 * Formula:
143 	 *   vcn_db_base  = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
144 	 *   dec_ring_i   = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
145 	 *   enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
146 	 */
147 	if (amdgpu_sriov_vf(adev)) {
148 		vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
149 		/* get DWORD offset */
150 		vcn_doorbell_index = vcn_doorbell_index << 1;
151 	}
152 
153 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
154 		volatile struct amdgpu_fw_shared *fw_shared;
155 
156 		if (adev->vcn.harvest_config & (1 << i))
157 			continue;
158 
159 		adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
160 		adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
161 		adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
162 		adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
163 		adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
164 		adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
165 
166 		adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
167 		adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
168 		adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
169 		adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
170 		adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
171 		adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
172 		adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
173 		adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
174 		adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
175 		adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
176 
177 		/* VCN DEC TRAP */
178 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
179 				VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
180 		if (r)
181 			return r;
182 
183 		atomic_set(&adev->vcn.inst[i].sched_score, 0);
184 
185 		ring = &adev->vcn.inst[i].ring_dec;
186 		ring->use_doorbell = true;
187 		if (amdgpu_sriov_vf(adev)) {
188 			ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
189 		} else {
190 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
191 		}
192 		sprintf(ring->name, "vcn_dec_%d", i);
193 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
194 				     AMDGPU_RING_PRIO_DEFAULT,
195 				     &adev->vcn.inst[i].sched_score);
196 		if (r)
197 			return r;
198 
199 		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
200 			enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(j);
201 
202 			/* VCN ENC TRAP */
203 			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
204 				j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
205 			if (r)
206 				return r;
207 
208 			ring = &adev->vcn.inst[i].ring_enc[j];
209 			ring->use_doorbell = true;
210 			if (amdgpu_sriov_vf(adev)) {
211 				ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
212 			} else {
213 				ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
214 			}
215 			sprintf(ring->name, "vcn_enc_%d.%d", i, j);
216 			r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
217 					     hw_prio, &adev->vcn.inst[i].sched_score);
218 			if (r)
219 				return r;
220 		}
221 
222 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
223 		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
224 					     cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |
225 					     cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);
226 		fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
227 		fw_shared->present_flag_0 |= AMDGPU_VCN_SMU_VERSION_INFO_FLAG;
228 		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 1, 2))
229 			fw_shared->smu_interface_info.smu_interface_type = 2;
230 		else if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 1, 1))
231 			fw_shared->smu_interface_info.smu_interface_type = 1;
232 
233 		if (amdgpu_vcnfw_log)
234 			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
235 	}
236 
237 	if (amdgpu_sriov_vf(adev)) {
238 		r = amdgpu_virt_alloc_mm_table(adev);
239 		if (r)
240 			return r;
241 	}
242 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
243 		adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
244 
245 	return 0;
246 }
247 
248 /**
249  * vcn_v3_0_sw_fini - sw fini for VCN block
250  *
251  * @handle: amdgpu_device pointer
252  *
253  * VCN suspend and free up sw allocation
254  */
255 static int vcn_v3_0_sw_fini(void *handle)
256 {
257 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
258 	int i, r, idx;
259 
260 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
261 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
262 			volatile struct amdgpu_fw_shared *fw_shared;
263 
264 			if (adev->vcn.harvest_config & (1 << i))
265 				continue;
266 			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
267 			fw_shared->present_flag_0 = 0;
268 			fw_shared->sw_ring.is_enabled = false;
269 		}
270 
271 		drm_dev_exit(idx);
272 	}
273 
274 	if (amdgpu_sriov_vf(adev))
275 		amdgpu_virt_free_mm_table(adev);
276 
277 	r = amdgpu_vcn_suspend(adev);
278 	if (r)
279 		return r;
280 
281 	r = amdgpu_vcn_sw_fini(adev);
282 
283 	return r;
284 }
285 
286 /**
287  * vcn_v3_0_hw_init - start and test VCN block
288  *
289  * @handle: amdgpu_device pointer
290  *
291  * Initialize the hardware, boot up the VCPU and do some testing
292  */
293 static int vcn_v3_0_hw_init(void *handle)
294 {
295 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
296 	struct amdgpu_ring *ring;
297 	int i, j, r;
298 
299 	if (amdgpu_sriov_vf(adev)) {
300 		r = vcn_v3_0_start_sriov(adev);
301 		if (r)
302 			goto done;
303 
304 		/* initialize VCN dec and enc ring buffers */
305 		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
306 			if (adev->vcn.harvest_config & (1 << i))
307 				continue;
308 
309 			ring = &adev->vcn.inst[i].ring_dec;
310 			if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) {
311 				ring->sched.ready = false;
312 				ring->no_scheduler = true;
313 				dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
314 			} else {
315 				ring->wptr = 0;
316 				ring->wptr_old = 0;
317 				vcn_v3_0_dec_ring_set_wptr(ring);
318 				ring->sched.ready = true;
319 			}
320 
321 			for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
322 				ring = &adev->vcn.inst[i].ring_enc[j];
323 				if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
324 					ring->sched.ready = false;
325 					ring->no_scheduler = true;
326 					dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
327 				} else {
328 					ring->wptr = 0;
329 					ring->wptr_old = 0;
330 					vcn_v3_0_enc_ring_set_wptr(ring);
331 					ring->sched.ready = true;
332 				}
333 			}
334 		}
335 	} else {
336 		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
337 			if (adev->vcn.harvest_config & (1 << i))
338 				continue;
339 
340 			ring = &adev->vcn.inst[i].ring_dec;
341 
342 			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
343 						     ring->doorbell_index, i);
344 
345 			r = amdgpu_ring_test_helper(ring);
346 			if (r)
347 				goto done;
348 
349 			for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
350 				ring = &adev->vcn.inst[i].ring_enc[j];
351 				r = amdgpu_ring_test_helper(ring);
352 				if (r)
353 					goto done;
354 			}
355 		}
356 	}
357 
358 done:
359 	if (!r)
360 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
361 			(adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
362 
363 	return r;
364 }
365 
366 /**
367  * vcn_v3_0_hw_fini - stop the hardware block
368  *
369  * @handle: amdgpu_device pointer
370  *
371  * Stop the VCN block, mark ring as not ready any more
372  */
373 static int vcn_v3_0_hw_fini(void *handle)
374 {
375 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
376 	int i;
377 
378 	cancel_delayed_work_sync(&adev->vcn.idle_work);
379 
380 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
381 		if (adev->vcn.harvest_config & (1 << i))
382 			continue;
383 
384 		if (!amdgpu_sriov_vf(adev)) {
385 			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
386 					(adev->vcn.cur_state != AMD_PG_STATE_GATE &&
387 					 RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
388 				vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
389 			}
390 		}
391 	}
392 
393 	return 0;
394 }
395 
396 /**
397  * vcn_v3_0_suspend - suspend VCN block
398  *
399  * @handle: amdgpu_device pointer
400  *
401  * HW fini and suspend VCN block
402  */
403 static int vcn_v3_0_suspend(void *handle)
404 {
405 	int r;
406 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
407 
408 	r = vcn_v3_0_hw_fini(adev);
409 	if (r)
410 		return r;
411 
412 	r = amdgpu_vcn_suspend(adev);
413 
414 	return r;
415 }
416 
417 /**
418  * vcn_v3_0_resume - resume VCN block
419  *
420  * @handle: amdgpu_device pointer
421  *
422  * Resume firmware and hw init VCN block
423  */
424 static int vcn_v3_0_resume(void *handle)
425 {
426 	int r;
427 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
428 
429 	r = amdgpu_vcn_resume(adev);
430 	if (r)
431 		return r;
432 
433 	r = vcn_v3_0_hw_init(adev);
434 
435 	return r;
436 }
437 
438 /**
439  * vcn_v3_0_mc_resume - memory controller programming
440  *
441  * @adev: amdgpu_device pointer
442  * @inst: instance number
443  *
444  * Let the VCN memory controller know it's offsets
445  */
446 static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
447 {
448 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
449 	uint32_t offset;
450 
451 	/* cache window 0: fw */
452 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
453 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
454 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
455 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
456 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
457 		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
458 		offset = 0;
459 	} else {
460 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
461 			lower_32_bits(adev->vcn.inst[inst].gpu_addr));
462 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
463 			upper_32_bits(adev->vcn.inst[inst].gpu_addr));
464 		offset = size;
465 		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
466 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
467 	}
468 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
469 
470 	/* cache window 1: stack */
471 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
472 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
473 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
474 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
475 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
476 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
477 
478 	/* cache window 2: context */
479 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
480 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
481 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
482 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
483 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
484 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
485 
486 	/* non-cache window */
487 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
488 		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
489 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
490 		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
491 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
492 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
493 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
494 }
495 
496 static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
497 {
498 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
499 	uint32_t offset;
500 
501 	/* cache window 0: fw */
502 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
503 		if (!indirect) {
504 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
505 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
506 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
507 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
508 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
509 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
510 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
511 				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
512 		} else {
513 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
514 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
515 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
516 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
517 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
518 				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
519 		}
520 		offset = 0;
521 	} else {
522 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
523 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
524 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
525 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
526 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
527 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
528 		offset = size;
529 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
530 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
531 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
532 	}
533 
534 	if (!indirect)
535 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
536 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
537 	else
538 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
539 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
540 
541 	/* cache window 1: stack */
542 	if (!indirect) {
543 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
544 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
545 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
546 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
547 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
548 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
549 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
550 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
551 	} else {
552 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
553 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
554 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
555 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
556 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
557 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
558 	}
559 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
560 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
561 
562 	/* cache window 2: context */
563 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
564 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
565 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
566 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
567 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
568 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
569 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
570 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
571 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
572 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
573 
574 	/* non-cache window */
575 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
576 			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
577 			lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
578 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
579 			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
580 			upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
581 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
582 			VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
583 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
584 			VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
585 			AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
586 
587 	/* VCN global tiling registers */
588 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
589 		UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
590 }
591 
592 static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
593 {
594 	uint32_t data = 0;
595 
596 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
597 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
598 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
599 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
600 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
601 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
602 			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
603 			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
604 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
605 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
606 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
607 			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
608 			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
609 			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
610 			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
611 
612 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
613 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
614 			UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
615 	} else {
616 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
617 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
618 			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
619 			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
620 			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
621 			| 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
622 			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
623 			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
624 			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
625 			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
626 			| 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
627 			| 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
628 			| 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
629 			| 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
630 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
631 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0,  0x3F3FFFFF);
632 	}
633 
634 	data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
635 	data &= ~0x103;
636 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
637 		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
638 			UVD_POWER_STATUS__UVD_PG_EN_MASK;
639 
640 	WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
641 }
642 
643 static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
644 {
645 	uint32_t data;
646 
647 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
648 		/* Before power off, this indicator has to be turned on */
649 		data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
650 		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
651 		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
652 		WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
653 
654 		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
655 			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
656 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
657 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
658 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
659 			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
660 			| 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
661 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
662 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
663 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
664 			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
665 			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
666 			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
667 			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
668 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
669 
670 		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
671 			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
672 			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
673 			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
674 			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
675 			| 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
676 			| 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
677 			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
678 			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
679 			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
680 			| 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
681 			| 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
682 			| 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
683 			| 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
684 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
685 	}
686 }
687 
688 /**
689  * vcn_v3_0_disable_clock_gating - disable VCN clock gating
690  *
691  * @adev: amdgpu_device pointer
692  * @inst: instance number
693  *
694  * Disable clock gating for VCN block
695  */
696 static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
697 {
698 	uint32_t data;
699 
700 	/* VCN disable CGC */
701 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
702 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
703 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
704 	else
705 		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
706 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
707 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
708 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
709 
710 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
711 	data &= ~(UVD_CGC_GATE__SYS_MASK
712 		| UVD_CGC_GATE__UDEC_MASK
713 		| UVD_CGC_GATE__MPEG2_MASK
714 		| UVD_CGC_GATE__REGS_MASK
715 		| UVD_CGC_GATE__RBC_MASK
716 		| UVD_CGC_GATE__LMI_MC_MASK
717 		| UVD_CGC_GATE__LMI_UMC_MASK
718 		| UVD_CGC_GATE__IDCT_MASK
719 		| UVD_CGC_GATE__MPRD_MASK
720 		| UVD_CGC_GATE__MPC_MASK
721 		| UVD_CGC_GATE__LBSI_MASK
722 		| UVD_CGC_GATE__LRBBM_MASK
723 		| UVD_CGC_GATE__UDEC_RE_MASK
724 		| UVD_CGC_GATE__UDEC_CM_MASK
725 		| UVD_CGC_GATE__UDEC_IT_MASK
726 		| UVD_CGC_GATE__UDEC_DB_MASK
727 		| UVD_CGC_GATE__UDEC_MP_MASK
728 		| UVD_CGC_GATE__WCB_MASK
729 		| UVD_CGC_GATE__VCPU_MASK
730 		| UVD_CGC_GATE__MMSCH_MASK);
731 
732 	WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
733 
734 	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0,  0xFFFFFFFF);
735 
736 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
737 	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
738 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
739 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
740 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
741 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
742 		| UVD_CGC_CTRL__SYS_MODE_MASK
743 		| UVD_CGC_CTRL__UDEC_MODE_MASK
744 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
745 		| UVD_CGC_CTRL__REGS_MODE_MASK
746 		| UVD_CGC_CTRL__RBC_MODE_MASK
747 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
748 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
749 		| UVD_CGC_CTRL__IDCT_MODE_MASK
750 		| UVD_CGC_CTRL__MPRD_MODE_MASK
751 		| UVD_CGC_CTRL__MPC_MODE_MASK
752 		| UVD_CGC_CTRL__LBSI_MODE_MASK
753 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
754 		| UVD_CGC_CTRL__WCB_MODE_MASK
755 		| UVD_CGC_CTRL__VCPU_MODE_MASK
756 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
757 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
758 
759 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
760 	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
761 		| UVD_SUVD_CGC_GATE__SIT_MASK
762 		| UVD_SUVD_CGC_GATE__SMP_MASK
763 		| UVD_SUVD_CGC_GATE__SCM_MASK
764 		| UVD_SUVD_CGC_GATE__SDB_MASK
765 		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
766 		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
767 		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
768 		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
769 		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
770 		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
771 		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
772 		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
773 		| UVD_SUVD_CGC_GATE__SCLR_MASK
774 		| UVD_SUVD_CGC_GATE__ENT_MASK
775 		| UVD_SUVD_CGC_GATE__IME_MASK
776 		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
777 		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
778 		| UVD_SUVD_CGC_GATE__SITE_MASK
779 		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
780 		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
781 		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
782 		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
783 		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK
784 		| UVD_SUVD_CGC_GATE__EFC_MASK
785 		| UVD_SUVD_CGC_GATE__SAOE_MASK
786 		| UVD_SUVD_CGC_GATE__SRE_AV1_MASK
787 		| UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
788 		| UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
789 		| UVD_SUVD_CGC_GATE__SCM_AV1_MASK
790 		| UVD_SUVD_CGC_GATE__SMPA_MASK);
791 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
792 
793 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
794 	data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
795 		| UVD_SUVD_CGC_GATE2__MPBE1_MASK
796 		| UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
797 		| UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
798 		| UVD_SUVD_CGC_GATE2__MPC1_MASK);
799 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
800 
801 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
802 	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
803 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
804 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
805 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
806 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
807 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
808 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
809 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
810 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
811 		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
812 		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
813 		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
814 		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
815 		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
816 		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
817 		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
818 		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
819 		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
820 		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
821 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
822 }
823 
824 static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
825 		uint8_t sram_sel, int inst_idx, uint8_t indirect)
826 {
827 	uint32_t reg_data = 0;
828 
829 	/* enable sw clock gating control */
830 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
831 		reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
832 	else
833 		reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
834 	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
835 	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
836 	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
837 		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
838 		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
839 		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
840 		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
841 		 UVD_CGC_CTRL__SYS_MODE_MASK |
842 		 UVD_CGC_CTRL__UDEC_MODE_MASK |
843 		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
844 		 UVD_CGC_CTRL__REGS_MODE_MASK |
845 		 UVD_CGC_CTRL__RBC_MODE_MASK |
846 		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
847 		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
848 		 UVD_CGC_CTRL__IDCT_MODE_MASK |
849 		 UVD_CGC_CTRL__MPRD_MODE_MASK |
850 		 UVD_CGC_CTRL__MPC_MODE_MASK |
851 		 UVD_CGC_CTRL__LBSI_MODE_MASK |
852 		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
853 		 UVD_CGC_CTRL__WCB_MODE_MASK |
854 		 UVD_CGC_CTRL__VCPU_MODE_MASK |
855 		 UVD_CGC_CTRL__MMSCH_MODE_MASK);
856 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
857 		VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
858 
859 	/* turn off clock gating */
860 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
861 		VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
862 
863 	/* turn on SUVD clock gating */
864 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
865 		VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
866 
867 	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
868 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
869 		VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
870 }
871 
872 /**
873  * vcn_v3_0_enable_clock_gating - enable VCN clock gating
874  *
875  * @adev: amdgpu_device pointer
876  * @inst: instance number
877  *
878  * Enable clock gating for VCN block
879  */
880 static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
881 {
882 	uint32_t data;
883 
884 	/* enable VCN CGC */
885 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
886 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
887 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
888 	else
889 		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
890 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
891 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
892 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
893 
894 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
895 	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
896 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
897 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
898 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
899 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
900 		| UVD_CGC_CTRL__SYS_MODE_MASK
901 		| UVD_CGC_CTRL__UDEC_MODE_MASK
902 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
903 		| UVD_CGC_CTRL__REGS_MODE_MASK
904 		| UVD_CGC_CTRL__RBC_MODE_MASK
905 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
906 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
907 		| UVD_CGC_CTRL__IDCT_MODE_MASK
908 		| UVD_CGC_CTRL__MPRD_MODE_MASK
909 		| UVD_CGC_CTRL__MPC_MODE_MASK
910 		| UVD_CGC_CTRL__LBSI_MODE_MASK
911 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
912 		| UVD_CGC_CTRL__WCB_MODE_MASK
913 		| UVD_CGC_CTRL__VCPU_MODE_MASK
914 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
915 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
916 
917 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
918 	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
919 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
920 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
921 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
922 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
923 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
924 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
925 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
926 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
927 		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
928 		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
929 		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
930 		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
931 		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
932 		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
933 		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
934 		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
935 		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
936 		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
937 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
938 }
939 
940 static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
941 {
942 	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
943 	struct amdgpu_ring *ring;
944 	uint32_t rb_bufsz, tmp;
945 
946 	/* disable register anti-hang mechanism */
947 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
948 		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
949 	/* enable dynamic power gating mode */
950 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
951 	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
952 	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
953 	WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
954 
955 	if (indirect)
956 		adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
957 
958 	/* enable clock gating */
959 	vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
960 
961 	/* enable VCPU clock */
962 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
963 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
964 	tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
965 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
966 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
967 
968 	/* disable master interupt */
969 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
970 		VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
971 
972 	/* setup mmUVD_LMI_CTRL */
973 	tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
974 		UVD_LMI_CTRL__REQ_MODE_MASK |
975 		UVD_LMI_CTRL__CRC_RESET_MASK |
976 		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
977 		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
978 		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
979 		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
980 		0x00100000L);
981 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
982 		VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
983 
984 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
985 		VCN, inst_idx, mmUVD_MPC_CNTL),
986 		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
987 
988 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
989 		VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
990 		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
991 		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
992 		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
993 		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
994 
995 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
996 		VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
997 		 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
998 		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
999 		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1000 		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1001 
1002 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1003 		VCN, inst_idx, mmUVD_MPC_SET_MUX),
1004 		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1005 		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1006 		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1007 
1008 	vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
1009 
1010 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1011 		VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1012 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1013 		VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1014 
1015 	/* enable LMI MC and UMC channels */
1016 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1017 		VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1018 
1019 	/* unblock VCPU register access */
1020 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1021 		VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1022 
1023 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1024 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1025 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1026 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1027 
1028 	/* enable master interrupt */
1029 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1030 		VCN, inst_idx, mmUVD_MASTINT_EN),
1031 		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1032 
1033 	/* add nop to workaround PSP size check */
1034 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1035 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1036 
1037 	if (indirect)
1038 		psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
1039 			(uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
1040 				(uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
1041 
1042 	ring = &adev->vcn.inst[inst_idx].ring_dec;
1043 	/* force RBC into idle state */
1044 	rb_bufsz = order_base_2(ring->ring_size);
1045 	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1046 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1047 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1048 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1049 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1050 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1051 
1052 	/* Stall DPG before WPTR/RPTR reset */
1053 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1054 		UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1055 		~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1056 	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1057 
1058 	/* set the write pointer delay */
1059 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1060 
1061 	/* set the wb address */
1062 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1063 		(upper_32_bits(ring->gpu_addr) >> 2));
1064 
1065 	/* programm the RB_BASE for ring buffer */
1066 	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1067 		lower_32_bits(ring->gpu_addr));
1068 	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1069 		upper_32_bits(ring->gpu_addr));
1070 
1071 	/* Initialize the ring buffer's read and write pointers */
1072 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1073 
1074 	WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1075 
1076 	ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1077 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1078 		lower_32_bits(ring->wptr));
1079 
1080 	/* Reset FW shared memory RBC WPTR/RPTR */
1081 	fw_shared->rb.rptr = 0;
1082 	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1083 
1084 	/*resetting done, fw can check RB ring */
1085 	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1086 
1087 	/* Unstall DPG */
1088 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1089 		0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1090 
1091 	return 0;
1092 }
1093 
1094 static int vcn_v3_0_start(struct amdgpu_device *adev)
1095 {
1096 	volatile struct amdgpu_fw_shared *fw_shared;
1097 	struct amdgpu_ring *ring;
1098 	uint32_t rb_bufsz, tmp;
1099 	int i, j, k, r;
1100 
1101 	if (adev->pm.dpm_enabled)
1102 		amdgpu_dpm_enable_uvd(adev, true);
1103 
1104 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1105 		if (adev->vcn.harvest_config & (1 << i))
1106 			continue;
1107 
1108 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG){
1109 			r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1110 			continue;
1111 		}
1112 
1113 		/* disable VCN power gating */
1114 		vcn_v3_0_disable_static_power_gating(adev, i);
1115 
1116 		/* set VCN status busy */
1117 		tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1118 		WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1119 
1120 		/*SW clock gating */
1121 		vcn_v3_0_disable_clock_gating(adev, i);
1122 
1123 		/* enable VCPU clock */
1124 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1125 			UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1126 
1127 		/* disable master interrupt */
1128 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1129 			~UVD_MASTINT_EN__VCPU_EN_MASK);
1130 
1131 		/* enable LMI MC and UMC channels */
1132 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1133 			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1134 
1135 		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1136 		tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1137 		tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1138 		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1139 
1140 		/* setup mmUVD_LMI_CTRL */
1141 		tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1142 		WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1143 			UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK	|
1144 			UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1145 			UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1146 			UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1147 
1148 		/* setup mmUVD_MPC_CNTL */
1149 		tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1150 		tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1151 		tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1152 		WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1153 
1154 		/* setup UVD_MPC_SET_MUXA0 */
1155 		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1156 			((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1157 			(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1158 			(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1159 			(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1160 
1161 		/* setup UVD_MPC_SET_MUXB0 */
1162 		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1163 			((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1164 			(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1165 			(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1166 			(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1167 
1168 		/* setup mmUVD_MPC_SET_MUX */
1169 		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1170 			((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1171 			(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1172 			(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1173 
1174 		vcn_v3_0_mc_resume(adev, i);
1175 
1176 		/* VCN global tiling registers */
1177 		WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1178 			adev->gfx.config.gb_addr_config);
1179 
1180 		/* unblock VCPU register access */
1181 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1182 			~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1183 
1184 		/* release VCPU reset to boot */
1185 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1186 			~UVD_VCPU_CNTL__BLK_RST_MASK);
1187 
1188 		for (j = 0; j < 10; ++j) {
1189 			uint32_t status;
1190 
1191 			for (k = 0; k < 100; ++k) {
1192 				status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1193 				if (status & 2)
1194 					break;
1195 				mdelay(10);
1196 			}
1197 			r = 0;
1198 			if (status & 2)
1199 				break;
1200 
1201 			DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1202 			WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1203 				UVD_VCPU_CNTL__BLK_RST_MASK,
1204 				~UVD_VCPU_CNTL__BLK_RST_MASK);
1205 			mdelay(10);
1206 			WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1207 				~UVD_VCPU_CNTL__BLK_RST_MASK);
1208 
1209 			mdelay(10);
1210 			r = -1;
1211 		}
1212 
1213 		if (r) {
1214 			DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1215 			return r;
1216 		}
1217 
1218 		/* enable master interrupt */
1219 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1220 			UVD_MASTINT_EN__VCPU_EN_MASK,
1221 			~UVD_MASTINT_EN__VCPU_EN_MASK);
1222 
1223 		/* clear the busy bit of VCN_STATUS */
1224 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1225 			~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1226 
1227 		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1228 
1229 		ring = &adev->vcn.inst[i].ring_dec;
1230 		/* force RBC into idle state */
1231 		rb_bufsz = order_base_2(ring->ring_size);
1232 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1233 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1234 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1235 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1236 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1237 		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1238 
1239 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1240 		fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1241 
1242 		/* programm the RB_BASE for ring buffer */
1243 		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1244 			lower_32_bits(ring->gpu_addr));
1245 		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1246 			upper_32_bits(ring->gpu_addr));
1247 
1248 		/* Initialize the ring buffer's read and write pointers */
1249 		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1250 
1251 		WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);
1252 		ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1253 		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1254 			lower_32_bits(ring->wptr));
1255 		fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1256 		fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1257 
1258 		if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1259 			fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1260 			ring = &adev->vcn.inst[i].ring_enc[0];
1261 			WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1262 			WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1263 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1264 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1265 			WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1266 			fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1267 
1268 			fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1269 			ring = &adev->vcn.inst[i].ring_enc[1];
1270 			WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1271 			WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1272 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1273 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1274 			WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1275 			fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1276 		}
1277 	}
1278 
1279 	return 0;
1280 }
1281 
1282 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1283 {
1284 	int i, j;
1285 	struct amdgpu_ring *ring;
1286 	uint64_t cache_addr;
1287 	uint64_t rb_addr;
1288 	uint64_t ctx_addr;
1289 	uint32_t param, resp, expected;
1290 	uint32_t offset, cache_size;
1291 	uint32_t tmp, timeout;
1292 
1293 	struct amdgpu_mm_table *table = &adev->virt.mm_table;
1294 	uint32_t *table_loc;
1295 	uint32_t table_size;
1296 	uint32_t size, size_dw;
1297 
1298 	struct mmsch_v3_0_cmd_direct_write
1299 		direct_wt = { {0} };
1300 	struct mmsch_v3_0_cmd_direct_read_modify_write
1301 		direct_rd_mod_wt = { {0} };
1302 	struct mmsch_v3_0_cmd_end end = { {0} };
1303 	struct mmsch_v3_0_init_header header;
1304 
1305 	direct_wt.cmd_header.command_type =
1306 		MMSCH_COMMAND__DIRECT_REG_WRITE;
1307 	direct_rd_mod_wt.cmd_header.command_type =
1308 		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1309 	end.cmd_header.command_type =
1310 		MMSCH_COMMAND__END;
1311 
1312 	header.version = MMSCH_VERSION;
1313 	header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1314 	for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1315 		header.inst[i].init_status = 0;
1316 		header.inst[i].table_offset = 0;
1317 		header.inst[i].table_size = 0;
1318 	}
1319 
1320 	table_loc = (uint32_t *)table->cpu_addr;
1321 	table_loc += header.total_size;
1322 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1323 		if (adev->vcn.harvest_config & (1 << i))
1324 			continue;
1325 
1326 		table_size = 0;
1327 
1328 		MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1329 			mmUVD_STATUS),
1330 			~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1331 
1332 		cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1333 
1334 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1335 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1336 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1337 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1338 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1339 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1340 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1341 			offset = 0;
1342 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1343 				mmUVD_VCPU_CACHE_OFFSET0),
1344 				0);
1345 		} else {
1346 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1347 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1348 				lower_32_bits(adev->vcn.inst[i].gpu_addr));
1349 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1350 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1351 				upper_32_bits(adev->vcn.inst[i].gpu_addr));
1352 			offset = cache_size;
1353 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1354 				mmUVD_VCPU_CACHE_OFFSET0),
1355 				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1356 		}
1357 
1358 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1359 			mmUVD_VCPU_CACHE_SIZE0),
1360 			cache_size);
1361 
1362 		cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1363 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1364 			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1365 			lower_32_bits(cache_addr));
1366 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1367 			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1368 			upper_32_bits(cache_addr));
1369 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1370 			mmUVD_VCPU_CACHE_OFFSET1),
1371 			0);
1372 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1373 			mmUVD_VCPU_CACHE_SIZE1),
1374 			AMDGPU_VCN_STACK_SIZE);
1375 
1376 		cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1377 			AMDGPU_VCN_STACK_SIZE;
1378 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1379 			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1380 			lower_32_bits(cache_addr));
1381 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1382 			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1383 			upper_32_bits(cache_addr));
1384 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1385 			mmUVD_VCPU_CACHE_OFFSET2),
1386 			0);
1387 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1388 			mmUVD_VCPU_CACHE_SIZE2),
1389 			AMDGPU_VCN_CONTEXT_SIZE);
1390 
1391 		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1392 			ring = &adev->vcn.inst[i].ring_enc[j];
1393 			ring->wptr = 0;
1394 			rb_addr = ring->gpu_addr;
1395 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1396 				mmUVD_RB_BASE_LO),
1397 				lower_32_bits(rb_addr));
1398 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1399 				mmUVD_RB_BASE_HI),
1400 				upper_32_bits(rb_addr));
1401 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1402 				mmUVD_RB_SIZE),
1403 				ring->ring_size / 4);
1404 		}
1405 
1406 		ring = &adev->vcn.inst[i].ring_dec;
1407 		ring->wptr = 0;
1408 		rb_addr = ring->gpu_addr;
1409 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1410 			mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1411 			lower_32_bits(rb_addr));
1412 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1413 			mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1414 			upper_32_bits(rb_addr));
1415 		/* force RBC into idle state */
1416 		tmp = order_base_2(ring->ring_size);
1417 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1418 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1419 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1420 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1421 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1422 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1423 			mmUVD_RBC_RB_CNTL),
1424 			tmp);
1425 
1426 		/* add end packet */
1427 		MMSCH_V3_0_INSERT_END();
1428 
1429 		/* refine header */
1430 		header.inst[i].init_status = 0;
1431 		header.inst[i].table_offset = header.total_size;
1432 		header.inst[i].table_size = table_size;
1433 		header.total_size += table_size;
1434 	}
1435 
1436 	/* Update init table header in memory */
1437 	size = sizeof(struct mmsch_v3_0_init_header);
1438 	table_loc = (uint32_t *)table->cpu_addr;
1439 	memcpy((void *)table_loc, &header, size);
1440 
1441 	/* message MMSCH (in VCN[0]) to initialize this client
1442 	 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1443 	 * of memory descriptor location
1444 	 */
1445 	ctx_addr = table->gpu_addr;
1446 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1447 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1448 
1449 	/* 2, update vmid of descriptor */
1450 	tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1451 	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1452 	/* use domain0 for MM scheduler */
1453 	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1454 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1455 
1456 	/* 3, notify mmsch about the size of this descriptor */
1457 	size = header.total_size;
1458 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1459 
1460 	/* 4, set resp to zero */
1461 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1462 
1463 	/* 5, kick off the initialization and wait until
1464 	 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1465 	 */
1466 	param = 0x10000001;
1467 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1468 	tmp = 0;
1469 	timeout = 1000;
1470 	resp = 0;
1471 	expected = param + 1;
1472 	while (resp != expected) {
1473 		resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1474 		if (resp == expected)
1475 			break;
1476 
1477 		udelay(10);
1478 		tmp = tmp + 10;
1479 		if (tmp >= timeout) {
1480 			DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1481 				" waiting for mmMMSCH_VF_MAILBOX_RESP "\
1482 				"(expected=0x%08x, readback=0x%08x)\n",
1483 				tmp, expected, resp);
1484 			return -EBUSY;
1485 		}
1486 	}
1487 
1488 	return 0;
1489 }
1490 
1491 static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1492 {
1493 	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1494 	uint32_t tmp;
1495 
1496 	vcn_v3_0_pause_dpg_mode(adev, inst_idx, &state);
1497 
1498 	/* Wait for power status to be 1 */
1499 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1500 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1501 
1502 	/* wait for read ptr to be equal to write ptr */
1503 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1504 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1505 
1506 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1507 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1508 
1509 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1510 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1511 
1512 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1513 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1514 
1515 	/* disable dynamic power gating mode */
1516 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1517 		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1518 
1519 	return 0;
1520 }
1521 
1522 static int vcn_v3_0_stop(struct amdgpu_device *adev)
1523 {
1524 	uint32_t tmp;
1525 	int i, r = 0;
1526 
1527 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1528 		if (adev->vcn.harvest_config & (1 << i))
1529 			continue;
1530 
1531 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1532 			r = vcn_v3_0_stop_dpg_mode(adev, i);
1533 			continue;
1534 		}
1535 
1536 		/* wait for vcn idle */
1537 		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1538 		if (r)
1539 			return r;
1540 
1541 		tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1542 			UVD_LMI_STATUS__READ_CLEAN_MASK |
1543 			UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1544 			UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1545 		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1546 		if (r)
1547 			return r;
1548 
1549 		/* disable LMI UMC channel */
1550 		tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1551 		tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1552 		WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1553 		tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1554 			UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1555 		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1556 		if (r)
1557 			return r;
1558 
1559 		/* block VCPU register access */
1560 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1561 			UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1562 			~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1563 
1564 		/* reset VCPU */
1565 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1566 			UVD_VCPU_CNTL__BLK_RST_MASK,
1567 			~UVD_VCPU_CNTL__BLK_RST_MASK);
1568 
1569 		/* disable VCPU clock */
1570 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1571 			~(UVD_VCPU_CNTL__CLK_EN_MASK));
1572 
1573 		/* apply soft reset */
1574 		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1575 		tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1576 		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1577 		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1578 		tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1579 		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1580 
1581 		/* clear status */
1582 		WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1583 
1584 		/* apply HW clock gating */
1585 		vcn_v3_0_enable_clock_gating(adev, i);
1586 
1587 		/* enable VCN power gating */
1588 		vcn_v3_0_enable_static_power_gating(adev, i);
1589 	}
1590 
1591 	if (adev->pm.dpm_enabled)
1592 		amdgpu_dpm_enable_uvd(adev, false);
1593 
1594 	return 0;
1595 }
1596 
1597 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1598 		   int inst_idx, struct dpg_pause_state *new_state)
1599 {
1600 	volatile struct amdgpu_fw_shared *fw_shared;
1601 	struct amdgpu_ring *ring;
1602 	uint32_t reg_data = 0;
1603 	int ret_code;
1604 
1605 	/* pause/unpause if state is changed */
1606 	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1607 		DRM_DEBUG("dpg pause state changed %d -> %d",
1608 			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1609 		reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1610 			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1611 
1612 		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1613 			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1614 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1615 
1616 			if (!ret_code) {
1617 				/* pause DPG */
1618 				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1619 				WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1620 
1621 				/* wait for ACK */
1622 				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1623 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1624 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1625 
1626 				/* Stall DPG before WPTR/RPTR reset */
1627 				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1628 					UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1629 					~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1630 
1631 				if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1632 					/* Restore */
1633 					fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1634 					fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1635 					ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1636 					ring->wptr = 0;
1637 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1638 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1639 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1640 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1641 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1642 					fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1643 
1644 					fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1645 					ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1646 					ring->wptr = 0;
1647 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1648 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1649 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1650 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1651 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1652 					fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1653 
1654 					/* restore wptr/rptr with pointers saved in FW shared memory*/
1655 					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);
1656 					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);
1657 				}
1658 
1659 				/* Unstall DPG */
1660 				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1661 					0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1662 
1663 				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1664 					UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1665 			}
1666 		} else {
1667 			/* unpause dpg, no need to wait */
1668 			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1669 			WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1670 		}
1671 		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1672 	}
1673 
1674 	return 0;
1675 }
1676 
1677 /**
1678  * vcn_v3_0_dec_ring_get_rptr - get read pointer
1679  *
1680  * @ring: amdgpu_ring pointer
1681  *
1682  * Returns the current hardware read pointer
1683  */
1684 static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1685 {
1686 	struct amdgpu_device *adev = ring->adev;
1687 
1688 	return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1689 }
1690 
1691 /**
1692  * vcn_v3_0_dec_ring_get_wptr - get write pointer
1693  *
1694  * @ring: amdgpu_ring pointer
1695  *
1696  * Returns the current hardware write pointer
1697  */
1698 static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1699 {
1700 	struct amdgpu_device *adev = ring->adev;
1701 
1702 	if (ring->use_doorbell)
1703 		return *ring->wptr_cpu_addr;
1704 	else
1705 		return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1706 }
1707 
1708 /**
1709  * vcn_v3_0_dec_ring_set_wptr - set write pointer
1710  *
1711  * @ring: amdgpu_ring pointer
1712  *
1713  * Commits the write pointer to the hardware
1714  */
1715 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1716 {
1717 	struct amdgpu_device *adev = ring->adev;
1718 	volatile struct amdgpu_fw_shared *fw_shared;
1719 
1720 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1721 		/*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */
1722 		fw_shared = adev->vcn.inst[ring->me].fw_shared.cpu_addr;
1723 		fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1724 		WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,
1725 			lower_32_bits(ring->wptr));
1726 	}
1727 
1728 	if (ring->use_doorbell) {
1729 		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1730 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1731 	} else {
1732 		WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1733 	}
1734 }
1735 
1736 static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1737 	.type = AMDGPU_RING_TYPE_VCN_DEC,
1738 	.align_mask = 0x3f,
1739 	.nop = VCN_DEC_SW_CMD_NO_OP,
1740 	.secure_submission_supported = true,
1741 	.vmhub = AMDGPU_MMHUB_0,
1742 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1743 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1744 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1745 	.emit_frame_size =
1746 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1747 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1748 		VCN_SW_RING_EMIT_FRAME_SIZE,
1749 	.emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */
1750 	.emit_ib = vcn_dec_sw_ring_emit_ib,
1751 	.emit_fence = vcn_dec_sw_ring_emit_fence,
1752 	.emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,
1753 	.test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1754 	.test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1755 	.insert_nop = amdgpu_ring_insert_nop,
1756 	.insert_end = vcn_dec_sw_ring_insert_end,
1757 	.pad_ib = amdgpu_ring_generic_pad_ib,
1758 	.begin_use = amdgpu_vcn_ring_begin_use,
1759 	.end_use = amdgpu_vcn_ring_end_use,
1760 	.emit_wreg = vcn_dec_sw_ring_emit_wreg,
1761 	.emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,
1762 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1763 };
1764 
1765 static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p,
1766 				struct amdgpu_job *job)
1767 {
1768 	struct drm_gpu_scheduler **scheds;
1769 
1770 	/* The create msg must be in the first IB submitted */
1771 	if (atomic_read(&job->base.entity->fence_seq))
1772 		return -EINVAL;
1773 
1774 	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
1775 		[AMDGPU_RING_PRIO_DEFAULT].sched;
1776 	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
1777 	return 0;
1778 }
1779 
1780 static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1781 			    uint64_t addr)
1782 {
1783 	struct ttm_operation_ctx ctx = { false, false };
1784 	struct amdgpu_bo_va_mapping *map;
1785 	uint32_t *msg, num_buffers;
1786 	struct amdgpu_bo *bo;
1787 	uint64_t start, end;
1788 	unsigned int i;
1789 	void * ptr;
1790 	int r;
1791 
1792 	addr &= AMDGPU_GMC_HOLE_MASK;
1793 	r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1794 	if (r) {
1795 		DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
1796 		return r;
1797 	}
1798 
1799 	start = map->start * AMDGPU_GPU_PAGE_SIZE;
1800 	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1801 	if (addr & 0x7) {
1802 		DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1803 		return -EINVAL;
1804 	}
1805 
1806 	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1807 	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1808 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1809 	if (r) {
1810 		DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1811 		return r;
1812 	}
1813 
1814 	r = amdgpu_bo_kmap(bo, &ptr);
1815 	if (r) {
1816 		DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1817 		return r;
1818 	}
1819 
1820 	msg = ptr + addr - start;
1821 
1822 	/* Check length */
1823 	if (msg[1] > end - addr) {
1824 		r = -EINVAL;
1825 		goto out;
1826 	}
1827 
1828 	if (msg[3] != RDECODE_MSG_CREATE)
1829 		goto out;
1830 
1831 	num_buffers = msg[2];
1832 	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1833 		uint32_t offset, size, *create;
1834 
1835 		if (msg[0] != RDECODE_MESSAGE_CREATE)
1836 			continue;
1837 
1838 		offset = msg[1];
1839 		size = msg[2];
1840 
1841 		if (offset + size > end) {
1842 			r = -EINVAL;
1843 			goto out;
1844 		}
1845 
1846 		create = ptr + addr + offset - start;
1847 
1848 		/* H246, HEVC and VP9 can run on any instance */
1849 		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1850 			continue;
1851 
1852 		r = vcn_v3_0_limit_sched(p, job);
1853 		if (r)
1854 			goto out;
1855 	}
1856 
1857 out:
1858 	amdgpu_bo_kunmap(bo);
1859 	return r;
1860 }
1861 
1862 static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1863 					   struct amdgpu_job *job,
1864 					   struct amdgpu_ib *ib)
1865 {
1866 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1867 	uint32_t msg_lo = 0, msg_hi = 0;
1868 	unsigned i;
1869 	int r;
1870 
1871 	/* The first instance can decode anything */
1872 	if (!ring->me)
1873 		return 0;
1874 
1875 	for (i = 0; i < ib->length_dw; i += 2) {
1876 		uint32_t reg = amdgpu_ib_get_value(ib, i);
1877 		uint32_t val = amdgpu_ib_get_value(ib, i + 1);
1878 
1879 		if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {
1880 			msg_lo = val;
1881 		} else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {
1882 			msg_hi = val;
1883 		} else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&
1884 			   val == 0) {
1885 			r = vcn_v3_0_dec_msg(p, job,
1886 					     ((u64)msg_hi) << 32 | msg_lo);
1887 			if (r)
1888 				return r;
1889 		}
1890 	}
1891 	return 0;
1892 }
1893 
1894 static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1895 	.type = AMDGPU_RING_TYPE_VCN_DEC,
1896 	.align_mask = 0xf,
1897 	.secure_submission_supported = true,
1898 	.vmhub = AMDGPU_MMHUB_0,
1899 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1900 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1901 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1902 	.patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,
1903 	.emit_frame_size =
1904 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1905 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1906 		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1907 		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1908 		6,
1909 	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1910 	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
1911 	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
1912 	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1913 	.test_ring = vcn_v2_0_dec_ring_test_ring,
1914 	.test_ib = amdgpu_vcn_dec_ring_test_ib,
1915 	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
1916 	.insert_start = vcn_v2_0_dec_ring_insert_start,
1917 	.insert_end = vcn_v2_0_dec_ring_insert_end,
1918 	.pad_ib = amdgpu_ring_generic_pad_ib,
1919 	.begin_use = amdgpu_vcn_ring_begin_use,
1920 	.end_use = amdgpu_vcn_ring_end_use,
1921 	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1922 	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1923 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1924 };
1925 
1926 /**
1927  * vcn_v3_0_enc_ring_get_rptr - get enc read pointer
1928  *
1929  * @ring: amdgpu_ring pointer
1930  *
1931  * Returns the current hardware enc read pointer
1932  */
1933 static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1934 {
1935 	struct amdgpu_device *adev = ring->adev;
1936 
1937 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1938 		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1939 	else
1940 		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1941 }
1942 
1943 /**
1944  * vcn_v3_0_enc_ring_get_wptr - get enc write pointer
1945  *
1946  * @ring: amdgpu_ring pointer
1947  *
1948  * Returns the current hardware enc write pointer
1949  */
1950 static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1951 {
1952 	struct amdgpu_device *adev = ring->adev;
1953 
1954 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1955 		if (ring->use_doorbell)
1956 			return *ring->wptr_cpu_addr;
1957 		else
1958 			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1959 	} else {
1960 		if (ring->use_doorbell)
1961 			return *ring->wptr_cpu_addr;
1962 		else
1963 			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1964 	}
1965 }
1966 
1967 /**
1968  * vcn_v3_0_enc_ring_set_wptr - set enc write pointer
1969  *
1970  * @ring: amdgpu_ring pointer
1971  *
1972  * Commits the enc write pointer to the hardware
1973  */
1974 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1975 {
1976 	struct amdgpu_device *adev = ring->adev;
1977 
1978 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1979 		if (ring->use_doorbell) {
1980 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1981 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1982 		} else {
1983 			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1984 		}
1985 	} else {
1986 		if (ring->use_doorbell) {
1987 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1988 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1989 		} else {
1990 			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1991 		}
1992 	}
1993 }
1994 
1995 static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
1996 	.type = AMDGPU_RING_TYPE_VCN_ENC,
1997 	.align_mask = 0x3f,
1998 	.nop = VCN_ENC_CMD_NO_OP,
1999 	.vmhub = AMDGPU_MMHUB_0,
2000 	.get_rptr = vcn_v3_0_enc_ring_get_rptr,
2001 	.get_wptr = vcn_v3_0_enc_ring_get_wptr,
2002 	.set_wptr = vcn_v3_0_enc_ring_set_wptr,
2003 	.emit_frame_size =
2004 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2005 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2006 		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2007 		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2008 		1, /* vcn_v2_0_enc_ring_insert_end */
2009 	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2010 	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
2011 	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
2012 	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2013 	.test_ring = amdgpu_vcn_enc_ring_test_ring,
2014 	.test_ib = amdgpu_vcn_enc_ring_test_ib,
2015 	.insert_nop = amdgpu_ring_insert_nop,
2016 	.insert_end = vcn_v2_0_enc_ring_insert_end,
2017 	.pad_ib = amdgpu_ring_generic_pad_ib,
2018 	.begin_use = amdgpu_vcn_ring_begin_use,
2019 	.end_use = amdgpu_vcn_ring_end_use,
2020 	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2021 	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2022 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2023 };
2024 
2025 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2026 {
2027 	int i;
2028 
2029 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2030 		if (adev->vcn.harvest_config & (1 << i))
2031 			continue;
2032 
2033 		if (!DEC_SW_RING_ENABLED)
2034 			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
2035 		else
2036 			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
2037 		adev->vcn.inst[i].ring_dec.me = i;
2038 		DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
2039 			  DEC_SW_RING_ENABLED?"(Software Ring)":"");
2040 	}
2041 }
2042 
2043 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2044 {
2045 	int i, j;
2046 
2047 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2048 		if (adev->vcn.harvest_config & (1 << i))
2049 			continue;
2050 
2051 		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
2052 			adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
2053 			adev->vcn.inst[i].ring_enc[j].me = i;
2054 		}
2055 		if (adev->vcn.num_enc_rings > 0)
2056 			DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
2057 	}
2058 }
2059 
2060 static bool vcn_v3_0_is_idle(void *handle)
2061 {
2062 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2063 	int i, ret = 1;
2064 
2065 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2066 		if (adev->vcn.harvest_config & (1 << i))
2067 			continue;
2068 
2069 		ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
2070 	}
2071 
2072 	return ret;
2073 }
2074 
2075 static int vcn_v3_0_wait_for_idle(void *handle)
2076 {
2077 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2078 	int i, ret = 0;
2079 
2080 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2081 		if (adev->vcn.harvest_config & (1 << i))
2082 			continue;
2083 
2084 		ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2085 			UVD_STATUS__IDLE);
2086 		if (ret)
2087 			return ret;
2088 	}
2089 
2090 	return ret;
2091 }
2092 
2093 static int vcn_v3_0_set_clockgating_state(void *handle,
2094 					  enum amd_clockgating_state state)
2095 {
2096 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2097 	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
2098 	int i;
2099 
2100 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2101 		if (adev->vcn.harvest_config & (1 << i))
2102 			continue;
2103 
2104 		if (enable) {
2105 			if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2106 				return -EBUSY;
2107 			vcn_v3_0_enable_clock_gating(adev, i);
2108 		} else {
2109 			vcn_v3_0_disable_clock_gating(adev, i);
2110 		}
2111 	}
2112 
2113 	return 0;
2114 }
2115 
2116 static int vcn_v3_0_set_powergating_state(void *handle,
2117 					  enum amd_powergating_state state)
2118 {
2119 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2120 	int ret;
2121 
2122 	/* for SRIOV, guest should not control VCN Power-gating
2123 	 * MMSCH FW should control Power-gating and clock-gating
2124 	 * guest should avoid touching CGC and PG
2125 	 */
2126 	if (amdgpu_sriov_vf(adev)) {
2127 		adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2128 		return 0;
2129 	}
2130 
2131 	if(state == adev->vcn.cur_state)
2132 		return 0;
2133 
2134 	if (state == AMD_PG_STATE_GATE)
2135 		ret = vcn_v3_0_stop(adev);
2136 	else
2137 		ret = vcn_v3_0_start(adev);
2138 
2139 	if(!ret)
2140 		adev->vcn.cur_state = state;
2141 
2142 	return ret;
2143 }
2144 
2145 static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2146 					struct amdgpu_irq_src *source,
2147 					unsigned type,
2148 					enum amdgpu_interrupt_state state)
2149 {
2150 	return 0;
2151 }
2152 
2153 static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2154 				      struct amdgpu_irq_src *source,
2155 				      struct amdgpu_iv_entry *entry)
2156 {
2157 	uint32_t ip_instance;
2158 
2159 	switch (entry->client_id) {
2160 	case SOC15_IH_CLIENTID_VCN:
2161 		ip_instance = 0;
2162 		break;
2163 	case SOC15_IH_CLIENTID_VCN1:
2164 		ip_instance = 1;
2165 		break;
2166 	default:
2167 		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2168 		return 0;
2169 	}
2170 
2171 	DRM_DEBUG("IH: VCN TRAP\n");
2172 
2173 	switch (entry->src_id) {
2174 	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2175 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2176 		break;
2177 	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2178 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2179 		break;
2180 	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2181 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2182 		break;
2183 	default:
2184 		DRM_ERROR("Unhandled interrupt: %d %d\n",
2185 			  entry->src_id, entry->src_data[0]);
2186 		break;
2187 	}
2188 
2189 	return 0;
2190 }
2191 
2192 static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2193 	.set = vcn_v3_0_set_interrupt_state,
2194 	.process = vcn_v3_0_process_interrupt,
2195 };
2196 
2197 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2198 {
2199 	int i;
2200 
2201 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2202 		if (adev->vcn.harvest_config & (1 << i))
2203 			continue;
2204 
2205 		adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2206 		adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2207 	}
2208 }
2209 
2210 static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2211 	.name = "vcn_v3_0",
2212 	.early_init = vcn_v3_0_early_init,
2213 	.late_init = NULL,
2214 	.sw_init = vcn_v3_0_sw_init,
2215 	.sw_fini = vcn_v3_0_sw_fini,
2216 	.hw_init = vcn_v3_0_hw_init,
2217 	.hw_fini = vcn_v3_0_hw_fini,
2218 	.suspend = vcn_v3_0_suspend,
2219 	.resume = vcn_v3_0_resume,
2220 	.is_idle = vcn_v3_0_is_idle,
2221 	.wait_for_idle = vcn_v3_0_wait_for_idle,
2222 	.check_soft_reset = NULL,
2223 	.pre_soft_reset = NULL,
2224 	.soft_reset = NULL,
2225 	.post_soft_reset = NULL,
2226 	.set_clockgating_state = vcn_v3_0_set_clockgating_state,
2227 	.set_powergating_state = vcn_v3_0_set_powergating_state,
2228 };
2229 
2230 const struct amdgpu_ip_block_version vcn_v3_0_ip_block =
2231 {
2232 	.type = AMD_IP_BLOCK_TYPE_VCN,
2233 	.major = 3,
2234 	.minor = 0,
2235 	.rev = 0,
2236 	.funcs = &vcn_v3_0_ip_funcs,
2237 };
2238