xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c (revision 859e4659)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 
27 #include <linux/firmware.h>
28 #include <drm/drm_drv.h>
29 
30 #include "amdgpu.h"
31 #include "amdgpu_vce.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "soc15_common.h"
35 #include "mmsch_v1_0.h"
36 
37 #include "vce/vce_4_0_offset.h"
38 #include "vce/vce_4_0_default.h"
39 #include "vce/vce_4_0_sh_mask.h"
40 #include "mmhub/mmhub_1_0_offset.h"
41 #include "mmhub/mmhub_1_0_sh_mask.h"
42 
43 #include "ivsrcid/vce/irqsrcs_vce_4_0.h"
44 
45 #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
46 
47 #define VCE_V4_0_FW_SIZE	(384 * 1024)
48 #define VCE_V4_0_STACK_SIZE	(64 * 1024)
49 #define VCE_V4_0_DATA_SIZE	((16 * 1024 * AMDGPU_MAX_VCE_HANDLES) + (52 * 1024))
50 
51 static void vce_v4_0_mc_resume(struct amdgpu_device *adev);
52 static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev);
53 static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev);
54 
55 /**
56  * vce_v4_0_ring_get_rptr - get read pointer
57  *
58  * @ring: amdgpu_ring pointer
59  *
60  * Returns the current hardware read pointer
61  */
62 static uint64_t vce_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
63 {
64 	struct amdgpu_device *adev = ring->adev;
65 
66 	if (ring->me == 0)
67 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR));
68 	else if (ring->me == 1)
69 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2));
70 	else
71 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3));
72 }
73 
74 /**
75  * vce_v4_0_ring_get_wptr - get write pointer
76  *
77  * @ring: amdgpu_ring pointer
78  *
79  * Returns the current hardware write pointer
80  */
81 static uint64_t vce_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
82 {
83 	struct amdgpu_device *adev = ring->adev;
84 
85 	if (ring->use_doorbell)
86 		return adev->wb.wb[ring->wptr_offs];
87 
88 	if (ring->me == 0)
89 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR));
90 	else if (ring->me == 1)
91 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2));
92 	else
93 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3));
94 }
95 
96 /**
97  * vce_v4_0_ring_set_wptr - set write pointer
98  *
99  * @ring: amdgpu_ring pointer
100  *
101  * Commits the write pointer to the hardware
102  */
103 static void vce_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
104 {
105 	struct amdgpu_device *adev = ring->adev;
106 
107 	if (ring->use_doorbell) {
108 		/* XXX check if swapping is necessary on BE */
109 		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
110 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
111 		return;
112 	}
113 
114 	if (ring->me == 0)
115 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR),
116 			lower_32_bits(ring->wptr));
117 	else if (ring->me == 1)
118 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2),
119 			lower_32_bits(ring->wptr));
120 	else
121 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3),
122 			lower_32_bits(ring->wptr));
123 }
124 
125 static int vce_v4_0_firmware_loaded(struct amdgpu_device *adev)
126 {
127 	int i, j;
128 
129 	for (i = 0; i < 10; ++i) {
130 		for (j = 0; j < 100; ++j) {
131 			uint32_t status =
132 				RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS));
133 
134 			if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK)
135 				return 0;
136 			mdelay(10);
137 		}
138 
139 		DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
140 		WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
141 				VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
142 				~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
143 		mdelay(10);
144 		WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 0,
145 				~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
146 		mdelay(10);
147 
148 	}
149 
150 	return -ETIMEDOUT;
151 }
152 
153 static int vce_v4_0_mmsch_start(struct amdgpu_device *adev,
154 				struct amdgpu_mm_table *table)
155 {
156 	uint32_t data = 0, loop;
157 	uint64_t addr = table->gpu_addr;
158 	struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;
159 	uint32_t size;
160 
161 	size = header->header_size + header->vce_table_size + header->uvd_table_size;
162 
163 	/* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of memory descriptor location */
164 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_LO), lower_32_bits(addr));
165 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_HI), upper_32_bits(addr));
166 
167 	/* 2, update vmid of descriptor */
168 	data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID));
169 	data &= ~VCE_MMSCH_VF_VMID__VF_CTX_VMID_MASK;
170 	data |= (0 << VCE_MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); /* use domain0 for MM scheduler */
171 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID), data);
172 
173 	/* 3, notify mmsch about the size of this descriptor */
174 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_SIZE), size);
175 
176 	/* 4, set resp to zero */
177 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP), 0);
178 
179 	WDOORBELL32(adev->vce.ring[0].doorbell_index, 0);
180 	adev->wb.wb[adev->vce.ring[0].wptr_offs] = 0;
181 	adev->vce.ring[0].wptr = 0;
182 	adev->vce.ring[0].wptr_old = 0;
183 
184 	/* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
185 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST), 0x10000001);
186 
187 	data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));
188 	loop = 1000;
189 	while ((data & 0x10000002) != 0x10000002) {
190 		udelay(10);
191 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));
192 		loop--;
193 		if (!loop)
194 			break;
195 	}
196 
197 	if (!loop) {
198 		dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);
199 		return -EBUSY;
200 	}
201 
202 	return 0;
203 }
204 
205 static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
206 {
207 	struct amdgpu_ring *ring;
208 	uint32_t offset, size;
209 	uint32_t table_size = 0;
210 	struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };
211 	struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };
212 	struct mmsch_v1_0_cmd_direct_polling direct_poll = { { 0 } };
213 	struct mmsch_v1_0_cmd_end end = { { 0 } };
214 	uint32_t *init_table = adev->virt.mm_table.cpu_addr;
215 	struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;
216 
217 	direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
218 	direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
219 	direct_poll.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_POLLING;
220 	end.cmd_header.command_type = MMSCH_COMMAND__END;
221 
222 	if (header->vce_table_offset == 0 && header->vce_table_size == 0) {
223 		header->version = MMSCH_VERSION;
224 		header->header_size = sizeof(struct mmsch_v1_0_init_header) >> 2;
225 
226 		if (header->uvd_table_offset == 0 && header->uvd_table_size == 0)
227 			header->vce_table_offset = header->header_size;
228 		else
229 			header->vce_table_offset = header->uvd_table_size + header->uvd_table_offset;
230 
231 		init_table += header->vce_table_offset;
232 
233 		ring = &adev->vce.ring[0];
234 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO),
235 					    lower_32_bits(ring->gpu_addr));
236 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI),
237 					    upper_32_bits(ring->gpu_addr));
238 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE),
239 					    ring->ring_size / 4);
240 
241 		/* BEGING OF MC_RESUME */
242 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x398000);
243 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), ~0x1, 0);
244 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);
245 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);
246 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
247 
248 		offset = AMDGPU_VCE_FIRMWARE_OFFSET;
249 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
250 			uint32_t low = adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_lo;
251 			uint32_t hi = adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_hi;
252 			uint64_t tmr_mc_addr = (uint64_t)(hi) << 32 | low;
253 
254 			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
255 						mmVCE_LMI_VCPU_CACHE_40BIT_BAR0), tmr_mc_addr >> 8);
256 			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
257 						mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
258 						(tmr_mc_addr >> 40) & 0xff);
259 			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), 0);
260 		} else {
261 			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
262 						mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
263 						adev->vce.gpu_addr >> 8);
264 			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
265 						mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
266 						(adev->vce.gpu_addr >> 40) & 0xff);
267 			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0),
268 						offset & ~0x0f000000);
269 
270 		}
271 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
272 						mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
273 						adev->vce.gpu_addr >> 8);
274 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
275 						mmVCE_LMI_VCPU_CACHE_64BIT_BAR1),
276 						(adev->vce.gpu_addr >> 40) & 0xff);
277 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
278 						mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
279 						adev->vce.gpu_addr >> 8);
280 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
281 						mmVCE_LMI_VCPU_CACHE_64BIT_BAR2),
282 						(adev->vce.gpu_addr >> 40) & 0xff);
283 
284 		size = VCE_V4_0_FW_SIZE;
285 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
286 
287 		offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;
288 		size = VCE_V4_0_STACK_SIZE;
289 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1),
290 					(offset & ~0x0f000000) | (1 << 24));
291 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
292 
293 		offset += size;
294 		size = VCE_V4_0_DATA_SIZE;
295 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2),
296 					(offset & ~0x0f000000) | (2 << 24));
297 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
298 
299 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);
300 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),
301 						   VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
302 						   VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
303 
304 		/* end of MC_RESUME */
305 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
306 						   VCE_STATUS__JOB_BUSY_MASK, ~VCE_STATUS__JOB_BUSY_MASK);
307 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL),
308 						   ~0x200001, VCE_VCPU_CNTL__CLK_EN_MASK);
309 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
310 						   ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, 0);
311 
312 		MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
313 					      VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK,
314 					      VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK);
315 
316 		/* clear BUSY flag */
317 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
318 						   ~VCE_STATUS__JOB_BUSY_MASK, 0);
319 
320 		/* add end packet */
321 		memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
322 		table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
323 		header->vce_table_size = table_size;
324 	}
325 
326 	return vce_v4_0_mmsch_start(adev, &adev->virt.mm_table);
327 }
328 
329 /**
330  * vce_v4_0_start - start VCE block
331  *
332  * @adev: amdgpu_device pointer
333  *
334  * Setup and start the VCE block
335  */
336 static int vce_v4_0_start(struct amdgpu_device *adev)
337 {
338 	struct amdgpu_ring *ring;
339 	int r;
340 
341 	ring = &adev->vce.ring[0];
342 
343 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR), lower_32_bits(ring->wptr));
344 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR), lower_32_bits(ring->wptr));
345 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO), ring->gpu_addr);
346 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
347 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE), ring->ring_size / 4);
348 
349 	ring = &adev->vce.ring[1];
350 
351 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2), lower_32_bits(ring->wptr));
352 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2), lower_32_bits(ring->wptr));
353 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO2), ring->gpu_addr);
354 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI2), upper_32_bits(ring->gpu_addr));
355 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE2), ring->ring_size / 4);
356 
357 	ring = &adev->vce.ring[2];
358 
359 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3), lower_32_bits(ring->wptr));
360 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3), lower_32_bits(ring->wptr));
361 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO3), ring->gpu_addr);
362 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI3), upper_32_bits(ring->gpu_addr));
363 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE3), ring->ring_size / 4);
364 
365 	vce_v4_0_mc_resume(adev);
366 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), VCE_STATUS__JOB_BUSY_MASK,
367 			~VCE_STATUS__JOB_BUSY_MASK);
368 
369 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 1, ~0x200001);
370 
371 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 0,
372 			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
373 	mdelay(100);
374 
375 	r = vce_v4_0_firmware_loaded(adev);
376 
377 	/* clear BUSY flag */
378 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0, ~VCE_STATUS__JOB_BUSY_MASK);
379 
380 	if (r) {
381 		DRM_ERROR("VCE not responding, giving up!!!\n");
382 		return r;
383 	}
384 
385 	return 0;
386 }
387 
388 static int vce_v4_0_stop(struct amdgpu_device *adev)
389 {
390 
391 	/* Disable VCPU */
392 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 0, ~0x200001);
393 
394 	/* hold on ECPU */
395 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
396 			VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
397 			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
398 
399 	/* clear VCE_STATUS */
400 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0);
401 
402 	/* Set Clock-Gating off */
403 	/* if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
404 		vce_v4_0_set_vce_sw_clock_gating(adev, false);
405 	*/
406 
407 	return 0;
408 }
409 
410 static int vce_v4_0_early_init(void *handle)
411 {
412 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
413 
414 	if (amdgpu_sriov_vf(adev)) /* currently only VCN0 support SRIOV */
415 		adev->vce.num_rings = 1;
416 	else
417 		adev->vce.num_rings = 3;
418 
419 	vce_v4_0_set_ring_funcs(adev);
420 	vce_v4_0_set_irq_funcs(adev);
421 
422 	return 0;
423 }
424 
425 static int vce_v4_0_sw_init(void *handle)
426 {
427 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
428 	struct amdgpu_ring *ring;
429 
430 	unsigned size;
431 	int r, i;
432 
433 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCE0, 167, &adev->vce.irq);
434 	if (r)
435 		return r;
436 
437 	size  = VCE_V4_0_STACK_SIZE + VCE_V4_0_DATA_SIZE;
438 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
439 		size += VCE_V4_0_FW_SIZE;
440 
441 	r = amdgpu_vce_sw_init(adev, size);
442 	if (r)
443 		return r;
444 
445 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
446 		const struct common_firmware_header *hdr;
447 		unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
448 
449 		adev->vce.saved_bo = kvmalloc(size, GFP_KERNEL);
450 		if (!adev->vce.saved_bo)
451 			return -ENOMEM;
452 
453 		hdr = (const struct common_firmware_header *)adev->vce.fw->data;
454 		adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].ucode_id = AMDGPU_UCODE_ID_VCE;
455 		adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].fw = adev->vce.fw;
456 		adev->firmware.fw_size +=
457 			ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
458 		DRM_INFO("PSP loading VCE firmware\n");
459 	} else {
460 		r = amdgpu_vce_resume(adev);
461 		if (r)
462 			return r;
463 	}
464 
465 	for (i = 0; i < adev->vce.num_rings; i++) {
466 		ring = &adev->vce.ring[i];
467 		sprintf(ring->name, "vce%d", i);
468 		if (amdgpu_sriov_vf(adev)) {
469 			/* DOORBELL only works under SRIOV */
470 			ring->use_doorbell = true;
471 
472 			/* currently only use the first encoding ring for sriov,
473 			 * so set unused location for other unused rings.
474 			 */
475 			if (i == 0)
476 				ring->doorbell_index = adev->doorbell_index.uvd_vce.vce_ring0_1 * 2;
477 			else
478 				ring->doorbell_index = adev->doorbell_index.uvd_vce.vce_ring2_3 * 2 + 1;
479 		}
480 		r = amdgpu_ring_init(adev, ring, 512, &adev->vce.irq, 0,
481 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
482 		if (r)
483 			return r;
484 	}
485 
486 
487 	r = amdgpu_vce_entity_init(adev);
488 	if (r)
489 		return r;
490 
491 	r = amdgpu_virt_alloc_mm_table(adev);
492 	if (r)
493 		return r;
494 
495 	return r;
496 }
497 
498 static int vce_v4_0_sw_fini(void *handle)
499 {
500 	int r;
501 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
502 
503 	/* free MM table */
504 	amdgpu_virt_free_mm_table(adev);
505 
506 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
507 		kvfree(adev->vce.saved_bo);
508 		adev->vce.saved_bo = NULL;
509 	}
510 
511 	r = amdgpu_vce_suspend(adev);
512 	if (r)
513 		return r;
514 
515 	return amdgpu_vce_sw_fini(adev);
516 }
517 
518 static int vce_v4_0_hw_init(void *handle)
519 {
520 	int r, i;
521 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
522 
523 	if (amdgpu_sriov_vf(adev))
524 		r = vce_v4_0_sriov_start(adev);
525 	else
526 		r = vce_v4_0_start(adev);
527 	if (r)
528 		return r;
529 
530 	for (i = 0; i < adev->vce.num_rings; i++) {
531 		r = amdgpu_ring_test_helper(&adev->vce.ring[i]);
532 		if (r)
533 			return r;
534 	}
535 
536 	DRM_INFO("VCE initialized successfully.\n");
537 
538 	return 0;
539 }
540 
541 static int vce_v4_0_hw_fini(void *handle)
542 {
543 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
544 
545 	/*
546 	 * Proper cleanups before halting the HW engine:
547 	 *   - cancel the delayed idle work
548 	 *   - enable powergating
549 	 *   - enable clockgating
550 	 *   - disable dpm
551 	 *
552 	 * TODO: to align with the VCN implementation, move the
553 	 * jobs for clockgating/powergating/dpm setting to
554 	 * ->set_powergating_state().
555 	 */
556 	cancel_delayed_work_sync(&adev->vce.idle_work);
557 
558 	if (adev->pm.dpm_enabled) {
559 		amdgpu_dpm_enable_vce(adev, false);
560 	} else {
561 		amdgpu_asic_set_vce_clocks(adev, 0, 0);
562 		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
563 						       AMD_PG_STATE_GATE);
564 		amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
565 						       AMD_CG_STATE_GATE);
566 	}
567 
568 	if (!amdgpu_sriov_vf(adev)) {
569 		/* vce_v4_0_wait_for_idle(handle); */
570 		vce_v4_0_stop(adev);
571 	} else {
572 		/* full access mode, so don't touch any VCE register */
573 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
574 	}
575 
576 	return 0;
577 }
578 
579 static int vce_v4_0_suspend(void *handle)
580 {
581 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
582 	int r, idx;
583 
584 	if (adev->vce.vcpu_bo == NULL)
585 		return 0;
586 
587 	if (drm_dev_enter(&adev->ddev, &idx)) {
588 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
589 			unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
590 			void *ptr = adev->vce.cpu_addr;
591 
592 			memcpy_fromio(adev->vce.saved_bo, ptr, size);
593 		}
594 		drm_dev_exit(idx);
595 	}
596 
597 	r = vce_v4_0_hw_fini(adev);
598 	if (r)
599 		return r;
600 
601 	return amdgpu_vce_suspend(adev);
602 }
603 
604 static int vce_v4_0_resume(void *handle)
605 {
606 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
607 	int r, idx;
608 
609 	if (adev->vce.vcpu_bo == NULL)
610 		return -EINVAL;
611 
612 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
613 
614 		if (drm_dev_enter(&adev->ddev, &idx)) {
615 			unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
616 			void *ptr = adev->vce.cpu_addr;
617 
618 			memcpy_toio(ptr, adev->vce.saved_bo, size);
619 			drm_dev_exit(idx);
620 		}
621 	} else {
622 		r = amdgpu_vce_resume(adev);
623 		if (r)
624 			return r;
625 	}
626 
627 	return vce_v4_0_hw_init(adev);
628 }
629 
630 static void vce_v4_0_mc_resume(struct amdgpu_device *adev)
631 {
632 	uint32_t offset, size;
633 	uint64_t tmr_mc_addr;
634 
635 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A), 0, ~(1 << 16));
636 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), 0x1FF000, ~0xFF9FF000);
637 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), 0x3F, ~0x3F);
638 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), 0x1FF);
639 
640 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x00398000);
641 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), 0x0, ~0x1);
642 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);
643 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);
644 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
645 
646 	offset = AMDGPU_VCE_FIRMWARE_OFFSET;
647 
648 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
649 		tmr_mc_addr = (uint64_t)(adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_hi) << 32 |
650 										adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].tmr_mc_addr_lo;
651 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
652 			(tmr_mc_addr >> 8));
653 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
654 			(tmr_mc_addr >> 40) & 0xff);
655 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), 0);
656 	} else {
657 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
658 			(adev->vce.gpu_addr >> 8));
659 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
660 			(adev->vce.gpu_addr >> 40) & 0xff);
661 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), offset & ~0x0f000000);
662 	}
663 
664 	size = VCE_V4_0_FW_SIZE;
665 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
666 
667 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1), (adev->vce.gpu_addr >> 8));
668 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR1), (adev->vce.gpu_addr >> 40) & 0xff);
669 	offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;
670 	size = VCE_V4_0_STACK_SIZE;
671 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1), (offset & ~0x0f000000) | (1 << 24));
672 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
673 
674 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2), (adev->vce.gpu_addr >> 8));
675 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR2), (adev->vce.gpu_addr >> 40) & 0xff);
676 	offset += size;
677 	size = VCE_V4_0_DATA_SIZE;
678 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2), (offset & ~0x0f000000) | (2 << 24));
679 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
680 
681 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), 0x0, ~0x100);
682 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),
683 			VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
684 			~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
685 }
686 
687 static int vce_v4_0_set_clockgating_state(void *handle,
688 					  enum amd_clockgating_state state)
689 {
690 	/* needed for driver unload*/
691 	return 0;
692 }
693 
694 #if 0
695 static bool vce_v4_0_is_idle(void *handle)
696 {
697 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
698 	u32 mask = 0;
699 
700 	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_STATUS2__VCE0_BUSY_MASK;
701 	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_STATUS2__VCE1_BUSY_MASK;
702 
703 	return !(RREG32(mmSRBM_STATUS2) & mask);
704 }
705 
706 static int vce_v4_0_wait_for_idle(void *handle)
707 {
708 	unsigned i;
709 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
710 
711 	for (i = 0; i < adev->usec_timeout; i++)
712 		if (vce_v4_0_is_idle(handle))
713 			return 0;
714 
715 	return -ETIMEDOUT;
716 }
717 
718 #define  VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK  0x00000008L   /* AUTO_BUSY */
719 #define  VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK   0x00000010L   /* RB0_BUSY */
720 #define  VCE_STATUS_VCPU_REPORT_RB1_BUSY_MASK   0x00000020L   /* RB1_BUSY */
721 #define  AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \
722 				      VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)
723 
724 static bool vce_v4_0_check_soft_reset(void *handle)
725 {
726 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
727 	u32 srbm_soft_reset = 0;
728 
729 	/* According to VCE team , we should use VCE_STATUS instead
730 	 * SRBM_STATUS.VCE_BUSY bit for busy status checking.
731 	 * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE
732 	 * instance's registers are accessed
733 	 * (0 for 1st instance, 10 for 2nd instance).
734 	 *
735 	 *VCE_STATUS
736 	 *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 |          |FW_LOADED|JOB |
737 	 *|----+----+-----------+----+----+----+----------+---------+----|
738 	 *|bit8|bit7|    bit6   |bit5|bit4|bit3|   bit2   |  bit1   |bit0|
739 	 *
740 	 * VCE team suggest use bit 3--bit 6 for busy status check
741 	 */
742 	mutex_lock(&adev->grbm_idx_mutex);
743 	WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
744 	if (RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
745 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
746 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
747 	}
748 	WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0x10);
749 	if (RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
750 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
751 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
752 	}
753 	WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
754 	mutex_unlock(&adev->grbm_idx_mutex);
755 
756 	if (srbm_soft_reset) {
757 		adev->vce.srbm_soft_reset = srbm_soft_reset;
758 		return true;
759 	} else {
760 		adev->vce.srbm_soft_reset = 0;
761 		return false;
762 	}
763 }
764 
765 static int vce_v4_0_soft_reset(void *handle)
766 {
767 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
768 	u32 srbm_soft_reset;
769 
770 	if (!adev->vce.srbm_soft_reset)
771 		return 0;
772 	srbm_soft_reset = adev->vce.srbm_soft_reset;
773 
774 	if (srbm_soft_reset) {
775 		u32 tmp;
776 
777 		tmp = RREG32(mmSRBM_SOFT_RESET);
778 		tmp |= srbm_soft_reset;
779 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
780 		WREG32(mmSRBM_SOFT_RESET, tmp);
781 		tmp = RREG32(mmSRBM_SOFT_RESET);
782 
783 		udelay(50);
784 
785 		tmp &= ~srbm_soft_reset;
786 		WREG32(mmSRBM_SOFT_RESET, tmp);
787 		tmp = RREG32(mmSRBM_SOFT_RESET);
788 
789 		/* Wait a little for things to settle down */
790 		udelay(50);
791 	}
792 
793 	return 0;
794 }
795 
796 static int vce_v4_0_pre_soft_reset(void *handle)
797 {
798 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
799 
800 	if (!adev->vce.srbm_soft_reset)
801 		return 0;
802 
803 	mdelay(5);
804 
805 	return vce_v4_0_suspend(adev);
806 }
807 
808 
809 static int vce_v4_0_post_soft_reset(void *handle)
810 {
811 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
812 
813 	if (!adev->vce.srbm_soft_reset)
814 		return 0;
815 
816 	mdelay(5);
817 
818 	return vce_v4_0_resume(adev);
819 }
820 
821 static void vce_v4_0_override_vce_clock_gating(struct amdgpu_device *adev, bool override)
822 {
823 	u32 tmp, data;
824 
825 	tmp = data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_ARB_CTRL));
826 	if (override)
827 		data |= VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
828 	else
829 		data &= ~VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
830 
831 	if (tmp != data)
832 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_ARB_CTRL), data);
833 }
834 
835 static void vce_v4_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
836 					     bool gated)
837 {
838 	u32 data;
839 
840 	/* Set Override to disable Clock Gating */
841 	vce_v4_0_override_vce_clock_gating(adev, true);
842 
843 	/* This function enables MGCG which is controlled by firmware.
844 	   With the clocks in the gated state the core is still
845 	   accessible but the firmware will throttle the clocks on the
846 	   fly as necessary.
847 	*/
848 	if (gated) {
849 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B));
850 		data |= 0x1ff;
851 		data &= ~0xef0000;
852 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), data);
853 
854 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING));
855 		data |= 0x3ff000;
856 		data &= ~0xffc00000;
857 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), data);
858 
859 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2));
860 		data |= 0x2;
861 		data &= ~0x00010000;
862 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2), data);
863 
864 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING));
865 		data |= 0x37f;
866 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), data);
867 
868 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL));
869 		data |= VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
870 			VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
871 			VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
872 			0x8;
873 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL), data);
874 	} else {
875 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B));
876 		data &= ~0x80010;
877 		data |= 0xe70008;
878 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), data);
879 
880 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING));
881 		data |= 0xffc00000;
882 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), data);
883 
884 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2));
885 		data |= 0x10000;
886 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2), data);
887 
888 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING));
889 		data &= ~0xffc00000;
890 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), data);
891 
892 		data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL));
893 		data &= ~(VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
894 			  VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
895 			  VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
896 			  0x8);
897 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL), data);
898 	}
899 	vce_v4_0_override_vce_clock_gating(adev, false);
900 }
901 
902 static void vce_v4_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
903 {
904 	u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
905 
906 	if (enable)
907 		tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
908 	else
909 		tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
910 
911 	WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
912 }
913 
914 static int vce_v4_0_set_clockgating_state(void *handle,
915 					  enum amd_clockgating_state state)
916 {
917 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
918 	bool enable = (state == AMD_CG_STATE_GATE);
919 	int i;
920 
921 	if ((adev->asic_type == CHIP_POLARIS10) ||
922 		(adev->asic_type == CHIP_TONGA) ||
923 		(adev->asic_type == CHIP_FIJI))
924 		vce_v4_0_set_bypass_mode(adev, enable);
925 
926 	if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))
927 		return 0;
928 
929 	mutex_lock(&adev->grbm_idx_mutex);
930 	for (i = 0; i < 2; i++) {
931 		/* Program VCE Instance 0 or 1 if not harvested */
932 		if (adev->vce.harvest_config & (1 << i))
933 			continue;
934 
935 		WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, i);
936 
937 		if (enable) {
938 			/* initialize VCE_CLOCK_GATING_A: Clock ON/OFF delay */
939 			uint32_t data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A);
940 			data &= ~(0xf | 0xff0);
941 			data |= ((0x0 << 0) | (0x04 << 4));
942 			WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A, data);
943 
944 			/* initialize VCE_UENC_CLOCK_GATING: Clock ON/OFF delay */
945 			data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING);
946 			data &= ~(0xf | 0xff0);
947 			data |= ((0x0 << 0) | (0x04 << 4));
948 			WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING, data);
949 		}
950 
951 		vce_v4_0_set_vce_sw_clock_gating(adev, enable);
952 	}
953 
954 	WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
955 	mutex_unlock(&adev->grbm_idx_mutex);
956 
957 	return 0;
958 }
959 #endif
960 
961 static int vce_v4_0_set_powergating_state(void *handle,
962 					  enum amd_powergating_state state)
963 {
964 	/* This doesn't actually powergate the VCE block.
965 	 * That's done in the dpm code via the SMC.  This
966 	 * just re-inits the block as necessary.  The actual
967 	 * gating still happens in the dpm code.  We should
968 	 * revisit this when there is a cleaner line between
969 	 * the smc and the hw blocks
970 	 */
971 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
972 
973 	if (state == AMD_PG_STATE_GATE)
974 		return vce_v4_0_stop(adev);
975 	else
976 		return vce_v4_0_start(adev);
977 }
978 
979 static void vce_v4_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job,
980 					struct amdgpu_ib *ib, uint32_t flags)
981 {
982 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
983 
984 	amdgpu_ring_write(ring, VCE_CMD_IB_VM);
985 	amdgpu_ring_write(ring, vmid);
986 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
987 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
988 	amdgpu_ring_write(ring, ib->length_dw);
989 }
990 
991 static void vce_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
992 			u64 seq, unsigned flags)
993 {
994 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
995 
996 	amdgpu_ring_write(ring, VCE_CMD_FENCE);
997 	amdgpu_ring_write(ring, addr);
998 	amdgpu_ring_write(ring, upper_32_bits(addr));
999 	amdgpu_ring_write(ring, seq);
1000 	amdgpu_ring_write(ring, VCE_CMD_TRAP);
1001 }
1002 
1003 static void vce_v4_0_ring_insert_end(struct amdgpu_ring *ring)
1004 {
1005 	amdgpu_ring_write(ring, VCE_CMD_END);
1006 }
1007 
1008 static void vce_v4_0_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1009 				   uint32_t val, uint32_t mask)
1010 {
1011 	amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
1012 	amdgpu_ring_write(ring,	reg << 2);
1013 	amdgpu_ring_write(ring, mask);
1014 	amdgpu_ring_write(ring, val);
1015 }
1016 
1017 static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
1018 				   unsigned int vmid, uint64_t pd_addr)
1019 {
1020 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1021 
1022 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1023 
1024 	/* wait for reg writes */
1025 	vce_v4_0_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 +
1026 			       vmid * hub->ctx_addr_distance,
1027 			       lower_32_bits(pd_addr), 0xffffffff);
1028 }
1029 
1030 static void vce_v4_0_emit_wreg(struct amdgpu_ring *ring,
1031 			       uint32_t reg, uint32_t val)
1032 {
1033 	amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
1034 	amdgpu_ring_write(ring,	reg << 2);
1035 	amdgpu_ring_write(ring, val);
1036 }
1037 
1038 static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev,
1039 					struct amdgpu_irq_src *source,
1040 					unsigned type,
1041 					enum amdgpu_interrupt_state state)
1042 {
1043 	uint32_t val = 0;
1044 
1045 	if (!amdgpu_sriov_vf(adev)) {
1046 		if (state == AMDGPU_IRQ_STATE_ENABLE)
1047 			val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;
1048 
1049 		WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN), val,
1050 				~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
1051 	}
1052 	return 0;
1053 }
1054 
1055 static int vce_v4_0_process_interrupt(struct amdgpu_device *adev,
1056 				      struct amdgpu_irq_src *source,
1057 				      struct amdgpu_iv_entry *entry)
1058 {
1059 	DRM_DEBUG("IH: VCE\n");
1060 
1061 	switch (entry->src_data[0]) {
1062 	case 0:
1063 	case 1:
1064 	case 2:
1065 		amdgpu_fence_process(&adev->vce.ring[entry->src_data[0]]);
1066 		break;
1067 	default:
1068 		DRM_ERROR("Unhandled interrupt: %d %d\n",
1069 			  entry->src_id, entry->src_data[0]);
1070 		break;
1071 	}
1072 
1073 	return 0;
1074 }
1075 
1076 const struct amd_ip_funcs vce_v4_0_ip_funcs = {
1077 	.name = "vce_v4_0",
1078 	.early_init = vce_v4_0_early_init,
1079 	.late_init = NULL,
1080 	.sw_init = vce_v4_0_sw_init,
1081 	.sw_fini = vce_v4_0_sw_fini,
1082 	.hw_init = vce_v4_0_hw_init,
1083 	.hw_fini = vce_v4_0_hw_fini,
1084 	.suspend = vce_v4_0_suspend,
1085 	.resume = vce_v4_0_resume,
1086 	.is_idle = NULL /* vce_v4_0_is_idle */,
1087 	.wait_for_idle = NULL /* vce_v4_0_wait_for_idle */,
1088 	.check_soft_reset = NULL /* vce_v4_0_check_soft_reset */,
1089 	.pre_soft_reset = NULL /* vce_v4_0_pre_soft_reset */,
1090 	.soft_reset = NULL /* vce_v4_0_soft_reset */,
1091 	.post_soft_reset = NULL /* vce_v4_0_post_soft_reset */,
1092 	.set_clockgating_state = vce_v4_0_set_clockgating_state,
1093 	.set_powergating_state = vce_v4_0_set_powergating_state,
1094 };
1095 
1096 static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
1097 	.type = AMDGPU_RING_TYPE_VCE,
1098 	.align_mask = 0x3f,
1099 	.nop = VCE_CMD_NO_OP,
1100 	.support_64bit_ptrs = false,
1101 	.no_user_fence = true,
1102 	.vmhub = AMDGPU_MMHUB_0,
1103 	.get_rptr = vce_v4_0_ring_get_rptr,
1104 	.get_wptr = vce_v4_0_ring_get_wptr,
1105 	.set_wptr = vce_v4_0_ring_set_wptr,
1106 	.parse_cs = amdgpu_vce_ring_parse_cs_vm,
1107 	.emit_frame_size =
1108 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1109 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1110 		4 + /* vce_v4_0_emit_vm_flush */
1111 		5 + 5 + /* amdgpu_vce_ring_emit_fence x2 vm fence */
1112 		1, /* vce_v4_0_ring_insert_end */
1113 	.emit_ib_size = 5, /* vce_v4_0_ring_emit_ib */
1114 	.emit_ib = vce_v4_0_ring_emit_ib,
1115 	.emit_vm_flush = vce_v4_0_emit_vm_flush,
1116 	.emit_fence = vce_v4_0_ring_emit_fence,
1117 	.test_ring = amdgpu_vce_ring_test_ring,
1118 	.test_ib = amdgpu_vce_ring_test_ib,
1119 	.insert_nop = amdgpu_ring_insert_nop,
1120 	.insert_end = vce_v4_0_ring_insert_end,
1121 	.pad_ib = amdgpu_ring_generic_pad_ib,
1122 	.begin_use = amdgpu_vce_ring_begin_use,
1123 	.end_use = amdgpu_vce_ring_end_use,
1124 	.emit_wreg = vce_v4_0_emit_wreg,
1125 	.emit_reg_wait = vce_v4_0_emit_reg_wait,
1126 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1127 };
1128 
1129 static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev)
1130 {
1131 	int i;
1132 
1133 	for (i = 0; i < adev->vce.num_rings; i++) {
1134 		adev->vce.ring[i].funcs = &vce_v4_0_ring_vm_funcs;
1135 		adev->vce.ring[i].me = i;
1136 	}
1137 	DRM_INFO("VCE enabled in VM mode\n");
1138 }
1139 
1140 static const struct amdgpu_irq_src_funcs vce_v4_0_irq_funcs = {
1141 	.set = vce_v4_0_set_interrupt_state,
1142 	.process = vce_v4_0_process_interrupt,
1143 };
1144 
1145 static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev)
1146 {
1147 	adev->vce.irq.num_types = 1;
1148 	adev->vce.irq.funcs = &vce_v4_0_irq_funcs;
1149 };
1150 
1151 const struct amdgpu_ip_block_version vce_v4_0_ip_block =
1152 {
1153 	.type = AMD_IP_BLOCK_TYPE_VCE,
1154 	.major = 4,
1155 	.minor = 0,
1156 	.rev = 0,
1157 	.funcs = &vce_v4_0_ip_funcs,
1158 };
1159