1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
28 #include "amdgpu_cs.h"
29 #include "soc15.h"
30 #include "soc15d.h"
31 #include "soc15_hw_ip.h"
32 #include "vcn_v2_0.h"
33 #include "mmsch_v4_0.h"
34 #include "vcn_v4_0.h"
35
36 #include "vcn/vcn_4_0_0_offset.h"
37 #include "vcn/vcn_4_0_0_sh_mask.h"
38 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
39
40 #include <drm/drm_drv.h>
41
42 #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL
43 #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX
44 #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA
45 #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX
46
47 #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00
48 #define VCN1_VID_SOC_ADDRESS_3_0 0x48300
49
50 #define VCN_HARVEST_MMSCH 0
51
52 #define RDECODE_MSG_CREATE 0x00000000
53 #define RDECODE_MESSAGE_CREATE 0x00000001
54
55 static int amdgpu_ih_clientid_vcns[] = {
56 SOC15_IH_CLIENTID_VCN,
57 SOC15_IH_CLIENTID_VCN1
58 };
59
60 static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);
61 static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);
62 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
63 static int vcn_v4_0_set_powergating_state(void *handle,
64 enum amd_powergating_state state);
65 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev,
66 int inst_idx, struct dpg_pause_state *new_state);
67 static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
68 static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev);
69
70 /**
71 * vcn_v4_0_early_init - set function pointers and load microcode
72 *
73 * @handle: amdgpu_device pointer
74 *
75 * Set ring and irq function pointers
76 * Load microcode from filesystem
77 */
vcn_v4_0_early_init(void * handle)78 static int vcn_v4_0_early_init(void *handle)
79 {
80 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
81 int i;
82
83 if (amdgpu_sriov_vf(adev)) {
84 adev->vcn.harvest_config = VCN_HARVEST_MMSCH;
85 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
86 if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
87 adev->vcn.harvest_config |= 1 << i;
88 dev_info(adev->dev, "VCN%d is disabled by hypervisor\n", i);
89 }
90 }
91 }
92
93 /* re-use enc ring as unified ring */
94 adev->vcn.num_enc_rings = 1;
95
96 vcn_v4_0_set_unified_ring_funcs(adev);
97 vcn_v4_0_set_irq_funcs(adev);
98 vcn_v4_0_set_ras_funcs(adev);
99
100 return amdgpu_vcn_early_init(adev);
101 }
102
103 /**
104 * vcn_v4_0_sw_init - sw init for VCN block
105 *
106 * @handle: amdgpu_device pointer
107 *
108 * Load firmware and sw initialization
109 */
vcn_v4_0_sw_init(void * handle)110 static int vcn_v4_0_sw_init(void *handle)
111 {
112 struct amdgpu_ring *ring;
113 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
114 int i, r;
115
116 r = amdgpu_vcn_sw_init(adev);
117 if (r)
118 return r;
119
120 amdgpu_vcn_setup_ucode(adev);
121
122 r = amdgpu_vcn_resume(adev);
123 if (r)
124 return r;
125
126 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
127 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
128
129 if (adev->vcn.harvest_config & (1 << i))
130 continue;
131
132 /* Init instance 0 sched_score to 1, so it's scheduled after other instances */
133 if (i == 0)
134 atomic_set(&adev->vcn.inst[i].sched_score, 1);
135 else
136 atomic_set(&adev->vcn.inst[i].sched_score, 0);
137
138 /* VCN UNIFIED TRAP */
139 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
140 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
141 if (r)
142 return r;
143
144 /* VCN POISON TRAP */
145 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
146 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].ras_poison_irq);
147 if (r)
148 return r;
149
150 ring = &adev->vcn.inst[i].ring_enc[0];
151 ring->use_doorbell = true;
152 if (amdgpu_sriov_vf(adev))
153 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + i * (adev->vcn.num_enc_rings + 1) + 1;
154 else
155 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
156 ring->vm_hub = AMDGPU_MMHUB0(0);
157 sprintf(ring->name, "vcn_unified_%d", i);
158
159 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
160 AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
161 if (r)
162 return r;
163
164 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
165 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
166 fw_shared->sq.is_enabled = 1;
167
168 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);
169 fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?
170 AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;
171
172 if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 2)) {
173 fw_shared->present_flag_0 |= AMDGPU_FW_SHARED_FLAG_0_DRM_KEY_INJECT;
174 fw_shared->drm_key_wa.method =
175 AMDGPU_DRM_KEY_INJECT_WORKAROUND_VCNFW_ASD_HANDSHAKING;
176 }
177
178 if (amdgpu_sriov_vf(adev))
179 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
180
181 if (amdgpu_vcnfw_log)
182 amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
183 }
184
185 if (amdgpu_sriov_vf(adev)) {
186 r = amdgpu_virt_alloc_mm_table(adev);
187 if (r)
188 return r;
189 }
190
191 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
192 adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
193
194 r = amdgpu_vcn_ras_sw_init(adev);
195 if (r)
196 return r;
197
198 return 0;
199 }
200
201 /**
202 * vcn_v4_0_sw_fini - sw fini for VCN block
203 *
204 * @handle: amdgpu_device pointer
205 *
206 * VCN suspend and free up sw allocation
207 */
vcn_v4_0_sw_fini(void * handle)208 static int vcn_v4_0_sw_fini(void *handle)
209 {
210 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
211 int i, r, idx;
212
213 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
214 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
215 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
216
217 if (adev->vcn.harvest_config & (1 << i))
218 continue;
219
220 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
221 fw_shared->present_flag_0 = 0;
222 fw_shared->sq.is_enabled = 0;
223 }
224
225 drm_dev_exit(idx);
226 }
227
228 if (amdgpu_sriov_vf(adev))
229 amdgpu_virt_free_mm_table(adev);
230
231 r = amdgpu_vcn_suspend(adev);
232 if (r)
233 return r;
234
235 r = amdgpu_vcn_sw_fini(adev);
236
237 return r;
238 }
239
240 /**
241 * vcn_v4_0_hw_init - start and test VCN block
242 *
243 * @handle: amdgpu_device pointer
244 *
245 * Initialize the hardware, boot up the VCPU and do some testing
246 */
vcn_v4_0_hw_init(void * handle)247 static int vcn_v4_0_hw_init(void *handle)
248 {
249 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
250 struct amdgpu_ring *ring;
251 int i, r;
252
253 if (amdgpu_sriov_vf(adev)) {
254 r = vcn_v4_0_start_sriov(adev);
255 if (r)
256 goto done;
257
258 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
259 if (adev->vcn.harvest_config & (1 << i))
260 continue;
261
262 ring = &adev->vcn.inst[i].ring_enc[0];
263 ring->wptr = 0;
264 ring->wptr_old = 0;
265 vcn_v4_0_unified_ring_set_wptr(ring);
266 ring->sched.ready = true;
267
268 }
269 } else {
270 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
271 if (adev->vcn.harvest_config & (1 << i))
272 continue;
273
274 ring = &adev->vcn.inst[i].ring_enc[0];
275
276 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
277 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
278
279 r = amdgpu_ring_test_helper(ring);
280 if (r)
281 goto done;
282
283 }
284 }
285
286 done:
287 if (!r)
288 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
289 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
290
291 return r;
292 }
293
294 /**
295 * vcn_v4_0_hw_fini - stop the hardware block
296 *
297 * @handle: amdgpu_device pointer
298 *
299 * Stop the VCN block, mark ring as not ready any more
300 */
vcn_v4_0_hw_fini(void * handle)301 static int vcn_v4_0_hw_fini(void *handle)
302 {
303 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
304 int i;
305
306 cancel_delayed_work_sync(&adev->vcn.idle_work);
307
308 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
309 if (adev->vcn.harvest_config & (1 << i))
310 continue;
311 if (!amdgpu_sriov_vf(adev)) {
312 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
313 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
314 RREG32_SOC15(VCN, i, regUVD_STATUS))) {
315 vcn_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
316 }
317 }
318 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))
319 amdgpu_irq_put(adev, &adev->vcn.inst[i].ras_poison_irq, 0);
320 }
321
322 return 0;
323 }
324
325 /**
326 * vcn_v4_0_suspend - suspend VCN block
327 *
328 * @handle: amdgpu_device pointer
329 *
330 * HW fini and suspend VCN block
331 */
vcn_v4_0_suspend(void * handle)332 static int vcn_v4_0_suspend(void *handle)
333 {
334 int r;
335 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
336
337 r = vcn_v4_0_hw_fini(adev);
338 if (r)
339 return r;
340
341 r = amdgpu_vcn_suspend(adev);
342
343 return r;
344 }
345
346 /**
347 * vcn_v4_0_resume - resume VCN block
348 *
349 * @handle: amdgpu_device pointer
350 *
351 * Resume firmware and hw init VCN block
352 */
vcn_v4_0_resume(void * handle)353 static int vcn_v4_0_resume(void *handle)
354 {
355 int r;
356 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
357
358 r = amdgpu_vcn_resume(adev);
359 if (r)
360 return r;
361
362 r = vcn_v4_0_hw_init(adev);
363
364 return r;
365 }
366
367 /**
368 * vcn_v4_0_mc_resume - memory controller programming
369 *
370 * @adev: amdgpu_device pointer
371 * @inst: instance number
372 *
373 * Let the VCN memory controller know it's offsets
374 */
vcn_v4_0_mc_resume(struct amdgpu_device * adev,int inst)375 static void vcn_v4_0_mc_resume(struct amdgpu_device *adev, int inst)
376 {
377 uint32_t offset, size;
378 const struct common_firmware_header *hdr;
379
380 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
381 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
382
383 /* cache window 0: fw */
384 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
385 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
386 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
387 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
388 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
389 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
390 offset = 0;
391 } else {
392 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
393 lower_32_bits(adev->vcn.inst[inst].gpu_addr));
394 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
395 upper_32_bits(adev->vcn.inst[inst].gpu_addr));
396 offset = size;
397 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
398 }
399 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
400
401 /* cache window 1: stack */
402 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
403 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
404 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
405 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
406 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
407 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
408
409 /* cache window 2: context */
410 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
411 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
412 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
413 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
414 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
415 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
416
417 /* non-cache window */
418 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
419 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
420 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
421 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
422 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
423 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
424 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
425 }
426
427 /**
428 * vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode
429 *
430 * @adev: amdgpu_device pointer
431 * @inst_idx: instance number index
432 * @indirect: indirectly write sram
433 *
434 * Let the VCN memory controller know it's offsets with dpg mode
435 */
vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_device * adev,int inst_idx,bool indirect)436 static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
437 {
438 uint32_t offset, size;
439 const struct common_firmware_header *hdr;
440 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
441 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
442
443 /* cache window 0: fw */
444 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
445 if (!indirect) {
446 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
447 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
448 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
449 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
450 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
451 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
452 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
453 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
454 } else {
455 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
456 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
457 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
458 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
459 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
460 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
461 }
462 offset = 0;
463 } else {
464 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
465 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
466 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
467 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
468 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
469 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
470 offset = size;
471 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
472 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
473 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
474 }
475
476 if (!indirect)
477 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
478 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
479 else
480 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
481 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
482
483 /* cache window 1: stack */
484 if (!indirect) {
485 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
486 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
487 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
488 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
489 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
490 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
491 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
492 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
493 } else {
494 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
495 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
496 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
497 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
498 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
499 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
500 }
501 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
502 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
503
504 /* cache window 2: context */
505 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
506 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
507 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
508 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
509 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
510 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
511 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
512 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
513 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
514 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
515
516 /* non-cache window */
517 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
518 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
519 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
520 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
521 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
522 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
523 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
524 VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
525 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
526 VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
527 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
528
529 /* VCN global tiling registers */
530 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
531 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
532 }
533
534 /**
535 * vcn_v4_0_disable_static_power_gating - disable VCN static power gating
536 *
537 * @adev: amdgpu_device pointer
538 * @inst: instance number
539 *
540 * Disable static power gating for VCN block
541 */
vcn_v4_0_disable_static_power_gating(struct amdgpu_device * adev,int inst)542 static void vcn_v4_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
543 {
544 uint32_t data = 0;
545
546 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
547 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
548 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
549 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
550 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
551 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
552 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
553 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
554 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
555 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
556 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
557 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
558 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
559 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
560 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
561
562 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
563 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,
564 UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
565 } else {
566 uint32_t value;
567
568 value = (inst) ? 0x2200800 : 0;
569 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
570 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
571 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
572 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
573 | 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
574 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
575 | 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
576 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
577 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
578 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
579 | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
580 | 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
581 | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
582 | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
583
584 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
585 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value, 0x3F3FFFFF);
586 }
587
588 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
589 data &= ~0x103;
590 if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
591 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
592 UVD_POWER_STATUS__UVD_PG_EN_MASK;
593
594 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
595
596 return;
597 }
598
599 /**
600 * vcn_v4_0_enable_static_power_gating - enable VCN static power gating
601 *
602 * @adev: amdgpu_device pointer
603 * @inst: instance number
604 *
605 * Enable static power gating for VCN block
606 */
vcn_v4_0_enable_static_power_gating(struct amdgpu_device * adev,int inst)607 static void vcn_v4_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
608 {
609 uint32_t data;
610
611 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
612 /* Before power off, this indicator has to be turned on */
613 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
614 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
615 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
616 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
617
618 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
619 | 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
620 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
621 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
622 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
623 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
624 | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
625 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
626 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
627 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
628 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
629 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
630 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
631 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
632 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
633
634 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
635 | 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT
636 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
637 | 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT
638 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
639 | 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT
640 | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
641 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
642 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
643 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
644 | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
645 | 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT
646 | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
647 | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
648 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
649 }
650
651 return;
652 }
653
654 /**
655 * vcn_v4_0_disable_clock_gating - disable VCN clock gating
656 *
657 * @adev: amdgpu_device pointer
658 * @inst: instance number
659 *
660 * Disable clock gating for VCN block
661 */
vcn_v4_0_disable_clock_gating(struct amdgpu_device * adev,int inst)662 static void vcn_v4_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
663 {
664 uint32_t data;
665
666 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
667 return;
668
669 /* VCN disable CGC */
670 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
671 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
672 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
673 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
674 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
675
676 data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
677 data &= ~(UVD_CGC_GATE__SYS_MASK
678 | UVD_CGC_GATE__UDEC_MASK
679 | UVD_CGC_GATE__MPEG2_MASK
680 | UVD_CGC_GATE__REGS_MASK
681 | UVD_CGC_GATE__RBC_MASK
682 | UVD_CGC_GATE__LMI_MC_MASK
683 | UVD_CGC_GATE__LMI_UMC_MASK
684 | UVD_CGC_GATE__IDCT_MASK
685 | UVD_CGC_GATE__MPRD_MASK
686 | UVD_CGC_GATE__MPC_MASK
687 | UVD_CGC_GATE__LBSI_MASK
688 | UVD_CGC_GATE__LRBBM_MASK
689 | UVD_CGC_GATE__UDEC_RE_MASK
690 | UVD_CGC_GATE__UDEC_CM_MASK
691 | UVD_CGC_GATE__UDEC_IT_MASK
692 | UVD_CGC_GATE__UDEC_DB_MASK
693 | UVD_CGC_GATE__UDEC_MP_MASK
694 | UVD_CGC_GATE__WCB_MASK
695 | UVD_CGC_GATE__VCPU_MASK
696 | UVD_CGC_GATE__MMSCH_MASK);
697
698 WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
699 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF);
700
701 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
702 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
703 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
704 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
705 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
706 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
707 | UVD_CGC_CTRL__SYS_MODE_MASK
708 | UVD_CGC_CTRL__UDEC_MODE_MASK
709 | UVD_CGC_CTRL__MPEG2_MODE_MASK
710 | UVD_CGC_CTRL__REGS_MODE_MASK
711 | UVD_CGC_CTRL__RBC_MODE_MASK
712 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
713 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
714 | UVD_CGC_CTRL__IDCT_MODE_MASK
715 | UVD_CGC_CTRL__MPRD_MODE_MASK
716 | UVD_CGC_CTRL__MPC_MODE_MASK
717 | UVD_CGC_CTRL__LBSI_MODE_MASK
718 | UVD_CGC_CTRL__LRBBM_MODE_MASK
719 | UVD_CGC_CTRL__WCB_MODE_MASK
720 | UVD_CGC_CTRL__VCPU_MODE_MASK
721 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
722 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
723
724 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
725 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
726 | UVD_SUVD_CGC_GATE__SIT_MASK
727 | UVD_SUVD_CGC_GATE__SMP_MASK
728 | UVD_SUVD_CGC_GATE__SCM_MASK
729 | UVD_SUVD_CGC_GATE__SDB_MASK
730 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
731 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
732 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
733 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
734 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
735 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
736 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
737 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
738 | UVD_SUVD_CGC_GATE__SCLR_MASK
739 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
740 | UVD_SUVD_CGC_GATE__ENT_MASK
741 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
742 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
743 | UVD_SUVD_CGC_GATE__SITE_MASK
744 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
745 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
746 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
747 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
748 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
749 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
750
751 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
752 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
753 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
754 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
755 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
756 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
757 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
758 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
759 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
760 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
761 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
762 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
763 }
764
765 /**
766 * vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
767 *
768 * @adev: amdgpu_device pointer
769 * @sram_sel: sram select
770 * @inst_idx: instance number index
771 * @indirect: indirectly write sram
772 *
773 * Disable clock gating for VCN block with dpg mode
774 */
vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_device * adev,uint8_t sram_sel,int inst_idx,uint8_t indirect)775 static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
776 int inst_idx, uint8_t indirect)
777 {
778 uint32_t reg_data = 0;
779
780 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
781 return;
782
783 /* enable sw clock gating control */
784 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
785 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
786 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
787 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
788 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
789 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
790 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
791 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
792 UVD_CGC_CTRL__SYS_MODE_MASK |
793 UVD_CGC_CTRL__UDEC_MODE_MASK |
794 UVD_CGC_CTRL__MPEG2_MODE_MASK |
795 UVD_CGC_CTRL__REGS_MODE_MASK |
796 UVD_CGC_CTRL__RBC_MODE_MASK |
797 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
798 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
799 UVD_CGC_CTRL__IDCT_MODE_MASK |
800 UVD_CGC_CTRL__MPRD_MODE_MASK |
801 UVD_CGC_CTRL__MPC_MODE_MASK |
802 UVD_CGC_CTRL__LBSI_MODE_MASK |
803 UVD_CGC_CTRL__LRBBM_MODE_MASK |
804 UVD_CGC_CTRL__WCB_MODE_MASK |
805 UVD_CGC_CTRL__VCPU_MODE_MASK);
806 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
807 VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
808
809 /* turn off clock gating */
810 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
811 VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
812
813 /* turn on SUVD clock gating */
814 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
815 VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
816
817 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
818 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
819 VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
820 }
821
822 /**
823 * vcn_v4_0_enable_clock_gating - enable VCN clock gating
824 *
825 * @adev: amdgpu_device pointer
826 * @inst: instance number
827 *
828 * Enable clock gating for VCN block
829 */
vcn_v4_0_enable_clock_gating(struct amdgpu_device * adev,int inst)830 static void vcn_v4_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
831 {
832 uint32_t data;
833
834 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
835 return;
836
837 /* enable VCN CGC */
838 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
839 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
840 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
841 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
842 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
843
844 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
845 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
846 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
847 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
848 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
849 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
850 | UVD_CGC_CTRL__SYS_MODE_MASK
851 | UVD_CGC_CTRL__UDEC_MODE_MASK
852 | UVD_CGC_CTRL__MPEG2_MODE_MASK
853 | UVD_CGC_CTRL__REGS_MODE_MASK
854 | UVD_CGC_CTRL__RBC_MODE_MASK
855 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
856 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
857 | UVD_CGC_CTRL__IDCT_MODE_MASK
858 | UVD_CGC_CTRL__MPRD_MODE_MASK
859 | UVD_CGC_CTRL__MPC_MODE_MASK
860 | UVD_CGC_CTRL__LBSI_MODE_MASK
861 | UVD_CGC_CTRL__LRBBM_MODE_MASK
862 | UVD_CGC_CTRL__WCB_MODE_MASK
863 | UVD_CGC_CTRL__VCPU_MODE_MASK
864 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
865 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
866
867 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
868 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
869 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
870 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
871 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
872 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
873 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
874 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
875 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
876 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
877 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
878 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
879
880 return;
881 }
882
vcn_v4_0_enable_ras(struct amdgpu_device * adev,int inst_idx,bool indirect)883 static void vcn_v4_0_enable_ras(struct amdgpu_device *adev, int inst_idx,
884 bool indirect)
885 {
886 uint32_t tmp;
887
888 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))
889 return;
890
891 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |
892 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |
893 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |
894 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;
895 WREG32_SOC15_DPG_MODE(inst_idx,
896 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL),
897 tmp, 0, indirect);
898
899 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
900 WREG32_SOC15_DPG_MODE(inst_idx,
901 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN),
902 tmp, 0, indirect);
903 }
904
905 /**
906 * vcn_v4_0_start_dpg_mode - VCN start with dpg mode
907 *
908 * @adev: amdgpu_device pointer
909 * @inst_idx: instance number index
910 * @indirect: indirectly write sram
911 *
912 * Start VCN block with dpg mode
913 */
vcn_v4_0_start_dpg_mode(struct amdgpu_device * adev,int inst_idx,bool indirect)914 static int vcn_v4_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
915 {
916 volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
917 struct amdgpu_ring *ring;
918 uint32_t tmp;
919
920 /* disable register anti-hang mechanism */
921 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
922 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
923 /* enable dynamic power gating mode */
924 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
925 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
926 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
927 WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
928
929 if (indirect)
930 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
931
932 /* enable clock gating */
933 vcn_v4_0_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
934
935 /* enable VCPU clock */
936 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
937 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
938 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
939 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
940
941 /* disable master interupt */
942 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
943 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
944
945 /* setup regUVD_LMI_CTRL */
946 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
947 UVD_LMI_CTRL__REQ_MODE_MASK |
948 UVD_LMI_CTRL__CRC_RESET_MASK |
949 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
950 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
951 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
952 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
953 0x00100000L);
954 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
955 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
956
957 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
958 VCN, inst_idx, regUVD_MPC_CNTL),
959 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
960
961 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
962 VCN, inst_idx, regUVD_MPC_SET_MUXA0),
963 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
964 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
965 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
966 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
967
968 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
969 VCN, inst_idx, regUVD_MPC_SET_MUXB0),
970 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
971 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
972 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
973 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
974
975 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
976 VCN, inst_idx, regUVD_MPC_SET_MUX),
977 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
978 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
979 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
980
981 vcn_v4_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
982
983 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
984 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
985 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
986 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
987
988 /* enable LMI MC and UMC channels */
989 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
990 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
991 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
992
993 vcn_v4_0_enable_ras(adev, inst_idx, indirect);
994
995 /* enable master interrupt */
996 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
997 VCN, inst_idx, regUVD_MASTINT_EN),
998 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
999
1000
1001 if (indirect)
1002 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
1003
1004 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1005
1006 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
1007 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1008 WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
1009
1010 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1011 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1012 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1013 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1014 WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
1015 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
1016
1017 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
1018 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
1019 ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1020
1021 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1022 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1023 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1024 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1025
1026 WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
1027 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1028 VCN_RB1_DB_CTRL__EN_MASK);
1029
1030 return 0;
1031 }
1032
1033
1034 /**
1035 * vcn_v4_0_start - VCN start
1036 *
1037 * @adev: amdgpu_device pointer
1038 *
1039 * Start VCN block
1040 */
vcn_v4_0_start(struct amdgpu_device * adev)1041 static int vcn_v4_0_start(struct amdgpu_device *adev)
1042 {
1043 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1044 struct amdgpu_ring *ring;
1045 uint32_t tmp;
1046 int i, j, k, r;
1047
1048 if (adev->pm.dpm_enabled)
1049 amdgpu_dpm_enable_uvd(adev, true);
1050
1051 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1052 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1053
1054 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1055 r = vcn_v4_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1056 continue;
1057 }
1058
1059 /* disable VCN power gating */
1060 vcn_v4_0_disable_static_power_gating(adev, i);
1061
1062 /* set VCN status busy */
1063 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1064 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1065
1066 /*SW clock gating */
1067 vcn_v4_0_disable_clock_gating(adev, i);
1068
1069 /* enable VCPU clock */
1070 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1071 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1072
1073 /* disable master interrupt */
1074 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
1075 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1076
1077 /* enable LMI MC and UMC channels */
1078 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
1079 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1080
1081 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1082 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1083 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1084 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1085
1086 /* setup regUVD_LMI_CTRL */
1087 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1088 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1089 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1090 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1091 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1092 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1093
1094 /* setup regUVD_MPC_CNTL */
1095 tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1096 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1097 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1098 WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1099
1100 /* setup UVD_MPC_SET_MUXA0 */
1101 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1102 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1103 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1104 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1105 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1106
1107 /* setup UVD_MPC_SET_MUXB0 */
1108 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1109 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1110 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1111 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1112 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1113
1114 /* setup UVD_MPC_SET_MUX */
1115 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1116 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1117 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1118 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1119
1120 vcn_v4_0_mc_resume(adev, i);
1121
1122 /* VCN global tiling registers */
1123 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1124 adev->gfx.config.gb_addr_config);
1125
1126 /* unblock VCPU register access */
1127 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1128 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1129
1130 /* release VCPU reset to boot */
1131 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1132 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1133
1134 for (j = 0; j < 10; ++j) {
1135 uint32_t status;
1136
1137 for (k = 0; k < 100; ++k) {
1138 status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1139 if (status & 2)
1140 break;
1141 mdelay(10);
1142 if (amdgpu_emu_mode == 1)
1143 msleep(1);
1144 }
1145
1146 if (amdgpu_emu_mode == 1) {
1147 r = -1;
1148 if (status & 2) {
1149 r = 0;
1150 break;
1151 }
1152 } else {
1153 r = 0;
1154 if (status & 2)
1155 break;
1156
1157 dev_err(adev->dev, "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
1158 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1159 UVD_VCPU_CNTL__BLK_RST_MASK,
1160 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1161 mdelay(10);
1162 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1163 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1164
1165 mdelay(10);
1166 r = -1;
1167 }
1168 }
1169
1170 if (r) {
1171 dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1172 return r;
1173 }
1174
1175 /* enable master interrupt */
1176 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1177 UVD_MASTINT_EN__VCPU_EN_MASK,
1178 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1179
1180 /* clear the busy bit of VCN_STATUS */
1181 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1182 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1183
1184 ring = &adev->vcn.inst[i].ring_enc[0];
1185 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1186 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1187 VCN_RB1_DB_CTRL__EN_MASK);
1188
1189 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1190 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1191 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1192
1193 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1194 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1195 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1196 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1197 WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1198 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
1199
1200 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1201 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1202 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1203
1204 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1205 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1206 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1207 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1208 }
1209
1210 return 0;
1211 }
1212
vcn_v4_0_start_sriov(struct amdgpu_device * adev)1213 static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)
1214 {
1215 int i;
1216 struct amdgpu_ring *ring_enc;
1217 uint64_t cache_addr;
1218 uint64_t rb_enc_addr;
1219 uint64_t ctx_addr;
1220 uint32_t param, resp, expected;
1221 uint32_t offset, cache_size;
1222 uint32_t tmp, timeout;
1223
1224 struct amdgpu_mm_table *table = &adev->virt.mm_table;
1225 uint32_t *table_loc;
1226 uint32_t table_size;
1227 uint32_t size, size_dw;
1228 uint32_t init_status;
1229 uint32_t enabled_vcn;
1230
1231 struct mmsch_v4_0_cmd_direct_write
1232 direct_wt = { {0} };
1233 struct mmsch_v4_0_cmd_direct_read_modify_write
1234 direct_rd_mod_wt = { {0} };
1235 struct mmsch_v4_0_cmd_end end = { {0} };
1236 struct mmsch_v4_0_init_header header;
1237
1238 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1239 volatile struct amdgpu_fw_shared_rb_setup *rb_setup;
1240
1241 direct_wt.cmd_header.command_type =
1242 MMSCH_COMMAND__DIRECT_REG_WRITE;
1243 direct_rd_mod_wt.cmd_header.command_type =
1244 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1245 end.cmd_header.command_type =
1246 MMSCH_COMMAND__END;
1247
1248 header.version = MMSCH_VERSION;
1249 header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
1250 for (i = 0; i < MMSCH_V4_0_VCN_INSTANCES; i++) {
1251 header.inst[i].init_status = 0;
1252 header.inst[i].table_offset = 0;
1253 header.inst[i].table_size = 0;
1254 }
1255
1256 table_loc = (uint32_t *)table->cpu_addr;
1257 table_loc += header.total_size;
1258 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1259 if (adev->vcn.harvest_config & (1 << i))
1260 continue;
1261
1262 table_size = 0;
1263
1264 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1265 regUVD_STATUS),
1266 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1267
1268 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1269
1270 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1271 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1272 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1273 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1274 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1275 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1276 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1277 offset = 0;
1278 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1279 regUVD_VCPU_CACHE_OFFSET0),
1280 0);
1281 } else {
1282 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1283 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1284 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1285 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1286 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1287 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1288 offset = cache_size;
1289 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1290 regUVD_VCPU_CACHE_OFFSET0),
1291 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1292 }
1293
1294 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1295 regUVD_VCPU_CACHE_SIZE0),
1296 cache_size);
1297
1298 cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1299 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1300 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1301 lower_32_bits(cache_addr));
1302 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1303 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1304 upper_32_bits(cache_addr));
1305 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1306 regUVD_VCPU_CACHE_OFFSET1),
1307 0);
1308 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1309 regUVD_VCPU_CACHE_SIZE1),
1310 AMDGPU_VCN_STACK_SIZE);
1311
1312 cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1313 AMDGPU_VCN_STACK_SIZE;
1314 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1315 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1316 lower_32_bits(cache_addr));
1317 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1318 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1319 upper_32_bits(cache_addr));
1320 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1321 regUVD_VCPU_CACHE_OFFSET2),
1322 0);
1323 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1324 regUVD_VCPU_CACHE_SIZE2),
1325 AMDGPU_VCN_CONTEXT_SIZE);
1326
1327 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1328 rb_setup = &fw_shared->rb_setup;
1329
1330 ring_enc = &adev->vcn.inst[i].ring_enc[0];
1331 ring_enc->wptr = 0;
1332 rb_enc_addr = ring_enc->gpu_addr;
1333
1334 rb_setup->is_rb_enabled_flags |= RB_ENABLED;
1335 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
1336 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
1337 rb_setup->rb_size = ring_enc->ring_size / 4;
1338 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1339
1340 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1341 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
1342 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1343 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1344 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
1345 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1346 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1347 regUVD_VCPU_NONCACHE_SIZE0),
1348 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
1349
1350 /* add end packet */
1351 MMSCH_V4_0_INSERT_END();
1352
1353 /* refine header */
1354 header.inst[i].init_status = 0;
1355 header.inst[i].table_offset = header.total_size;
1356 header.inst[i].table_size = table_size;
1357 header.total_size += table_size;
1358 }
1359
1360 /* Update init table header in memory */
1361 size = sizeof(struct mmsch_v4_0_init_header);
1362 table_loc = (uint32_t *)table->cpu_addr;
1363 memcpy((void *)table_loc, &header, size);
1364
1365 /* message MMSCH (in VCN[0]) to initialize this client
1366 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1367 * of memory descriptor location
1368 */
1369 ctx_addr = table->gpu_addr;
1370 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1371 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1372
1373 /* 2, update vmid of descriptor */
1374 tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
1375 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1376 /* use domain0 for MM scheduler */
1377 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1378 WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
1379
1380 /* 3, notify mmsch about the size of this descriptor */
1381 size = header.total_size;
1382 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
1383
1384 /* 4, set resp to zero */
1385 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
1386
1387 /* 5, kick off the initialization and wait until
1388 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1389 */
1390 param = 0x00000001;
1391 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
1392 tmp = 0;
1393 timeout = 1000;
1394 resp = 0;
1395 expected = MMSCH_VF_MAILBOX_RESP__OK;
1396 while (resp != expected) {
1397 resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
1398 if (resp != 0)
1399 break;
1400
1401 udelay(10);
1402 tmp = tmp + 10;
1403 if (tmp >= timeout) {
1404 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1405 " waiting for regMMSCH_VF_MAILBOX_RESP "\
1406 "(expected=0x%08x, readback=0x%08x)\n",
1407 tmp, expected, resp);
1408 return -EBUSY;
1409 }
1410 }
1411 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;
1412 init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;
1413 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
1414 && init_status != MMSCH_VF_ENGINE_STATUS__PASS)
1415 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
1416 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
1417
1418 return 0;
1419 }
1420
1421 /**
1422 * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
1423 *
1424 * @adev: amdgpu_device pointer
1425 * @inst_idx: instance number index
1426 *
1427 * Stop VCN block with dpg mode
1428 */
vcn_v4_0_stop_dpg_mode(struct amdgpu_device * adev,int inst_idx)1429 static void vcn_v4_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1430 {
1431 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1432 uint32_t tmp;
1433
1434 vcn_v4_0_pause_dpg_mode(adev, inst_idx, &state);
1435 /* Wait for power status to be 1 */
1436 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1437 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1438
1439 /* wait for read ptr to be equal to write ptr */
1440 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1441 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1442
1443 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1444 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1445
1446 /* disable dynamic power gating mode */
1447 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1448 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1449 }
1450
1451 /**
1452 * vcn_v4_0_stop - VCN stop
1453 *
1454 * @adev: amdgpu_device pointer
1455 *
1456 * Stop VCN block
1457 */
vcn_v4_0_stop(struct amdgpu_device * adev)1458 static int vcn_v4_0_stop(struct amdgpu_device *adev)
1459 {
1460 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1461 uint32_t tmp;
1462 int i, r = 0;
1463
1464 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1465 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1466 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1467
1468 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1469 vcn_v4_0_stop_dpg_mode(adev, i);
1470 continue;
1471 }
1472
1473 /* wait for vcn idle */
1474 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1475 if (r)
1476 return r;
1477
1478 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1479 UVD_LMI_STATUS__READ_CLEAN_MASK |
1480 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1481 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1482 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1483 if (r)
1484 return r;
1485
1486 /* disable LMI UMC channel */
1487 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1488 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1489 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1490 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1491 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1492 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1493 if (r)
1494 return r;
1495
1496 /* block VCPU register access */
1497 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1498 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1499 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1500
1501 /* reset VCPU */
1502 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1503 UVD_VCPU_CNTL__BLK_RST_MASK,
1504 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1505
1506 /* disable VCPU clock */
1507 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1508 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1509
1510 /* apply soft reset */
1511 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1512 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1513 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1514 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1515 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1516 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1517
1518 /* clear status */
1519 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1520
1521 /* apply HW clock gating */
1522 vcn_v4_0_enable_clock_gating(adev, i);
1523
1524 /* enable VCN power gating */
1525 vcn_v4_0_enable_static_power_gating(adev, i);
1526 }
1527
1528 if (adev->pm.dpm_enabled)
1529 amdgpu_dpm_enable_uvd(adev, false);
1530
1531 return 0;
1532 }
1533
1534 /**
1535 * vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode
1536 *
1537 * @adev: amdgpu_device pointer
1538 * @inst_idx: instance number index
1539 * @new_state: pause state
1540 *
1541 * Pause dpg mode for VCN block
1542 */
vcn_v4_0_pause_dpg_mode(struct amdgpu_device * adev,int inst_idx,struct dpg_pause_state * new_state)1543 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
1544 struct dpg_pause_state *new_state)
1545 {
1546 uint32_t reg_data = 0;
1547 int ret_code;
1548
1549 /* pause/unpause if state is changed */
1550 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1551 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1552 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1553 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1554 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1555
1556 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1557 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1558 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1559
1560 if (!ret_code) {
1561 /* pause DPG */
1562 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1563 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1564
1565 /* wait for ACK */
1566 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1567 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1568 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1569
1570 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1571 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1572 }
1573 } else {
1574 /* unpause dpg, no need to wait */
1575 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1576 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1577 }
1578 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1579 }
1580
1581 return 0;
1582 }
1583
1584 /**
1585 * vcn_v4_0_unified_ring_get_rptr - get unified read pointer
1586 *
1587 * @ring: amdgpu_ring pointer
1588 *
1589 * Returns the current hardware unified read pointer
1590 */
vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring * ring)1591 static uint64_t vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
1592 {
1593 struct amdgpu_device *adev = ring->adev;
1594
1595 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1596 DRM_ERROR("wrong ring id is identified in %s", __func__);
1597
1598 return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1599 }
1600
1601 /**
1602 * vcn_v4_0_unified_ring_get_wptr - get unified write pointer
1603 *
1604 * @ring: amdgpu_ring pointer
1605 *
1606 * Returns the current hardware unified write pointer
1607 */
vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring * ring)1608 static uint64_t vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
1609 {
1610 struct amdgpu_device *adev = ring->adev;
1611
1612 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1613 DRM_ERROR("wrong ring id is identified in %s", __func__);
1614
1615 if (ring->use_doorbell)
1616 return *ring->wptr_cpu_addr;
1617 else
1618 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1619 }
1620
1621 /**
1622 * vcn_v4_0_unified_ring_set_wptr - set enc write pointer
1623 *
1624 * @ring: amdgpu_ring pointer
1625 *
1626 * Commits the enc write pointer to the hardware
1627 */
vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring * ring)1628 static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
1629 {
1630 struct amdgpu_device *adev = ring->adev;
1631
1632 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1633 DRM_ERROR("wrong ring id is identified in %s", __func__);
1634
1635 if (ring->use_doorbell) {
1636 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1637 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1638 } else {
1639 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1640 }
1641 }
1642
vcn_v4_0_limit_sched(struct amdgpu_cs_parser * p,struct amdgpu_job * job)1643 static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p,
1644 struct amdgpu_job *job)
1645 {
1646 struct drm_gpu_scheduler **scheds;
1647
1648 /* The create msg must be in the first IB submitted */
1649 if (atomic_read(&job->base.entity->fence_seq))
1650 return -EINVAL;
1651
1652 /* if VCN0 is harvested, we can't support AV1 */
1653 if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1654 return -EINVAL;
1655
1656 scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
1657 [AMDGPU_RING_PRIO_0].sched;
1658 drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
1659 return 0;
1660 }
1661
vcn_v4_0_dec_msg(struct amdgpu_cs_parser * p,struct amdgpu_job * job,uint64_t addr)1662 static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1663 uint64_t addr)
1664 {
1665 struct ttm_operation_ctx ctx = { false, false };
1666 struct amdgpu_bo_va_mapping *map;
1667 uint32_t *msg, num_buffers;
1668 struct amdgpu_bo *bo;
1669 uint64_t start, end;
1670 unsigned int i;
1671 void *ptr;
1672 int r;
1673
1674 addr &= AMDGPU_GMC_HOLE_MASK;
1675 r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1676 if (r) {
1677 DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);
1678 return r;
1679 }
1680
1681 start = map->start * AMDGPU_GPU_PAGE_SIZE;
1682 end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1683 if (addr & 0x7) {
1684 DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1685 return -EINVAL;
1686 }
1687
1688 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1689 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1690 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1691 if (r) {
1692 DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1693 return r;
1694 }
1695
1696 r = amdgpu_bo_kmap(bo, &ptr);
1697 if (r) {
1698 DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1699 return r;
1700 }
1701
1702 msg = ptr + addr - start;
1703
1704 /* Check length */
1705 if (msg[1] > end - addr) {
1706 r = -EINVAL;
1707 goto out;
1708 }
1709
1710 if (msg[3] != RDECODE_MSG_CREATE)
1711 goto out;
1712
1713 num_buffers = msg[2];
1714 for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1715 uint32_t offset, size, *create;
1716
1717 if (msg[0] != RDECODE_MESSAGE_CREATE)
1718 continue;
1719
1720 offset = msg[1];
1721 size = msg[2];
1722
1723 if (offset + size > end) {
1724 r = -EINVAL;
1725 goto out;
1726 }
1727
1728 create = ptr + addr + offset - start;
1729
1730 /* H264, HEVC and VP9 can run on any instance */
1731 if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1732 continue;
1733
1734 r = vcn_v4_0_limit_sched(p, job);
1735 if (r)
1736 goto out;
1737 }
1738
1739 out:
1740 amdgpu_bo_kunmap(bo);
1741 return r;
1742 }
1743
1744 #define RADEON_VCN_ENGINE_TYPE_ENCODE (0x00000002)
1745 #define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003)
1746
1747 #define RADEON_VCN_ENGINE_INFO (0x30000001)
1748 #define RADEON_VCN_ENGINE_INFO_MAX_OFFSET 16
1749
1750 #define RENCODE_ENCODE_STANDARD_AV1 2
1751 #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003
1752 #define RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET 64
1753
1754 /* return the offset in ib if id is found, -1 otherwise
1755 * to speed up the searching we only search upto max_offset
1756 */
vcn_v4_0_enc_find_ib_param(struct amdgpu_ib * ib,uint32_t id,int max_offset)1757 static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int max_offset)
1758 {
1759 int i;
1760
1761 for (i = 0; i < ib->length_dw && i < max_offset && ib->ptr[i] >= 8; i += ib->ptr[i]/4) {
1762 if (ib->ptr[i + 1] == id)
1763 return i;
1764 }
1765 return -1;
1766 }
1767
vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser * p,struct amdgpu_job * job,struct amdgpu_ib * ib)1768 static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1769 struct amdgpu_job *job,
1770 struct amdgpu_ib *ib)
1771 {
1772 struct amdgpu_ring *ring = amdgpu_job_ring(job);
1773 struct amdgpu_vcn_decode_buffer *decode_buffer;
1774 uint64_t addr;
1775 uint32_t val;
1776 int idx;
1777
1778 /* The first instance can decode anything */
1779 if (!ring->me)
1780 return 0;
1781
1782 /* RADEON_VCN_ENGINE_INFO is at the top of ib block */
1783 idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO,
1784 RADEON_VCN_ENGINE_INFO_MAX_OFFSET);
1785 if (idx < 0) /* engine info is missing */
1786 return 0;
1787
1788 val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */
1789 if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
1790 decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6];
1791
1792 if (!(decode_buffer->valid_buf_flag & 0x1))
1793 return 0;
1794
1795 addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
1796 decode_buffer->msg_buffer_address_lo;
1797 return vcn_v4_0_dec_msg(p, job, addr);
1798 } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) {
1799 idx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT,
1800 RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET);
1801 if (idx >= 0 && ib->ptr[idx + 2] == RENCODE_ENCODE_STANDARD_AV1)
1802 return vcn_v4_0_limit_sched(p, job);
1803 }
1804 return 0;
1805 }
1806
1807 static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
1808 .type = AMDGPU_RING_TYPE_VCN_ENC,
1809 .align_mask = 0x3f,
1810 .nop = VCN_ENC_CMD_NO_OP,
1811 .get_rptr = vcn_v4_0_unified_ring_get_rptr,
1812 .get_wptr = vcn_v4_0_unified_ring_get_wptr,
1813 .set_wptr = vcn_v4_0_unified_ring_set_wptr,
1814 .patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,
1815 .emit_frame_size =
1816 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1817 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1818 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1819 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1820 1, /* vcn_v2_0_enc_ring_insert_end */
1821 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1822 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1823 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1824 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1825 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1826 .test_ib = amdgpu_vcn_unified_ring_test_ib,
1827 .insert_nop = amdgpu_ring_insert_nop,
1828 .insert_end = vcn_v2_0_enc_ring_insert_end,
1829 .pad_ib = amdgpu_ring_generic_pad_ib,
1830 .begin_use = amdgpu_vcn_ring_begin_use,
1831 .end_use = amdgpu_vcn_ring_end_use,
1832 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1833 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1834 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1835 };
1836
1837 /**
1838 * vcn_v4_0_set_unified_ring_funcs - set unified ring functions
1839 *
1840 * @adev: amdgpu_device pointer
1841 *
1842 * Set unified ring functions
1843 */
vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device * adev)1844 static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
1845 {
1846 int i;
1847
1848 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1849 if (adev->vcn.harvest_config & (1 << i))
1850 continue;
1851
1852 if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 2))
1853 vcn_v4_0_unified_ring_vm_funcs.secure_submission_supported = true;
1854
1855 adev->vcn.inst[i].ring_enc[0].funcs =
1856 (const struct amdgpu_ring_funcs *)&vcn_v4_0_unified_ring_vm_funcs;
1857 adev->vcn.inst[i].ring_enc[0].me = i;
1858
1859 DRM_INFO("VCN(%d) encode/decode are enabled in VM mode\n", i);
1860 }
1861 }
1862
1863 /**
1864 * vcn_v4_0_is_idle - check VCN block is idle
1865 *
1866 * @handle: amdgpu_device pointer
1867 *
1868 * Check whether VCN block is idle
1869 */
vcn_v4_0_is_idle(void * handle)1870 static bool vcn_v4_0_is_idle(void *handle)
1871 {
1872 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1873 int i, ret = 1;
1874
1875 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1876 if (adev->vcn.harvest_config & (1 << i))
1877 continue;
1878
1879 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1880 }
1881
1882 return ret;
1883 }
1884
1885 /**
1886 * vcn_v4_0_wait_for_idle - wait for VCN block idle
1887 *
1888 * @handle: amdgpu_device pointer
1889 *
1890 * Wait for VCN block idle
1891 */
vcn_v4_0_wait_for_idle(void * handle)1892 static int vcn_v4_0_wait_for_idle(void *handle)
1893 {
1894 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1895 int i, ret = 0;
1896
1897 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1898 if (adev->vcn.harvest_config & (1 << i))
1899 continue;
1900
1901 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1902 UVD_STATUS__IDLE);
1903 if (ret)
1904 return ret;
1905 }
1906
1907 return ret;
1908 }
1909
1910 /**
1911 * vcn_v4_0_set_clockgating_state - set VCN block clockgating state
1912 *
1913 * @handle: amdgpu_device pointer
1914 * @state: clock gating state
1915 *
1916 * Set VCN block clockgating state
1917 */
vcn_v4_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)1918 static int vcn_v4_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1919 {
1920 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1921 bool enable = state == AMD_CG_STATE_GATE;
1922 int i;
1923
1924 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1925 if (adev->vcn.harvest_config & (1 << i))
1926 continue;
1927
1928 if (enable) {
1929 if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1930 return -EBUSY;
1931 vcn_v4_0_enable_clock_gating(adev, i);
1932 } else {
1933 vcn_v4_0_disable_clock_gating(adev, i);
1934 }
1935 }
1936
1937 return 0;
1938 }
1939
1940 /**
1941 * vcn_v4_0_set_powergating_state - set VCN block powergating state
1942 *
1943 * @handle: amdgpu_device pointer
1944 * @state: power gating state
1945 *
1946 * Set VCN block powergating state
1947 */
vcn_v4_0_set_powergating_state(void * handle,enum amd_powergating_state state)1948 static int vcn_v4_0_set_powergating_state(void *handle, enum amd_powergating_state state)
1949 {
1950 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1951 int ret;
1952
1953 /* for SRIOV, guest should not control VCN Power-gating
1954 * MMSCH FW should control Power-gating and clock-gating
1955 * guest should avoid touching CGC and PG
1956 */
1957 if (amdgpu_sriov_vf(adev)) {
1958 adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
1959 return 0;
1960 }
1961
1962 if (state == adev->vcn.cur_state)
1963 return 0;
1964
1965 if (state == AMD_PG_STATE_GATE)
1966 ret = vcn_v4_0_stop(adev);
1967 else
1968 ret = vcn_v4_0_start(adev);
1969
1970 if (!ret)
1971 adev->vcn.cur_state = state;
1972
1973 return ret;
1974 }
1975
1976 /**
1977 * vcn_v4_0_set_interrupt_state - set VCN block interrupt state
1978 *
1979 * @adev: amdgpu_device pointer
1980 * @source: interrupt sources
1981 * @type: interrupt types
1982 * @state: interrupt states
1983 *
1984 * Set VCN block interrupt state
1985 */
vcn_v4_0_set_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1986 static int vcn_v4_0_set_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1987 unsigned type, enum amdgpu_interrupt_state state)
1988 {
1989 return 0;
1990 }
1991
1992 /**
1993 * vcn_v4_0_set_ras_interrupt_state - set VCN block RAS interrupt state
1994 *
1995 * @adev: amdgpu_device pointer
1996 * @source: interrupt sources
1997 * @type: interrupt types
1998 * @state: interrupt states
1999 *
2000 * Set VCN block RAS interrupt state
2001 */
vcn_v4_0_set_ras_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)2002 static int vcn_v4_0_set_ras_interrupt_state(struct amdgpu_device *adev,
2003 struct amdgpu_irq_src *source,
2004 unsigned int type,
2005 enum amdgpu_interrupt_state state)
2006 {
2007 return 0;
2008 }
2009
2010 /**
2011 * vcn_v4_0_process_interrupt - process VCN block interrupt
2012 *
2013 * @adev: amdgpu_device pointer
2014 * @source: interrupt sources
2015 * @entry: interrupt entry from clients and sources
2016 *
2017 * Process VCN block interrupt
2018 */
vcn_v4_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)2019 static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
2020 struct amdgpu_iv_entry *entry)
2021 {
2022 uint32_t ip_instance;
2023
2024 switch (entry->client_id) {
2025 case SOC15_IH_CLIENTID_VCN:
2026 ip_instance = 0;
2027 break;
2028 case SOC15_IH_CLIENTID_VCN1:
2029 ip_instance = 1;
2030 break;
2031 default:
2032 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2033 return 0;
2034 }
2035
2036 DRM_DEBUG("IH: VCN TRAP\n");
2037
2038 switch (entry->src_id) {
2039 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2040 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2041 break;
2042 default:
2043 DRM_ERROR("Unhandled interrupt: %d %d\n",
2044 entry->src_id, entry->src_data[0]);
2045 break;
2046 }
2047
2048 return 0;
2049 }
2050
2051 static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {
2052 .set = vcn_v4_0_set_interrupt_state,
2053 .process = vcn_v4_0_process_interrupt,
2054 };
2055
2056 static const struct amdgpu_irq_src_funcs vcn_v4_0_ras_irq_funcs = {
2057 .set = vcn_v4_0_set_ras_interrupt_state,
2058 .process = amdgpu_vcn_process_poison_irq,
2059 };
2060
2061 /**
2062 * vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions
2063 *
2064 * @adev: amdgpu_device pointer
2065 *
2066 * Set VCN block interrupt irq functions
2067 */
vcn_v4_0_set_irq_funcs(struct amdgpu_device * adev)2068 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)
2069 {
2070 int i;
2071
2072 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2073 if (adev->vcn.harvest_config & (1 << i))
2074 continue;
2075
2076 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2077 adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;
2078
2079 adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.num_enc_rings + 1;
2080 adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v4_0_ras_irq_funcs;
2081 }
2082 }
2083
2084 static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
2085 .name = "vcn_v4_0",
2086 .early_init = vcn_v4_0_early_init,
2087 .late_init = NULL,
2088 .sw_init = vcn_v4_0_sw_init,
2089 .sw_fini = vcn_v4_0_sw_fini,
2090 .hw_init = vcn_v4_0_hw_init,
2091 .hw_fini = vcn_v4_0_hw_fini,
2092 .suspend = vcn_v4_0_suspend,
2093 .resume = vcn_v4_0_resume,
2094 .is_idle = vcn_v4_0_is_idle,
2095 .wait_for_idle = vcn_v4_0_wait_for_idle,
2096 .check_soft_reset = NULL,
2097 .pre_soft_reset = NULL,
2098 .soft_reset = NULL,
2099 .post_soft_reset = NULL,
2100 .set_clockgating_state = vcn_v4_0_set_clockgating_state,
2101 .set_powergating_state = vcn_v4_0_set_powergating_state,
2102 };
2103
2104 const struct amdgpu_ip_block_version vcn_v4_0_ip_block = {
2105 .type = AMD_IP_BLOCK_TYPE_VCN,
2106 .major = 4,
2107 .minor = 0,
2108 .rev = 0,
2109 .funcs = &vcn_v4_0_ip_funcs,
2110 };
2111
vcn_v4_0_query_poison_by_instance(struct amdgpu_device * adev,uint32_t instance,uint32_t sub_block)2112 static uint32_t vcn_v4_0_query_poison_by_instance(struct amdgpu_device *adev,
2113 uint32_t instance, uint32_t sub_block)
2114 {
2115 uint32_t poison_stat = 0, reg_value = 0;
2116
2117 switch (sub_block) {
2118 case AMDGPU_VCN_V4_0_VCPU_VCODEC:
2119 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS);
2120 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
2121 break;
2122 default:
2123 break;
2124 }
2125
2126 if (poison_stat)
2127 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
2128 instance, sub_block);
2129
2130 return poison_stat;
2131 }
2132
vcn_v4_0_query_ras_poison_status(struct amdgpu_device * adev)2133 static bool vcn_v4_0_query_ras_poison_status(struct amdgpu_device *adev)
2134 {
2135 uint32_t inst, sub;
2136 uint32_t poison_stat = 0;
2137
2138 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
2139 for (sub = 0; sub < AMDGPU_VCN_V4_0_MAX_SUB_BLOCK; sub++)
2140 poison_stat +=
2141 vcn_v4_0_query_poison_by_instance(adev, inst, sub);
2142
2143 return !!poison_stat;
2144 }
2145
2146 const struct amdgpu_ras_block_hw_ops vcn_v4_0_ras_hw_ops = {
2147 .query_poison_status = vcn_v4_0_query_ras_poison_status,
2148 };
2149
2150 static struct amdgpu_vcn_ras vcn_v4_0_ras = {
2151 .ras_block = {
2152 .hw_ops = &vcn_v4_0_ras_hw_ops,
2153 .ras_late_init = amdgpu_vcn_ras_late_init,
2154 },
2155 };
2156
vcn_v4_0_set_ras_funcs(struct amdgpu_device * adev)2157 static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev)
2158 {
2159 switch (adev->ip_versions[VCN_HWIP][0]) {
2160 case IP_VERSION(4, 0, 0):
2161 adev->vcn.ras = &vcn_v4_0_ras;
2162 break;
2163 default:
2164 break;
2165 }
2166 }
2167