1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_drv.h> 26 27 #include "amdgpu.h" 28 #include "amdgpu_vcn.h" 29 #include "soc15.h" 30 #include "soc15d.h" 31 #include "amdgpu_pm.h" 32 #include "amdgpu_psp.h" 33 #include "mmsch_v2_0.h" 34 #include "vcn_v2_0.h" 35 36 #include "vcn/vcn_2_0_0_offset.h" 37 #include "vcn/vcn_2_0_0_sh_mask.h" 38 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h" 39 40 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x1fd 41 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x503 42 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x504 43 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x505 44 #define mmUVD_NO_OP_INTERNAL_OFFSET 0x53f 45 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x54a 46 #define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d 47 48 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x1e1 49 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x5a6 50 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x5a7 51 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x1e2 52 53 static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev); 54 static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev); 55 static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev); 56 static int vcn_v2_0_set_powergating_state(void *handle, 57 enum amd_powergating_state state); 58 static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev, 59 int inst_idx, struct dpg_pause_state *new_state); 60 static int vcn_v2_0_start_sriov(struct amdgpu_device *adev); 61 /** 62 * vcn_v2_0_early_init - set function pointers 63 * 64 * @handle: amdgpu_device pointer 65 * 66 * Set ring and irq function pointers 67 */ 68 static int vcn_v2_0_early_init(void *handle) 69 { 70 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 71 72 if (amdgpu_sriov_vf(adev)) 73 adev->vcn.num_enc_rings = 1; 74 else 75 adev->vcn.num_enc_rings = 2; 76 77 vcn_v2_0_set_dec_ring_funcs(adev); 78 vcn_v2_0_set_enc_ring_funcs(adev); 79 vcn_v2_0_set_irq_funcs(adev); 80 81 return 0; 82 } 83 84 /** 85 * vcn_v2_0_sw_init - sw init for VCN block 86 * 87 * @handle: amdgpu_device pointer 88 * 89 * Load firmware and sw initialization 90 */ 91 static int vcn_v2_0_sw_init(void *handle) 92 { 93 struct amdgpu_ring *ring; 94 int i, r; 95 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 96 volatile struct amdgpu_fw_shared *fw_shared; 97 98 /* VCN DEC TRAP */ 99 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 100 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, 101 &adev->vcn.inst->irq); 102 if (r) 103 return r; 104 105 /* VCN ENC TRAP */ 106 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 107 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 108 i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, 109 &adev->vcn.inst->irq); 110 if (r) 111 return r; 112 } 113 114 r = amdgpu_vcn_sw_init(adev); 115 if (r) 116 return r; 117 118 amdgpu_vcn_setup_ucode(adev); 119 120 r = amdgpu_vcn_resume(adev); 121 if (r) 122 return r; 123 124 ring = &adev->vcn.inst->ring_dec; 125 126 ring->use_doorbell = true; 127 ring->doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 << 1; 128 129 sprintf(ring->name, "vcn_dec"); 130 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, 131 AMDGPU_RING_PRIO_DEFAULT, NULL); 132 if (r) 133 return r; 134 135 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET; 136 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET; 137 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET; 138 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET; 139 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET; 140 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET; 141 142 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET; 143 adev->vcn.inst->external.scratch9 = SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9); 144 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET; 145 adev->vcn.inst->external.data0 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0); 146 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET; 147 adev->vcn.inst->external.data1 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1); 148 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET; 149 adev->vcn.inst->external.cmd = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD); 150 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET; 151 adev->vcn.inst->external.nop = SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP); 152 153 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 154 enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i); 155 156 ring = &adev->vcn.inst->ring_enc[i]; 157 ring->use_doorbell = true; 158 if (!amdgpu_sriov_vf(adev)) 159 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + i; 160 else 161 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + i; 162 sprintf(ring->name, "vcn_enc%d", i); 163 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, 164 hw_prio, NULL); 165 if (r) 166 return r; 167 } 168 169 adev->vcn.pause_dpg_mode = vcn_v2_0_pause_dpg_mode; 170 171 r = amdgpu_virt_alloc_mm_table(adev); 172 if (r) 173 return r; 174 175 fw_shared = adev->vcn.inst->fw_shared_cpu_addr; 176 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG); 177 return 0; 178 } 179 180 /** 181 * vcn_v2_0_sw_fini - sw fini for VCN block 182 * 183 * @handle: amdgpu_device pointer 184 * 185 * VCN suspend and free up sw allocation 186 */ 187 static int vcn_v2_0_sw_fini(void *handle) 188 { 189 int r, idx; 190 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 191 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared_cpu_addr; 192 193 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 194 fw_shared->present_flag_0 = 0; 195 drm_dev_exit(idx); 196 } 197 198 amdgpu_virt_free_mm_table(adev); 199 200 r = amdgpu_vcn_suspend(adev); 201 if (r) 202 return r; 203 204 r = amdgpu_vcn_sw_fini(adev); 205 206 return r; 207 } 208 209 /** 210 * vcn_v2_0_hw_init - start and test VCN block 211 * 212 * @handle: amdgpu_device pointer 213 * 214 * Initialize the hardware, boot up the VCPU and do some testing 215 */ 216 static int vcn_v2_0_hw_init(void *handle) 217 { 218 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 219 struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec; 220 int i, r; 221 222 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, 223 ring->doorbell_index, 0); 224 225 if (amdgpu_sriov_vf(adev)) 226 vcn_v2_0_start_sriov(adev); 227 228 r = amdgpu_ring_test_helper(ring); 229 if (r) 230 goto done; 231 232 //Disable vcn decode for sriov 233 if (amdgpu_sriov_vf(adev)) 234 ring->sched.ready = false; 235 236 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 237 ring = &adev->vcn.inst->ring_enc[i]; 238 r = amdgpu_ring_test_helper(ring); 239 if (r) 240 goto done; 241 } 242 243 done: 244 if (!r) 245 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n", 246 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode"); 247 248 return r; 249 } 250 251 /** 252 * vcn_v2_0_hw_fini - stop the hardware block 253 * 254 * @handle: amdgpu_device pointer 255 * 256 * Stop the VCN block, mark ring as not ready any more 257 */ 258 static int vcn_v2_0_hw_fini(void *handle) 259 { 260 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 261 262 cancel_delayed_work_sync(&adev->vcn.idle_work); 263 264 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) || 265 (adev->vcn.cur_state != AMD_PG_STATE_GATE && 266 RREG32_SOC15(VCN, 0, mmUVD_STATUS))) 267 vcn_v2_0_set_powergating_state(adev, AMD_PG_STATE_GATE); 268 269 return 0; 270 } 271 272 /** 273 * vcn_v2_0_suspend - suspend VCN block 274 * 275 * @handle: amdgpu_device pointer 276 * 277 * HW fini and suspend VCN block 278 */ 279 static int vcn_v2_0_suspend(void *handle) 280 { 281 int r; 282 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 283 284 r = vcn_v2_0_hw_fini(adev); 285 if (r) 286 return r; 287 288 r = amdgpu_vcn_suspend(adev); 289 290 return r; 291 } 292 293 /** 294 * vcn_v2_0_resume - resume VCN block 295 * 296 * @handle: amdgpu_device pointer 297 * 298 * Resume firmware and hw init VCN block 299 */ 300 static int vcn_v2_0_resume(void *handle) 301 { 302 int r; 303 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 304 305 r = amdgpu_vcn_resume(adev); 306 if (r) 307 return r; 308 309 r = vcn_v2_0_hw_init(adev); 310 311 return r; 312 } 313 314 /** 315 * vcn_v2_0_mc_resume - memory controller programming 316 * 317 * @adev: amdgpu_device pointer 318 * 319 * Let the VCN memory controller know it's offsets 320 */ 321 static void vcn_v2_0_mc_resume(struct amdgpu_device *adev) 322 { 323 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); 324 uint32_t offset; 325 326 if (amdgpu_sriov_vf(adev)) 327 return; 328 329 /* cache window 0: fw */ 330 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 331 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 332 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo)); 333 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 334 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi)); 335 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0); 336 offset = 0; 337 } else { 338 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 339 lower_32_bits(adev->vcn.inst->gpu_addr)); 340 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 341 upper_32_bits(adev->vcn.inst->gpu_addr)); 342 offset = size; 343 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 344 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 345 } 346 347 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size); 348 349 /* cache window 1: stack */ 350 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, 351 lower_32_bits(adev->vcn.inst->gpu_addr + offset)); 352 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, 353 upper_32_bits(adev->vcn.inst->gpu_addr + offset)); 354 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0); 355 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE); 356 357 /* cache window 2: context */ 358 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, 359 lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); 360 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, 361 upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); 362 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0); 363 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE); 364 365 /* non-cache window */ 366 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, 367 lower_32_bits(adev->vcn.inst->fw_shared_gpu_addr)); 368 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, 369 upper_32_bits(adev->vcn.inst->fw_shared_gpu_addr)); 370 WREG32_SOC15(UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0, 0); 371 WREG32_SOC15(UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0, 372 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared))); 373 374 WREG32_SOC15(UVD, 0, mmUVD_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 375 } 376 377 static void vcn_v2_0_mc_resume_dpg_mode(struct amdgpu_device *adev, bool indirect) 378 { 379 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); 380 uint32_t offset; 381 382 /* cache window 0: fw */ 383 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 384 if (!indirect) { 385 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 386 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 387 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo), 0, indirect); 388 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 389 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 390 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi), 0, indirect); 391 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 392 UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 393 } else { 394 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 395 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); 396 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 397 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); 398 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 399 UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 400 } 401 offset = 0; 402 } else { 403 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 404 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 405 lower_32_bits(adev->vcn.inst->gpu_addr), 0, indirect); 406 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 407 UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 408 upper_32_bits(adev->vcn.inst->gpu_addr), 0, indirect); 409 offset = size; 410 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 411 UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 412 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); 413 } 414 415 if (!indirect) 416 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 417 UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect); 418 else 419 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 420 UVD, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); 421 422 /* cache window 1: stack */ 423 if (!indirect) { 424 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 425 UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 426 lower_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect); 427 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 428 UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 429 upper_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect); 430 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 431 UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 432 } else { 433 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 434 UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); 435 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 436 UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); 437 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 438 UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 439 } 440 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 441 UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); 442 443 /* cache window 2: context */ 444 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 445 UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 446 lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect); 447 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 448 UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 449 upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect); 450 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 451 UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); 452 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 453 UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); 454 455 /* non-cache window */ 456 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 457 UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 458 lower_32_bits(adev->vcn.inst->fw_shared_gpu_addr), 0, indirect); 459 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 460 UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 461 upper_32_bits(adev->vcn.inst->fw_shared_gpu_addr), 0, indirect); 462 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 463 UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); 464 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 465 UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0), 466 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect); 467 468 /* VCN global tiling registers */ 469 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 470 UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 471 } 472 473 /** 474 * vcn_v2_0_disable_clock_gating - disable VCN clock gating 475 * 476 * @adev: amdgpu_device pointer 477 * 478 * Disable clock gating for VCN block 479 */ 480 static void vcn_v2_0_disable_clock_gating(struct amdgpu_device *adev) 481 { 482 uint32_t data; 483 484 if (amdgpu_sriov_vf(adev)) 485 return; 486 487 /* UVD disable CGC */ 488 data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL); 489 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 490 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 491 else 492 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; 493 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 494 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 495 WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data); 496 497 data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE); 498 data &= ~(UVD_CGC_GATE__SYS_MASK 499 | UVD_CGC_GATE__UDEC_MASK 500 | UVD_CGC_GATE__MPEG2_MASK 501 | UVD_CGC_GATE__REGS_MASK 502 | UVD_CGC_GATE__RBC_MASK 503 | UVD_CGC_GATE__LMI_MC_MASK 504 | UVD_CGC_GATE__LMI_UMC_MASK 505 | UVD_CGC_GATE__IDCT_MASK 506 | UVD_CGC_GATE__MPRD_MASK 507 | UVD_CGC_GATE__MPC_MASK 508 | UVD_CGC_GATE__LBSI_MASK 509 | UVD_CGC_GATE__LRBBM_MASK 510 | UVD_CGC_GATE__UDEC_RE_MASK 511 | UVD_CGC_GATE__UDEC_CM_MASK 512 | UVD_CGC_GATE__UDEC_IT_MASK 513 | UVD_CGC_GATE__UDEC_DB_MASK 514 | UVD_CGC_GATE__UDEC_MP_MASK 515 | UVD_CGC_GATE__WCB_MASK 516 | UVD_CGC_GATE__VCPU_MASK 517 | UVD_CGC_GATE__SCPU_MASK); 518 WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data); 519 520 data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL); 521 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK 522 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK 523 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK 524 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK 525 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK 526 | UVD_CGC_CTRL__SYS_MODE_MASK 527 | UVD_CGC_CTRL__UDEC_MODE_MASK 528 | UVD_CGC_CTRL__MPEG2_MODE_MASK 529 | UVD_CGC_CTRL__REGS_MODE_MASK 530 | UVD_CGC_CTRL__RBC_MODE_MASK 531 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 532 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 533 | UVD_CGC_CTRL__IDCT_MODE_MASK 534 | UVD_CGC_CTRL__MPRD_MODE_MASK 535 | UVD_CGC_CTRL__MPC_MODE_MASK 536 | UVD_CGC_CTRL__LBSI_MODE_MASK 537 | UVD_CGC_CTRL__LRBBM_MODE_MASK 538 | UVD_CGC_CTRL__WCB_MODE_MASK 539 | UVD_CGC_CTRL__VCPU_MODE_MASK 540 | UVD_CGC_CTRL__SCPU_MODE_MASK); 541 WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data); 542 543 /* turn on */ 544 data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE); 545 data |= (UVD_SUVD_CGC_GATE__SRE_MASK 546 | UVD_SUVD_CGC_GATE__SIT_MASK 547 | UVD_SUVD_CGC_GATE__SMP_MASK 548 | UVD_SUVD_CGC_GATE__SCM_MASK 549 | UVD_SUVD_CGC_GATE__SDB_MASK 550 | UVD_SUVD_CGC_GATE__SRE_H264_MASK 551 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK 552 | UVD_SUVD_CGC_GATE__SIT_H264_MASK 553 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK 554 | UVD_SUVD_CGC_GATE__SCM_H264_MASK 555 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK 556 | UVD_SUVD_CGC_GATE__SDB_H264_MASK 557 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK 558 | UVD_SUVD_CGC_GATE__SCLR_MASK 559 | UVD_SUVD_CGC_GATE__UVD_SC_MASK 560 | UVD_SUVD_CGC_GATE__ENT_MASK 561 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK 562 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK 563 | UVD_SUVD_CGC_GATE__SITE_MASK 564 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK 565 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK 566 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK 567 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK 568 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); 569 WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data); 570 571 data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL); 572 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 573 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 574 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 575 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 576 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 577 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK 578 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK 579 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 580 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 581 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 582 WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data); 583 } 584 585 static void vcn_v2_0_clock_gating_dpg_mode(struct amdgpu_device *adev, 586 uint8_t sram_sel, uint8_t indirect) 587 { 588 uint32_t reg_data = 0; 589 590 /* enable sw clock gating control */ 591 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 592 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 593 else 594 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 595 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 596 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 597 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK | 598 UVD_CGC_CTRL__UDEC_CM_MODE_MASK | 599 UVD_CGC_CTRL__UDEC_IT_MODE_MASK | 600 UVD_CGC_CTRL__UDEC_DB_MODE_MASK | 601 UVD_CGC_CTRL__UDEC_MP_MODE_MASK | 602 UVD_CGC_CTRL__SYS_MODE_MASK | 603 UVD_CGC_CTRL__UDEC_MODE_MASK | 604 UVD_CGC_CTRL__MPEG2_MODE_MASK | 605 UVD_CGC_CTRL__REGS_MODE_MASK | 606 UVD_CGC_CTRL__RBC_MODE_MASK | 607 UVD_CGC_CTRL__LMI_MC_MODE_MASK | 608 UVD_CGC_CTRL__LMI_UMC_MODE_MASK | 609 UVD_CGC_CTRL__IDCT_MODE_MASK | 610 UVD_CGC_CTRL__MPRD_MODE_MASK | 611 UVD_CGC_CTRL__MPC_MODE_MASK | 612 UVD_CGC_CTRL__LBSI_MODE_MASK | 613 UVD_CGC_CTRL__LRBBM_MODE_MASK | 614 UVD_CGC_CTRL__WCB_MODE_MASK | 615 UVD_CGC_CTRL__VCPU_MODE_MASK | 616 UVD_CGC_CTRL__SCPU_MODE_MASK); 617 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 618 UVD, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect); 619 620 /* turn off clock gating */ 621 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 622 UVD, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect); 623 624 /* turn on SUVD clock gating */ 625 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 626 UVD, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); 627 628 /* turn on sw mode in UVD_SUVD_CGC_CTRL */ 629 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 630 UVD, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); 631 } 632 633 /** 634 * vcn_v2_0_enable_clock_gating - enable VCN clock gating 635 * 636 * @adev: amdgpu_device pointer 637 * 638 * Enable clock gating for VCN block 639 */ 640 static void vcn_v2_0_enable_clock_gating(struct amdgpu_device *adev) 641 { 642 uint32_t data = 0; 643 644 if (amdgpu_sriov_vf(adev)) 645 return; 646 647 /* enable UVD CGC */ 648 data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL); 649 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 650 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 651 else 652 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 653 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 654 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 655 WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data); 656 657 data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL); 658 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK 659 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK 660 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK 661 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK 662 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK 663 | UVD_CGC_CTRL__SYS_MODE_MASK 664 | UVD_CGC_CTRL__UDEC_MODE_MASK 665 | UVD_CGC_CTRL__MPEG2_MODE_MASK 666 | UVD_CGC_CTRL__REGS_MODE_MASK 667 | UVD_CGC_CTRL__RBC_MODE_MASK 668 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 669 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 670 | UVD_CGC_CTRL__IDCT_MODE_MASK 671 | UVD_CGC_CTRL__MPRD_MODE_MASK 672 | UVD_CGC_CTRL__MPC_MODE_MASK 673 | UVD_CGC_CTRL__LBSI_MODE_MASK 674 | UVD_CGC_CTRL__LRBBM_MODE_MASK 675 | UVD_CGC_CTRL__WCB_MODE_MASK 676 | UVD_CGC_CTRL__VCPU_MODE_MASK 677 | UVD_CGC_CTRL__SCPU_MODE_MASK); 678 WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data); 679 680 data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL); 681 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 682 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 683 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 684 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 685 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 686 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK 687 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK 688 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 689 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 690 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 691 WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data); 692 } 693 694 static void vcn_v2_0_disable_static_power_gating(struct amdgpu_device *adev) 695 { 696 uint32_t data = 0; 697 698 if (amdgpu_sriov_vf(adev)) 699 return; 700 701 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) { 702 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT 703 | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT 704 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT 705 | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT 706 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT 707 | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT 708 | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT 709 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT 710 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT 711 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT); 712 713 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data); 714 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 715 UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0, 0xFFFFF); 716 } else { 717 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT 718 | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT 719 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT 720 | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT 721 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT 722 | 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT 723 | 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT 724 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT 725 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT 726 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT); 727 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data); 728 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0, 0xFFFFF); 729 } 730 731 /* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS, 732 * UVDU_PWR_STATUS are 0 (power on) */ 733 734 data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS); 735 data &= ~0x103; 736 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) 737 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON | 738 UVD_POWER_STATUS__UVD_PG_EN_MASK; 739 740 WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data); 741 } 742 743 static void vcn_v2_0_enable_static_power_gating(struct amdgpu_device *adev) 744 { 745 uint32_t data = 0; 746 747 if (amdgpu_sriov_vf(adev)) 748 return; 749 750 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) { 751 /* Before power off, this indicator has to be turned on */ 752 data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS); 753 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK; 754 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF; 755 WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data); 756 757 758 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT 759 | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT 760 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT 761 | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT 762 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT 763 | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT 764 | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT 765 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT 766 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT 767 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT); 768 769 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data); 770 771 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT 772 | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT 773 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT 774 | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT 775 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT 776 | 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT 777 | 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT 778 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT 779 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT 780 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT); 781 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFF); 782 } 783 } 784 785 static int vcn_v2_0_start_dpg_mode(struct amdgpu_device *adev, bool indirect) 786 { 787 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared_cpu_addr; 788 struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec; 789 uint32_t rb_bufsz, tmp; 790 791 vcn_v2_0_enable_static_power_gating(adev); 792 793 /* enable dynamic power gating mode */ 794 tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS); 795 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; 796 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; 797 WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp); 798 799 if (indirect) 800 adev->vcn.inst->dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst->dpg_sram_cpu_addr; 801 802 /* enable clock gating */ 803 vcn_v2_0_clock_gating_dpg_mode(adev, 0, indirect); 804 805 /* enable VCPU clock */ 806 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 807 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 808 tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK; 809 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 810 UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect); 811 812 /* disable master interupt */ 813 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 814 UVD, 0, mmUVD_MASTINT_EN), 0, 0, indirect); 815 816 /* setup mmUVD_LMI_CTRL */ 817 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 818 UVD_LMI_CTRL__REQ_MODE_MASK | 819 UVD_LMI_CTRL__CRC_RESET_MASK | 820 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 821 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 822 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | 823 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 824 0x00100000L); 825 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 826 UVD, 0, mmUVD_LMI_CTRL), tmp, 0, indirect); 827 828 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 829 UVD, 0, mmUVD_MPC_CNTL), 830 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); 831 832 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 833 UVD, 0, mmUVD_MPC_SET_MUXA0), 834 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 835 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 836 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 837 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); 838 839 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 840 UVD, 0, mmUVD_MPC_SET_MUXB0), 841 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 842 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 843 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 844 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); 845 846 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 847 UVD, 0, mmUVD_MPC_SET_MUX), 848 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 849 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 850 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); 851 852 vcn_v2_0_mc_resume_dpg_mode(adev, indirect); 853 854 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 855 UVD, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect); 856 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 857 UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect); 858 859 /* release VCPU reset to boot */ 860 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 861 UVD, 0, mmUVD_SOFT_RESET), 0, 0, indirect); 862 863 /* enable LMI MC and UMC channels */ 864 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 865 UVD, 0, mmUVD_LMI_CTRL2), 866 0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT, 0, indirect); 867 868 /* enable master interrupt */ 869 WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET( 870 UVD, 0, mmUVD_MASTINT_EN), 871 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); 872 873 if (indirect) 874 psp_update_vcn_sram(adev, 0, adev->vcn.inst->dpg_sram_gpu_addr, 875 (uint32_t)((uintptr_t)adev->vcn.inst->dpg_sram_curr_addr - 876 (uintptr_t)adev->vcn.inst->dpg_sram_cpu_addr)); 877 878 /* force RBC into idle state */ 879 rb_bufsz = order_base_2(ring->ring_size); 880 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz); 881 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1); 882 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1); 883 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); 884 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); 885 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp); 886 887 /* Stall DPG before WPTR/RPTR reset */ 888 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 889 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK, 890 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 891 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET; 892 893 /* set the write pointer delay */ 894 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0); 895 896 /* set the wb address */ 897 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR, 898 (upper_32_bits(ring->gpu_addr) >> 2)); 899 900 /* program the RB_BASE for ring buffer */ 901 WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW, 902 lower_32_bits(ring->gpu_addr)); 903 WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH, 904 upper_32_bits(ring->gpu_addr)); 905 906 /* Initialize the ring buffer's read and write pointers */ 907 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0); 908 909 WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0); 910 911 ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR); 912 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, 913 lower_32_bits(ring->wptr)); 914 915 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET; 916 /* Unstall DPG */ 917 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 918 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 919 return 0; 920 } 921 922 static int vcn_v2_0_start(struct amdgpu_device *adev) 923 { 924 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared_cpu_addr; 925 struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec; 926 uint32_t rb_bufsz, tmp; 927 uint32_t lmi_swap_cntl; 928 int i, j, r; 929 930 if (adev->pm.dpm_enabled) 931 amdgpu_dpm_enable_uvd(adev, true); 932 933 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 934 return vcn_v2_0_start_dpg_mode(adev, adev->vcn.indirect_sram); 935 936 vcn_v2_0_disable_static_power_gating(adev); 937 938 /* set uvd status busy */ 939 tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY; 940 WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp); 941 942 /*SW clock gating */ 943 vcn_v2_0_disable_clock_gating(adev); 944 945 /* enable VCPU clock */ 946 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 947 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK); 948 949 /* disable master interrupt */ 950 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0, 951 ~UVD_MASTINT_EN__VCPU_EN_MASK); 952 953 /* setup mmUVD_LMI_CTRL */ 954 tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL); 955 WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp | 956 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 957 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 958 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 959 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); 960 961 /* setup mmUVD_MPC_CNTL */ 962 tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL); 963 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; 964 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; 965 WREG32_SOC15(VCN, 0, mmUVD_MPC_CNTL, tmp); 966 967 /* setup UVD_MPC_SET_MUXA0 */ 968 WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0, 969 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 970 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 971 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 972 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); 973 974 /* setup UVD_MPC_SET_MUXB0 */ 975 WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0, 976 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 977 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 978 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 979 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); 980 981 /* setup mmUVD_MPC_SET_MUX */ 982 WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX, 983 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 984 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 985 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); 986 987 vcn_v2_0_mc_resume(adev); 988 989 /* release VCPU reset to boot */ 990 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0, 991 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK); 992 993 /* enable LMI MC and UMC channels */ 994 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0, 995 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); 996 997 tmp = RREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET); 998 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 999 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1000 WREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET, tmp); 1001 1002 /* disable byte swapping */ 1003 lmi_swap_cntl = 0; 1004 #ifdef __BIG_ENDIAN 1005 /* swap (8 in 32) RB and IB */ 1006 lmi_swap_cntl = 0xa; 1007 #endif 1008 WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl); 1009 1010 for (i = 0; i < 10; ++i) { 1011 uint32_t status; 1012 1013 for (j = 0; j < 100; ++j) { 1014 status = RREG32_SOC15(UVD, 0, mmUVD_STATUS); 1015 if (status & 2) 1016 break; 1017 mdelay(10); 1018 } 1019 r = 0; 1020 if (status & 2) 1021 break; 1022 1023 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n"); 1024 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 1025 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK, 1026 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK); 1027 mdelay(10); 1028 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0, 1029 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK); 1030 mdelay(10); 1031 r = -1; 1032 } 1033 1034 if (r) { 1035 DRM_ERROR("VCN decode not responding, giving up!!!\n"); 1036 return r; 1037 } 1038 1039 /* enable master interrupt */ 1040 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 1041 UVD_MASTINT_EN__VCPU_EN_MASK, 1042 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1043 1044 /* clear the busy bit of VCN_STATUS */ 1045 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0, 1046 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); 1047 1048 WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_VMID, 0); 1049 1050 /* force RBC into idle state */ 1051 rb_bufsz = order_base_2(ring->ring_size); 1052 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz); 1053 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1); 1054 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1); 1055 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); 1056 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); 1057 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp); 1058 1059 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET; 1060 /* program the RB_BASE for ring buffer */ 1061 WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW, 1062 lower_32_bits(ring->gpu_addr)); 1063 WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH, 1064 upper_32_bits(ring->gpu_addr)); 1065 1066 /* Initialize the ring buffer's read and write pointers */ 1067 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0); 1068 1069 ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR); 1070 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, 1071 lower_32_bits(ring->wptr)); 1072 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET; 1073 1074 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET; 1075 ring = &adev->vcn.inst->ring_enc[0]; 1076 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr)); 1077 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr)); 1078 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr); 1079 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); 1080 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4); 1081 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET; 1082 1083 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET; 1084 ring = &adev->vcn.inst->ring_enc[1]; 1085 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr)); 1086 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr)); 1087 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr); 1088 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); 1089 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4); 1090 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET; 1091 1092 return 0; 1093 } 1094 1095 static int vcn_v2_0_stop_dpg_mode(struct amdgpu_device *adev) 1096 { 1097 uint32_t tmp; 1098 1099 /* Wait for power status to be 1 */ 1100 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1, 1101 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1102 1103 /* wait for read ptr to be equal to write ptr */ 1104 tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR); 1105 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF); 1106 1107 tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2); 1108 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF); 1109 1110 tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF; 1111 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF); 1112 1113 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1, 1114 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1115 1116 /* disable dynamic power gating mode */ 1117 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0, 1118 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); 1119 1120 return 0; 1121 } 1122 1123 static int vcn_v2_0_stop(struct amdgpu_device *adev) 1124 { 1125 uint32_t tmp; 1126 int r; 1127 1128 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 1129 r = vcn_v2_0_stop_dpg_mode(adev); 1130 if (r) 1131 return r; 1132 goto power_off; 1133 } 1134 1135 /* wait for uvd idle */ 1136 r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7); 1137 if (r) 1138 return r; 1139 1140 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | 1141 UVD_LMI_STATUS__READ_CLEAN_MASK | 1142 UVD_LMI_STATUS__WRITE_CLEAN_MASK | 1143 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; 1144 r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp); 1145 if (r) 1146 return r; 1147 1148 /* stall UMC channel */ 1149 tmp = RREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2); 1150 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; 1151 WREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2, tmp); 1152 1153 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK| 1154 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; 1155 r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp); 1156 if (r) 1157 return r; 1158 1159 /* disable VCPU clock */ 1160 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0, 1161 ~(UVD_VCPU_CNTL__CLK_EN_MASK)); 1162 1163 /* reset LMI UMC */ 1164 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 1165 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK, 1166 ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK); 1167 1168 /* reset LMI */ 1169 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 1170 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK, 1171 ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK); 1172 1173 /* reset VCPU */ 1174 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 1175 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK, 1176 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK); 1177 1178 /* clear status */ 1179 WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0); 1180 1181 vcn_v2_0_enable_clock_gating(adev); 1182 vcn_v2_0_enable_static_power_gating(adev); 1183 1184 power_off: 1185 if (adev->pm.dpm_enabled) 1186 amdgpu_dpm_enable_uvd(adev, false); 1187 1188 return 0; 1189 } 1190 1191 static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev, 1192 int inst_idx, struct dpg_pause_state *new_state) 1193 { 1194 struct amdgpu_ring *ring; 1195 uint32_t reg_data = 0; 1196 int ret_code; 1197 1198 /* pause/unpause if state is changed */ 1199 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) { 1200 DRM_DEBUG("dpg pause state changed %d -> %d", 1201 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based); 1202 reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) & 1203 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK); 1204 1205 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) { 1206 ret_code = SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 0x1, 1207 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1208 1209 if (!ret_code) { 1210 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared_cpu_addr; 1211 /* pause DPG */ 1212 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK; 1213 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data); 1214 1215 /* wait for ACK */ 1216 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE, 1217 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, 1218 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK); 1219 1220 /* Stall DPG before WPTR/RPTR reset */ 1221 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 1222 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK, 1223 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 1224 /* Restore */ 1225 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET; 1226 ring = &adev->vcn.inst->ring_enc[0]; 1227 ring->wptr = 0; 1228 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr); 1229 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); 1230 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4); 1231 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr)); 1232 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr)); 1233 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET; 1234 1235 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET; 1236 ring = &adev->vcn.inst->ring_enc[1]; 1237 ring->wptr = 0; 1238 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr); 1239 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); 1240 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4); 1241 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr)); 1242 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr)); 1243 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET; 1244 1245 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET; 1246 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, 1247 RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF); 1248 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET; 1249 /* Unstall DPG */ 1250 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 1251 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 1252 1253 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1254 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, 1255 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1256 } 1257 } else { 1258 /* unpause dpg, no need to wait */ 1259 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK; 1260 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data); 1261 } 1262 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based; 1263 } 1264 1265 return 0; 1266 } 1267 1268 static bool vcn_v2_0_is_idle(void *handle) 1269 { 1270 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1271 1272 return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE); 1273 } 1274 1275 static int vcn_v2_0_wait_for_idle(void *handle) 1276 { 1277 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1278 int ret; 1279 1280 ret = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 1281 UVD_STATUS__IDLE); 1282 1283 return ret; 1284 } 1285 1286 static int vcn_v2_0_set_clockgating_state(void *handle, 1287 enum amd_clockgating_state state) 1288 { 1289 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1290 bool enable = (state == AMD_CG_STATE_GATE); 1291 1292 if (amdgpu_sriov_vf(adev)) 1293 return 0; 1294 1295 if (enable) { 1296 /* wait for STATUS to clear */ 1297 if (!vcn_v2_0_is_idle(handle)) 1298 return -EBUSY; 1299 vcn_v2_0_enable_clock_gating(adev); 1300 } else { 1301 /* disable HW gating and enable Sw gating */ 1302 vcn_v2_0_disable_clock_gating(adev); 1303 } 1304 return 0; 1305 } 1306 1307 /** 1308 * vcn_v2_0_dec_ring_get_rptr - get read pointer 1309 * 1310 * @ring: amdgpu_ring pointer 1311 * 1312 * Returns the current hardware read pointer 1313 */ 1314 static uint64_t vcn_v2_0_dec_ring_get_rptr(struct amdgpu_ring *ring) 1315 { 1316 struct amdgpu_device *adev = ring->adev; 1317 1318 return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR); 1319 } 1320 1321 /** 1322 * vcn_v2_0_dec_ring_get_wptr - get write pointer 1323 * 1324 * @ring: amdgpu_ring pointer 1325 * 1326 * Returns the current hardware write pointer 1327 */ 1328 static uint64_t vcn_v2_0_dec_ring_get_wptr(struct amdgpu_ring *ring) 1329 { 1330 struct amdgpu_device *adev = ring->adev; 1331 1332 if (ring->use_doorbell) 1333 return adev->wb.wb[ring->wptr_offs]; 1334 else 1335 return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR); 1336 } 1337 1338 /** 1339 * vcn_v2_0_dec_ring_set_wptr - set write pointer 1340 * 1341 * @ring: amdgpu_ring pointer 1342 * 1343 * Commits the write pointer to the hardware 1344 */ 1345 static void vcn_v2_0_dec_ring_set_wptr(struct amdgpu_ring *ring) 1346 { 1347 struct amdgpu_device *adev = ring->adev; 1348 1349 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 1350 WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 1351 lower_32_bits(ring->wptr) | 0x80000000); 1352 1353 if (ring->use_doorbell) { 1354 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 1355 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1356 } else { 1357 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr)); 1358 } 1359 } 1360 1361 /** 1362 * vcn_v2_0_dec_ring_insert_start - insert a start command 1363 * 1364 * @ring: amdgpu_ring pointer 1365 * 1366 * Write a start command to the ring. 1367 */ 1368 void vcn_v2_0_dec_ring_insert_start(struct amdgpu_ring *ring) 1369 { 1370 struct amdgpu_device *adev = ring->adev; 1371 1372 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0)); 1373 amdgpu_ring_write(ring, 0); 1374 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1375 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1)); 1376 } 1377 1378 /** 1379 * vcn_v2_0_dec_ring_insert_end - insert a end command 1380 * 1381 * @ring: amdgpu_ring pointer 1382 * 1383 * Write a end command to the ring. 1384 */ 1385 void vcn_v2_0_dec_ring_insert_end(struct amdgpu_ring *ring) 1386 { 1387 struct amdgpu_device *adev = ring->adev; 1388 1389 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1390 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_END << 1)); 1391 } 1392 1393 /** 1394 * vcn_v2_0_dec_ring_insert_nop - insert a nop command 1395 * 1396 * @ring: amdgpu_ring pointer 1397 * @count: the number of NOP packets to insert 1398 * 1399 * Write a nop command to the ring. 1400 */ 1401 void vcn_v2_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) 1402 { 1403 struct amdgpu_device *adev = ring->adev; 1404 int i; 1405 1406 WARN_ON(ring->wptr % 2 || count % 2); 1407 1408 for (i = 0; i < count / 2; i++) { 1409 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.nop, 0)); 1410 amdgpu_ring_write(ring, 0); 1411 } 1412 } 1413 1414 /** 1415 * vcn_v2_0_dec_ring_emit_fence - emit an fence & trap command 1416 * 1417 * @ring: amdgpu_ring pointer 1418 * @addr: address 1419 * @seq: sequence number 1420 * @flags: fence related flags 1421 * 1422 * Write a fence and a trap command to the ring. 1423 */ 1424 void vcn_v2_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, 1425 unsigned flags) 1426 { 1427 struct amdgpu_device *adev = ring->adev; 1428 1429 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 1430 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.context_id, 0)); 1431 amdgpu_ring_write(ring, seq); 1432 1433 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0)); 1434 amdgpu_ring_write(ring, addr & 0xffffffff); 1435 1436 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0)); 1437 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff); 1438 1439 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1440 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_FENCE << 1)); 1441 1442 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0)); 1443 amdgpu_ring_write(ring, 0); 1444 1445 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0)); 1446 amdgpu_ring_write(ring, 0); 1447 1448 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1449 1450 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_TRAP << 1)); 1451 } 1452 1453 /** 1454 * vcn_v2_0_dec_ring_emit_ib - execute indirect buffer 1455 * 1456 * @ring: amdgpu_ring pointer 1457 * @job: job to retrieve vmid from 1458 * @ib: indirect buffer to execute 1459 * @flags: unused 1460 * 1461 * Write ring commands to execute the indirect buffer 1462 */ 1463 void vcn_v2_0_dec_ring_emit_ib(struct amdgpu_ring *ring, 1464 struct amdgpu_job *job, 1465 struct amdgpu_ib *ib, 1466 uint32_t flags) 1467 { 1468 struct amdgpu_device *adev = ring->adev; 1469 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 1470 1471 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_vmid, 0)); 1472 amdgpu_ring_write(ring, vmid); 1473 1474 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_bar_low, 0)); 1475 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)); 1476 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_bar_high, 0)); 1477 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 1478 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_size, 0)); 1479 amdgpu_ring_write(ring, ib->length_dw); 1480 } 1481 1482 void vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 1483 uint32_t val, uint32_t mask) 1484 { 1485 struct amdgpu_device *adev = ring->adev; 1486 1487 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0)); 1488 amdgpu_ring_write(ring, reg << 2); 1489 1490 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0)); 1491 amdgpu_ring_write(ring, val); 1492 1493 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.gp_scratch8, 0)); 1494 amdgpu_ring_write(ring, mask); 1495 1496 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1497 1498 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_REG_READ_COND_WAIT << 1)); 1499 } 1500 1501 void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring, 1502 unsigned vmid, uint64_t pd_addr) 1503 { 1504 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub]; 1505 uint32_t data0, data1, mask; 1506 1507 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 1508 1509 /* wait for register write */ 1510 data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance; 1511 data1 = lower_32_bits(pd_addr); 1512 mask = 0xffffffff; 1513 vcn_v2_0_dec_ring_emit_reg_wait(ring, data0, data1, mask); 1514 } 1515 1516 void vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring, 1517 uint32_t reg, uint32_t val) 1518 { 1519 struct amdgpu_device *adev = ring->adev; 1520 1521 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0)); 1522 amdgpu_ring_write(ring, reg << 2); 1523 1524 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0)); 1525 amdgpu_ring_write(ring, val); 1526 1527 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1528 1529 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_WRITE_REG << 1)); 1530 } 1531 1532 /** 1533 * vcn_v2_0_enc_ring_get_rptr - get enc read pointer 1534 * 1535 * @ring: amdgpu_ring pointer 1536 * 1537 * Returns the current hardware enc read pointer 1538 */ 1539 static uint64_t vcn_v2_0_enc_ring_get_rptr(struct amdgpu_ring *ring) 1540 { 1541 struct amdgpu_device *adev = ring->adev; 1542 1543 if (ring == &adev->vcn.inst->ring_enc[0]) 1544 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR); 1545 else 1546 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2); 1547 } 1548 1549 /** 1550 * vcn_v2_0_enc_ring_get_wptr - get enc write pointer 1551 * 1552 * @ring: amdgpu_ring pointer 1553 * 1554 * Returns the current hardware enc write pointer 1555 */ 1556 static uint64_t vcn_v2_0_enc_ring_get_wptr(struct amdgpu_ring *ring) 1557 { 1558 struct amdgpu_device *adev = ring->adev; 1559 1560 if (ring == &adev->vcn.inst->ring_enc[0]) { 1561 if (ring->use_doorbell) 1562 return adev->wb.wb[ring->wptr_offs]; 1563 else 1564 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR); 1565 } else { 1566 if (ring->use_doorbell) 1567 return adev->wb.wb[ring->wptr_offs]; 1568 else 1569 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2); 1570 } 1571 } 1572 1573 /** 1574 * vcn_v2_0_enc_ring_set_wptr - set enc write pointer 1575 * 1576 * @ring: amdgpu_ring pointer 1577 * 1578 * Commits the enc write pointer to the hardware 1579 */ 1580 static void vcn_v2_0_enc_ring_set_wptr(struct amdgpu_ring *ring) 1581 { 1582 struct amdgpu_device *adev = ring->adev; 1583 1584 if (ring == &adev->vcn.inst->ring_enc[0]) { 1585 if (ring->use_doorbell) { 1586 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 1587 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1588 } else { 1589 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr)); 1590 } 1591 } else { 1592 if (ring->use_doorbell) { 1593 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 1594 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1595 } else { 1596 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr)); 1597 } 1598 } 1599 } 1600 1601 /** 1602 * vcn_v2_0_enc_ring_emit_fence - emit an enc fence & trap command 1603 * 1604 * @ring: amdgpu_ring pointer 1605 * @addr: address 1606 * @seq: sequence number 1607 * @flags: fence related flags 1608 * 1609 * Write enc a fence and a trap command to the ring. 1610 */ 1611 void vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 1612 u64 seq, unsigned flags) 1613 { 1614 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 1615 1616 amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE); 1617 amdgpu_ring_write(ring, addr); 1618 amdgpu_ring_write(ring, upper_32_bits(addr)); 1619 amdgpu_ring_write(ring, seq); 1620 amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP); 1621 } 1622 1623 void vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring *ring) 1624 { 1625 amdgpu_ring_write(ring, VCN_ENC_CMD_END); 1626 } 1627 1628 /** 1629 * vcn_v2_0_enc_ring_emit_ib - enc execute indirect buffer 1630 * 1631 * @ring: amdgpu_ring pointer 1632 * @job: job to retrive vmid from 1633 * @ib: indirect buffer to execute 1634 * @flags: unused 1635 * 1636 * Write enc ring commands to execute the indirect buffer 1637 */ 1638 void vcn_v2_0_enc_ring_emit_ib(struct amdgpu_ring *ring, 1639 struct amdgpu_job *job, 1640 struct amdgpu_ib *ib, 1641 uint32_t flags) 1642 { 1643 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 1644 1645 amdgpu_ring_write(ring, VCN_ENC_CMD_IB); 1646 amdgpu_ring_write(ring, vmid); 1647 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)); 1648 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 1649 amdgpu_ring_write(ring, ib->length_dw); 1650 } 1651 1652 void vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 1653 uint32_t val, uint32_t mask) 1654 { 1655 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); 1656 amdgpu_ring_write(ring, reg << 2); 1657 amdgpu_ring_write(ring, mask); 1658 amdgpu_ring_write(ring, val); 1659 } 1660 1661 void vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1662 unsigned int vmid, uint64_t pd_addr) 1663 { 1664 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub]; 1665 1666 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 1667 1668 /* wait for reg writes */ 1669 vcn_v2_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + 1670 vmid * hub->ctx_addr_distance, 1671 lower_32_bits(pd_addr), 0xffffffff); 1672 } 1673 1674 void vcn_v2_0_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) 1675 { 1676 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); 1677 amdgpu_ring_write(ring, reg << 2); 1678 amdgpu_ring_write(ring, val); 1679 } 1680 1681 static int vcn_v2_0_set_interrupt_state(struct amdgpu_device *adev, 1682 struct amdgpu_irq_src *source, 1683 unsigned type, 1684 enum amdgpu_interrupt_state state) 1685 { 1686 return 0; 1687 } 1688 1689 static int vcn_v2_0_process_interrupt(struct amdgpu_device *adev, 1690 struct amdgpu_irq_src *source, 1691 struct amdgpu_iv_entry *entry) 1692 { 1693 DRM_DEBUG("IH: VCN TRAP\n"); 1694 1695 switch (entry->src_id) { 1696 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT: 1697 amdgpu_fence_process(&adev->vcn.inst->ring_dec); 1698 break; 1699 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE: 1700 amdgpu_fence_process(&adev->vcn.inst->ring_enc[0]); 1701 break; 1702 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY: 1703 amdgpu_fence_process(&adev->vcn.inst->ring_enc[1]); 1704 break; 1705 default: 1706 DRM_ERROR("Unhandled interrupt: %d %d\n", 1707 entry->src_id, entry->src_data[0]); 1708 break; 1709 } 1710 1711 return 0; 1712 } 1713 1714 int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring) 1715 { 1716 struct amdgpu_device *adev = ring->adev; 1717 uint32_t tmp = 0; 1718 unsigned i; 1719 int r; 1720 1721 if (amdgpu_sriov_vf(adev)) 1722 return 0; 1723 1724 WREG32(adev->vcn.inst[ring->me].external.scratch9, 0xCAFEDEAD); 1725 r = amdgpu_ring_alloc(ring, 4); 1726 if (r) 1727 return r; 1728 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0)); 1729 amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1)); 1730 amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0)); 1731 amdgpu_ring_write(ring, 0xDEADBEEF); 1732 amdgpu_ring_commit(ring); 1733 for (i = 0; i < adev->usec_timeout; i++) { 1734 tmp = RREG32(adev->vcn.inst[ring->me].external.scratch9); 1735 if (tmp == 0xDEADBEEF) 1736 break; 1737 udelay(1); 1738 } 1739 1740 if (i >= adev->usec_timeout) 1741 r = -ETIMEDOUT; 1742 1743 return r; 1744 } 1745 1746 1747 static int vcn_v2_0_set_powergating_state(void *handle, 1748 enum amd_powergating_state state) 1749 { 1750 /* This doesn't actually powergate the VCN block. 1751 * That's done in the dpm code via the SMC. This 1752 * just re-inits the block as necessary. The actual 1753 * gating still happens in the dpm code. We should 1754 * revisit this when there is a cleaner line between 1755 * the smc and the hw blocks 1756 */ 1757 int ret; 1758 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1759 1760 if (amdgpu_sriov_vf(adev)) { 1761 adev->vcn.cur_state = AMD_PG_STATE_UNGATE; 1762 return 0; 1763 } 1764 1765 if (state == adev->vcn.cur_state) 1766 return 0; 1767 1768 if (state == AMD_PG_STATE_GATE) 1769 ret = vcn_v2_0_stop(adev); 1770 else 1771 ret = vcn_v2_0_start(adev); 1772 1773 if (!ret) 1774 adev->vcn.cur_state = state; 1775 return ret; 1776 } 1777 1778 static int vcn_v2_0_start_mmsch(struct amdgpu_device *adev, 1779 struct amdgpu_mm_table *table) 1780 { 1781 uint32_t data = 0, loop; 1782 uint64_t addr = table->gpu_addr; 1783 struct mmsch_v2_0_init_header *header; 1784 uint32_t size; 1785 int i; 1786 1787 header = (struct mmsch_v2_0_init_header *)table->cpu_addr; 1788 size = header->header_size + header->vcn_table_size; 1789 1790 /* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr 1791 * of memory descriptor location 1792 */ 1793 WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr)); 1794 WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr)); 1795 1796 /* 2, update vmid of descriptor */ 1797 data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_VMID); 1798 data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; 1799 /* use domain0 for MM scheduler */ 1800 data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); 1801 WREG32_SOC15(UVD, 0, mmMMSCH_VF_VMID, data); 1802 1803 /* 3, notify mmsch about the size of this descriptor */ 1804 WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_SIZE, size); 1805 1806 /* 4, set resp to zero */ 1807 WREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP, 0); 1808 1809 adev->vcn.inst->ring_dec.wptr = 0; 1810 adev->vcn.inst->ring_dec.wptr_old = 0; 1811 vcn_v2_0_dec_ring_set_wptr(&adev->vcn.inst->ring_dec); 1812 1813 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 1814 adev->vcn.inst->ring_enc[i].wptr = 0; 1815 adev->vcn.inst->ring_enc[i].wptr_old = 0; 1816 vcn_v2_0_enc_ring_set_wptr(&adev->vcn.inst->ring_enc[i]); 1817 } 1818 1819 /* 5, kick off the initialization and wait until 1820 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero 1821 */ 1822 WREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001); 1823 1824 data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP); 1825 loop = 1000; 1826 while ((data & 0x10000002) != 0x10000002) { 1827 udelay(10); 1828 data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP); 1829 loop--; 1830 if (!loop) 1831 break; 1832 } 1833 1834 if (!loop) { 1835 DRM_ERROR("failed to init MMSCH, " \ 1836 "mmMMSCH_VF_MAILBOX_RESP = 0x%08x\n", data); 1837 return -EBUSY; 1838 } 1839 1840 return 0; 1841 } 1842 1843 static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) 1844 { 1845 int r; 1846 uint32_t tmp; 1847 struct amdgpu_ring *ring; 1848 uint32_t offset, size; 1849 uint32_t table_size = 0; 1850 struct mmsch_v2_0_cmd_direct_write direct_wt = { {0} }; 1851 struct mmsch_v2_0_cmd_direct_read_modify_write direct_rd_mod_wt = { {0} }; 1852 struct mmsch_v2_0_cmd_end end = { {0} }; 1853 struct mmsch_v2_0_init_header *header; 1854 uint32_t *init_table = adev->virt.mm_table.cpu_addr; 1855 uint8_t i = 0; 1856 1857 header = (struct mmsch_v2_0_init_header *)init_table; 1858 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE; 1859 direct_rd_mod_wt.cmd_header.command_type = 1860 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; 1861 end.cmd_header.command_type = MMSCH_COMMAND__END; 1862 1863 if (header->vcn_table_offset == 0 && header->vcn_table_size == 0) { 1864 header->version = MMSCH_VERSION; 1865 header->header_size = sizeof(struct mmsch_v2_0_init_header) >> 2; 1866 1867 header->vcn_table_offset = header->header_size; 1868 1869 init_table += header->vcn_table_offset; 1870 1871 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); 1872 1873 MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT( 1874 SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), 1875 0xFFFFFFFF, 0x00000004); 1876 1877 /* mc resume*/ 1878 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1879 MMSCH_V2_0_INSERT_DIRECT_WT( 1880 SOC15_REG_OFFSET(UVD, i, 1881 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1882 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo); 1883 MMSCH_V2_0_INSERT_DIRECT_WT( 1884 SOC15_REG_OFFSET(UVD, i, 1885 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1886 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi); 1887 offset = 0; 1888 } else { 1889 MMSCH_V2_0_INSERT_DIRECT_WT( 1890 SOC15_REG_OFFSET(UVD, i, 1891 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1892 lower_32_bits(adev->vcn.inst->gpu_addr)); 1893 MMSCH_V2_0_INSERT_DIRECT_WT( 1894 SOC15_REG_OFFSET(UVD, i, 1895 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1896 upper_32_bits(adev->vcn.inst->gpu_addr)); 1897 offset = size; 1898 } 1899 1900 MMSCH_V2_0_INSERT_DIRECT_WT( 1901 SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0), 1902 0); 1903 MMSCH_V2_0_INSERT_DIRECT_WT( 1904 SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), 1905 size); 1906 1907 MMSCH_V2_0_INSERT_DIRECT_WT( 1908 SOC15_REG_OFFSET(UVD, i, 1909 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 1910 lower_32_bits(adev->vcn.inst->gpu_addr + offset)); 1911 MMSCH_V2_0_INSERT_DIRECT_WT( 1912 SOC15_REG_OFFSET(UVD, i, 1913 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 1914 upper_32_bits(adev->vcn.inst->gpu_addr + offset)); 1915 MMSCH_V2_0_INSERT_DIRECT_WT( 1916 SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), 1917 0); 1918 MMSCH_V2_0_INSERT_DIRECT_WT( 1919 SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), 1920 AMDGPU_VCN_STACK_SIZE); 1921 1922 MMSCH_V2_0_INSERT_DIRECT_WT( 1923 SOC15_REG_OFFSET(UVD, i, 1924 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 1925 lower_32_bits(adev->vcn.inst->gpu_addr + offset + 1926 AMDGPU_VCN_STACK_SIZE)); 1927 MMSCH_V2_0_INSERT_DIRECT_WT( 1928 SOC15_REG_OFFSET(UVD, i, 1929 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 1930 upper_32_bits(adev->vcn.inst->gpu_addr + offset + 1931 AMDGPU_VCN_STACK_SIZE)); 1932 MMSCH_V2_0_INSERT_DIRECT_WT( 1933 SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), 1934 0); 1935 MMSCH_V2_0_INSERT_DIRECT_WT( 1936 SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2), 1937 AMDGPU_VCN_CONTEXT_SIZE); 1938 1939 for (r = 0; r < adev->vcn.num_enc_rings; ++r) { 1940 ring = &adev->vcn.inst->ring_enc[r]; 1941 ring->wptr = 0; 1942 MMSCH_V2_0_INSERT_DIRECT_WT( 1943 SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), 1944 lower_32_bits(ring->gpu_addr)); 1945 MMSCH_V2_0_INSERT_DIRECT_WT( 1946 SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), 1947 upper_32_bits(ring->gpu_addr)); 1948 MMSCH_V2_0_INSERT_DIRECT_WT( 1949 SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), 1950 ring->ring_size / 4); 1951 } 1952 1953 ring = &adev->vcn.inst->ring_dec; 1954 ring->wptr = 0; 1955 MMSCH_V2_0_INSERT_DIRECT_WT( 1956 SOC15_REG_OFFSET(UVD, i, 1957 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW), 1958 lower_32_bits(ring->gpu_addr)); 1959 MMSCH_V2_0_INSERT_DIRECT_WT( 1960 SOC15_REG_OFFSET(UVD, i, 1961 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH), 1962 upper_32_bits(ring->gpu_addr)); 1963 /* force RBC into idle state */ 1964 tmp = order_base_2(ring->ring_size); 1965 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp); 1966 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1); 1967 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1); 1968 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); 1969 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); 1970 MMSCH_V2_0_INSERT_DIRECT_WT( 1971 SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp); 1972 1973 /* add end packet */ 1974 tmp = sizeof(struct mmsch_v2_0_cmd_end); 1975 memcpy((void *)init_table, &end, tmp); 1976 table_size += (tmp / 4); 1977 header->vcn_table_size = table_size; 1978 1979 } 1980 return vcn_v2_0_start_mmsch(adev, &adev->virt.mm_table); 1981 } 1982 1983 static const struct amd_ip_funcs vcn_v2_0_ip_funcs = { 1984 .name = "vcn_v2_0", 1985 .early_init = vcn_v2_0_early_init, 1986 .late_init = NULL, 1987 .sw_init = vcn_v2_0_sw_init, 1988 .sw_fini = vcn_v2_0_sw_fini, 1989 .hw_init = vcn_v2_0_hw_init, 1990 .hw_fini = vcn_v2_0_hw_fini, 1991 .suspend = vcn_v2_0_suspend, 1992 .resume = vcn_v2_0_resume, 1993 .is_idle = vcn_v2_0_is_idle, 1994 .wait_for_idle = vcn_v2_0_wait_for_idle, 1995 .check_soft_reset = NULL, 1996 .pre_soft_reset = NULL, 1997 .soft_reset = NULL, 1998 .post_soft_reset = NULL, 1999 .set_clockgating_state = vcn_v2_0_set_clockgating_state, 2000 .set_powergating_state = vcn_v2_0_set_powergating_state, 2001 }; 2002 2003 static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = { 2004 .type = AMDGPU_RING_TYPE_VCN_DEC, 2005 .align_mask = 0xf, 2006 .vmhub = AMDGPU_MMHUB_0, 2007 .get_rptr = vcn_v2_0_dec_ring_get_rptr, 2008 .get_wptr = vcn_v2_0_dec_ring_get_wptr, 2009 .set_wptr = vcn_v2_0_dec_ring_set_wptr, 2010 .emit_frame_size = 2011 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 + 2012 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 + 2013 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */ 2014 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */ 2015 6, 2016 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */ 2017 .emit_ib = vcn_v2_0_dec_ring_emit_ib, 2018 .emit_fence = vcn_v2_0_dec_ring_emit_fence, 2019 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush, 2020 .test_ring = vcn_v2_0_dec_ring_test_ring, 2021 .test_ib = amdgpu_vcn_dec_ring_test_ib, 2022 .insert_nop = vcn_v2_0_dec_ring_insert_nop, 2023 .insert_start = vcn_v2_0_dec_ring_insert_start, 2024 .insert_end = vcn_v2_0_dec_ring_insert_end, 2025 .pad_ib = amdgpu_ring_generic_pad_ib, 2026 .begin_use = amdgpu_vcn_ring_begin_use, 2027 .end_use = amdgpu_vcn_ring_end_use, 2028 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg, 2029 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait, 2030 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 2031 }; 2032 2033 static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = { 2034 .type = AMDGPU_RING_TYPE_VCN_ENC, 2035 .align_mask = 0x3f, 2036 .nop = VCN_ENC_CMD_NO_OP, 2037 .vmhub = AMDGPU_MMHUB_0, 2038 .get_rptr = vcn_v2_0_enc_ring_get_rptr, 2039 .get_wptr = vcn_v2_0_enc_ring_get_wptr, 2040 .set_wptr = vcn_v2_0_enc_ring_set_wptr, 2041 .emit_frame_size = 2042 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 2043 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 2044 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 2045 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 2046 1, /* vcn_v2_0_enc_ring_insert_end */ 2047 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 2048 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 2049 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 2050 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush, 2051 .test_ring = amdgpu_vcn_enc_ring_test_ring, 2052 .test_ib = amdgpu_vcn_enc_ring_test_ib, 2053 .insert_nop = amdgpu_ring_insert_nop, 2054 .insert_end = vcn_v2_0_enc_ring_insert_end, 2055 .pad_ib = amdgpu_ring_generic_pad_ib, 2056 .begin_use = amdgpu_vcn_ring_begin_use, 2057 .end_use = amdgpu_vcn_ring_end_use, 2058 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg, 2059 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait, 2060 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 2061 }; 2062 2063 static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev) 2064 { 2065 adev->vcn.inst->ring_dec.funcs = &vcn_v2_0_dec_ring_vm_funcs; 2066 DRM_INFO("VCN decode is enabled in VM mode\n"); 2067 } 2068 2069 static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev) 2070 { 2071 int i; 2072 2073 for (i = 0; i < adev->vcn.num_enc_rings; ++i) 2074 adev->vcn.inst->ring_enc[i].funcs = &vcn_v2_0_enc_ring_vm_funcs; 2075 2076 DRM_INFO("VCN encode is enabled in VM mode\n"); 2077 } 2078 2079 static const struct amdgpu_irq_src_funcs vcn_v2_0_irq_funcs = { 2080 .set = vcn_v2_0_set_interrupt_state, 2081 .process = vcn_v2_0_process_interrupt, 2082 }; 2083 2084 static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev) 2085 { 2086 adev->vcn.inst->irq.num_types = adev->vcn.num_enc_rings + 1; 2087 adev->vcn.inst->irq.funcs = &vcn_v2_0_irq_funcs; 2088 } 2089 2090 const struct amdgpu_ip_block_version vcn_v2_0_ip_block = 2091 { 2092 .type = AMD_IP_BLOCK_TYPE_VCN, 2093 .major = 2, 2094 .minor = 0, 2095 .rev = 0, 2096 .funcs = &vcn_v2_0_ip_funcs, 2097 }; 2098