1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_drv.h> 26 27 #include "amdgpu.h" 28 #include "amdgpu_vcn.h" 29 #include "amdgpu_pm.h" 30 #include "soc15.h" 31 #include "soc15d.h" 32 #include "vcn_v2_0.h" 33 #include "mmsch_v1_0.h" 34 35 #include "vcn/vcn_2_5_offset.h" 36 #include "vcn/vcn_2_5_sh_mask.h" 37 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h" 38 39 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27 40 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f 41 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10 42 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11 43 #define mmUVD_NO_OP_INTERNAL_OFFSET 0x29 44 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66 45 #define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d 46 47 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431 48 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4 49 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5 50 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c 51 52 #define VCN25_MAX_HW_INSTANCES_ARCTURUS 2 53 54 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev); 55 static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev); 56 static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev); 57 static int vcn_v2_5_set_powergating_state(void *handle, 58 enum amd_powergating_state state); 59 static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev, 60 int inst_idx, struct dpg_pause_state *new_state); 61 static int vcn_v2_5_sriov_start(struct amdgpu_device *adev); 62 63 static int amdgpu_ih_clientid_vcns[] = { 64 SOC15_IH_CLIENTID_VCN, 65 SOC15_IH_CLIENTID_VCN1 66 }; 67 68 /** 69 * vcn_v2_5_early_init - set function pointers 70 * 71 * @handle: amdgpu_device pointer 72 * 73 * Set ring and irq function pointers 74 */ 75 static int vcn_v2_5_early_init(void *handle) 76 { 77 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 78 79 if (amdgpu_sriov_vf(adev)) { 80 adev->vcn.num_vcn_inst = 2; 81 adev->vcn.harvest_config = 0; 82 adev->vcn.num_enc_rings = 1; 83 } else { 84 u32 harvest; 85 int i; 86 87 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 88 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING); 89 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK) 90 adev->vcn.harvest_config |= 1 << i; 91 } 92 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 | 93 AMDGPU_VCN_HARVEST_VCN1)) 94 /* both instances are harvested, disable the block */ 95 return -ENOENT; 96 97 adev->vcn.num_enc_rings = 2; 98 } 99 100 vcn_v2_5_set_dec_ring_funcs(adev); 101 vcn_v2_5_set_enc_ring_funcs(adev); 102 vcn_v2_5_set_irq_funcs(adev); 103 104 return 0; 105 } 106 107 /** 108 * vcn_v2_5_sw_init - sw init for VCN block 109 * 110 * @handle: amdgpu_device pointer 111 * 112 * Load firmware and sw initialization 113 */ 114 static int vcn_v2_5_sw_init(void *handle) 115 { 116 struct amdgpu_ring *ring; 117 int i, j, r; 118 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 119 120 for (j = 0; j < adev->vcn.num_vcn_inst; j++) { 121 if (adev->vcn.harvest_config & (1 << j)) 122 continue; 123 /* VCN DEC TRAP */ 124 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j], 125 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[j].irq); 126 if (r) 127 return r; 128 129 /* VCN ENC TRAP */ 130 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 131 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j], 132 i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[j].irq); 133 if (r) 134 return r; 135 } 136 } 137 138 r = amdgpu_vcn_sw_init(adev); 139 if (r) 140 return r; 141 142 amdgpu_vcn_setup_ucode(adev); 143 144 r = amdgpu_vcn_resume(adev); 145 if (r) 146 return r; 147 148 for (j = 0; j < adev->vcn.num_vcn_inst; j++) { 149 volatile struct amdgpu_fw_shared *fw_shared; 150 151 if (adev->vcn.harvest_config & (1 << j)) 152 continue; 153 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET; 154 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET; 155 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET; 156 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET; 157 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET; 158 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET; 159 160 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET; 161 adev->vcn.inst[j].external.scratch9 = SOC15_REG_OFFSET(VCN, j, mmUVD_SCRATCH9); 162 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET; 163 adev->vcn.inst[j].external.data0 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA0); 164 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET; 165 adev->vcn.inst[j].external.data1 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA1); 166 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET; 167 adev->vcn.inst[j].external.cmd = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_CMD); 168 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET; 169 adev->vcn.inst[j].external.nop = SOC15_REG_OFFSET(VCN, j, mmUVD_NO_OP); 170 171 ring = &adev->vcn.inst[j].ring_dec; 172 ring->use_doorbell = true; 173 174 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 175 (amdgpu_sriov_vf(adev) ? 2*j : 8*j); 176 sprintf(ring->name, "vcn_dec_%d", j); 177 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq, 178 0, AMDGPU_RING_PRIO_DEFAULT, NULL); 179 if (r) 180 return r; 181 182 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 183 enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i); 184 185 ring = &adev->vcn.inst[j].ring_enc[i]; 186 ring->use_doorbell = true; 187 188 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 189 (amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j)); 190 191 sprintf(ring->name, "vcn_enc_%d.%d", j, i); 192 r = amdgpu_ring_init(adev, ring, 512, 193 &adev->vcn.inst[j].irq, 0, 194 hw_prio, NULL); 195 if (r) 196 return r; 197 } 198 199 fw_shared = adev->vcn.inst[j].fw_shared.cpu_addr; 200 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG); 201 202 if (amdgpu_vcnfw_log) 203 amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]); 204 } 205 206 if (amdgpu_sriov_vf(adev)) { 207 r = amdgpu_virt_alloc_mm_table(adev); 208 if (r) 209 return r; 210 } 211 212 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 213 adev->vcn.pause_dpg_mode = vcn_v2_5_pause_dpg_mode; 214 215 return 0; 216 } 217 218 /** 219 * vcn_v2_5_sw_fini - sw fini for VCN block 220 * 221 * @handle: amdgpu_device pointer 222 * 223 * VCN suspend and free up sw allocation 224 */ 225 static int vcn_v2_5_sw_fini(void *handle) 226 { 227 int i, r, idx; 228 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 229 volatile struct amdgpu_fw_shared *fw_shared; 230 231 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 232 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 233 if (adev->vcn.harvest_config & (1 << i)) 234 continue; 235 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 236 fw_shared->present_flag_0 = 0; 237 } 238 drm_dev_exit(idx); 239 } 240 241 242 if (amdgpu_sriov_vf(adev)) 243 amdgpu_virt_free_mm_table(adev); 244 245 r = amdgpu_vcn_suspend(adev); 246 if (r) 247 return r; 248 249 r = amdgpu_vcn_sw_fini(adev); 250 251 return r; 252 } 253 254 /** 255 * vcn_v2_5_hw_init - start and test VCN block 256 * 257 * @handle: amdgpu_device pointer 258 * 259 * Initialize the hardware, boot up the VCPU and do some testing 260 */ 261 static int vcn_v2_5_hw_init(void *handle) 262 { 263 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 264 struct amdgpu_ring *ring; 265 int i, j, r = 0; 266 267 if (amdgpu_sriov_vf(adev)) 268 r = vcn_v2_5_sriov_start(adev); 269 270 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) { 271 if (adev->vcn.harvest_config & (1 << j)) 272 continue; 273 274 if (amdgpu_sriov_vf(adev)) { 275 adev->vcn.inst[j].ring_enc[0].sched.ready = true; 276 adev->vcn.inst[j].ring_enc[1].sched.ready = false; 277 adev->vcn.inst[j].ring_enc[2].sched.ready = false; 278 adev->vcn.inst[j].ring_dec.sched.ready = true; 279 } else { 280 281 ring = &adev->vcn.inst[j].ring_dec; 282 283 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, 284 ring->doorbell_index, j); 285 286 r = amdgpu_ring_test_helper(ring); 287 if (r) 288 goto done; 289 290 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 291 ring = &adev->vcn.inst[j].ring_enc[i]; 292 r = amdgpu_ring_test_helper(ring); 293 if (r) 294 goto done; 295 } 296 } 297 } 298 299 done: 300 if (!r) 301 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n", 302 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode"); 303 304 return r; 305 } 306 307 /** 308 * vcn_v2_5_hw_fini - stop the hardware block 309 * 310 * @handle: amdgpu_device pointer 311 * 312 * Stop the VCN block, mark ring as not ready any more 313 */ 314 static int vcn_v2_5_hw_fini(void *handle) 315 { 316 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 317 int i; 318 319 cancel_delayed_work_sync(&adev->vcn.idle_work); 320 321 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 322 if (adev->vcn.harvest_config & (1 << i)) 323 continue; 324 325 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) || 326 (adev->vcn.cur_state != AMD_PG_STATE_GATE && 327 RREG32_SOC15(VCN, i, mmUVD_STATUS))) 328 vcn_v2_5_set_powergating_state(adev, AMD_PG_STATE_GATE); 329 } 330 331 return 0; 332 } 333 334 /** 335 * vcn_v2_5_suspend - suspend VCN block 336 * 337 * @handle: amdgpu_device pointer 338 * 339 * HW fini and suspend VCN block 340 */ 341 static int vcn_v2_5_suspend(void *handle) 342 { 343 int r; 344 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 345 346 r = vcn_v2_5_hw_fini(adev); 347 if (r) 348 return r; 349 350 r = amdgpu_vcn_suspend(adev); 351 352 return r; 353 } 354 355 /** 356 * vcn_v2_5_resume - resume VCN block 357 * 358 * @handle: amdgpu_device pointer 359 * 360 * Resume firmware and hw init VCN block 361 */ 362 static int vcn_v2_5_resume(void *handle) 363 { 364 int r; 365 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 366 367 r = amdgpu_vcn_resume(adev); 368 if (r) 369 return r; 370 371 r = vcn_v2_5_hw_init(adev); 372 373 return r; 374 } 375 376 /** 377 * vcn_v2_5_mc_resume - memory controller programming 378 * 379 * @adev: amdgpu_device pointer 380 * 381 * Let the VCN memory controller know it's offsets 382 */ 383 static void vcn_v2_5_mc_resume(struct amdgpu_device *adev) 384 { 385 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); 386 uint32_t offset; 387 int i; 388 389 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 390 if (adev->vcn.harvest_config & (1 << i)) 391 continue; 392 /* cache window 0: fw */ 393 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 394 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 395 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo)); 396 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 397 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi)); 398 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 0); 399 offset = 0; 400 } else { 401 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 402 lower_32_bits(adev->vcn.inst[i].gpu_addr)); 403 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 404 upper_32_bits(adev->vcn.inst[i].gpu_addr)); 405 offset = size; 406 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 407 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 408 } 409 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE0, size); 410 411 /* cache window 1: stack */ 412 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, 413 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset)); 414 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, 415 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset)); 416 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET1, 0); 417 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE); 418 419 /* cache window 2: context */ 420 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, 421 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); 422 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, 423 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); 424 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET2, 0); 425 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE); 426 427 /* non-cache window */ 428 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, 429 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr)); 430 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, 431 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr)); 432 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_OFFSET0, 0); 433 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_SIZE0, 434 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared))); 435 } 436 } 437 438 static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) 439 { 440 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); 441 uint32_t offset; 442 443 /* cache window 0: fw */ 444 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 445 if (!indirect) { 446 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 447 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 448 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect); 449 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 450 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 451 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect); 452 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 453 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 454 } else { 455 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 456 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); 457 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 458 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); 459 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 460 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 461 } 462 offset = 0; 463 } else { 464 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 465 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 466 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 467 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 468 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 469 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 470 offset = size; 471 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 472 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 473 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); 474 } 475 476 if (!indirect) 477 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 478 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect); 479 else 480 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 481 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); 482 483 /* cache window 1: stack */ 484 if (!indirect) { 485 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 486 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 487 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 488 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 489 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 490 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 491 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 492 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 493 } else { 494 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 495 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); 496 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 497 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); 498 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 499 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 500 } 501 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 502 VCN, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); 503 504 /* cache window 2: context */ 505 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 506 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 507 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect); 508 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 509 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 510 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect); 511 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 512 VCN, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); 513 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 514 VCN, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); 515 516 /* non-cache window */ 517 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 518 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 519 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 520 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 521 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 522 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 523 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 524 VCN, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); 525 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 526 VCN, 0, mmUVD_VCPU_NONCACHE_SIZE0), 527 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect); 528 529 /* VCN global tiling registers */ 530 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 531 VCN, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 532 } 533 534 /** 535 * vcn_v2_5_disable_clock_gating - disable VCN clock gating 536 * 537 * @adev: amdgpu_device pointer 538 * 539 * Disable clock gating for VCN block 540 */ 541 static void vcn_v2_5_disable_clock_gating(struct amdgpu_device *adev) 542 { 543 uint32_t data; 544 int i; 545 546 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 547 if (adev->vcn.harvest_config & (1 << i)) 548 continue; 549 /* UVD disable CGC */ 550 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL); 551 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 552 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 553 else 554 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; 555 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 556 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 557 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data); 558 559 data = RREG32_SOC15(VCN, i, mmUVD_CGC_GATE); 560 data &= ~(UVD_CGC_GATE__SYS_MASK 561 | UVD_CGC_GATE__UDEC_MASK 562 | UVD_CGC_GATE__MPEG2_MASK 563 | UVD_CGC_GATE__REGS_MASK 564 | UVD_CGC_GATE__RBC_MASK 565 | UVD_CGC_GATE__LMI_MC_MASK 566 | UVD_CGC_GATE__LMI_UMC_MASK 567 | UVD_CGC_GATE__IDCT_MASK 568 | UVD_CGC_GATE__MPRD_MASK 569 | UVD_CGC_GATE__MPC_MASK 570 | UVD_CGC_GATE__LBSI_MASK 571 | UVD_CGC_GATE__LRBBM_MASK 572 | UVD_CGC_GATE__UDEC_RE_MASK 573 | UVD_CGC_GATE__UDEC_CM_MASK 574 | UVD_CGC_GATE__UDEC_IT_MASK 575 | UVD_CGC_GATE__UDEC_DB_MASK 576 | UVD_CGC_GATE__UDEC_MP_MASK 577 | UVD_CGC_GATE__WCB_MASK 578 | UVD_CGC_GATE__VCPU_MASK 579 | UVD_CGC_GATE__MMSCH_MASK); 580 581 WREG32_SOC15(VCN, i, mmUVD_CGC_GATE, data); 582 583 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_CGC_GATE, 0, 0xFFFFFFFF); 584 585 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL); 586 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK 587 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK 588 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK 589 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK 590 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK 591 | UVD_CGC_CTRL__SYS_MODE_MASK 592 | UVD_CGC_CTRL__UDEC_MODE_MASK 593 | UVD_CGC_CTRL__MPEG2_MODE_MASK 594 | UVD_CGC_CTRL__REGS_MODE_MASK 595 | UVD_CGC_CTRL__RBC_MODE_MASK 596 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 597 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 598 | UVD_CGC_CTRL__IDCT_MODE_MASK 599 | UVD_CGC_CTRL__MPRD_MODE_MASK 600 | UVD_CGC_CTRL__MPC_MODE_MASK 601 | UVD_CGC_CTRL__LBSI_MODE_MASK 602 | UVD_CGC_CTRL__LRBBM_MODE_MASK 603 | UVD_CGC_CTRL__WCB_MODE_MASK 604 | UVD_CGC_CTRL__VCPU_MODE_MASK 605 | UVD_CGC_CTRL__MMSCH_MODE_MASK); 606 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data); 607 608 /* turn on */ 609 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE); 610 data |= (UVD_SUVD_CGC_GATE__SRE_MASK 611 | UVD_SUVD_CGC_GATE__SIT_MASK 612 | UVD_SUVD_CGC_GATE__SMP_MASK 613 | UVD_SUVD_CGC_GATE__SCM_MASK 614 | UVD_SUVD_CGC_GATE__SDB_MASK 615 | UVD_SUVD_CGC_GATE__SRE_H264_MASK 616 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK 617 | UVD_SUVD_CGC_GATE__SIT_H264_MASK 618 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK 619 | UVD_SUVD_CGC_GATE__SCM_H264_MASK 620 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK 621 | UVD_SUVD_CGC_GATE__SDB_H264_MASK 622 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK 623 | UVD_SUVD_CGC_GATE__SCLR_MASK 624 | UVD_SUVD_CGC_GATE__UVD_SC_MASK 625 | UVD_SUVD_CGC_GATE__ENT_MASK 626 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK 627 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK 628 | UVD_SUVD_CGC_GATE__SITE_MASK 629 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK 630 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK 631 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK 632 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK 633 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); 634 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE, data); 635 636 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL); 637 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 638 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 639 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 640 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 641 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 642 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK 643 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK 644 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 645 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 646 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 647 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data); 648 } 649 } 650 651 static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_device *adev, 652 uint8_t sram_sel, int inst_idx, uint8_t indirect) 653 { 654 uint32_t reg_data = 0; 655 656 /* enable sw clock gating control */ 657 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 658 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 659 else 660 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 661 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 662 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 663 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK | 664 UVD_CGC_CTRL__UDEC_CM_MODE_MASK | 665 UVD_CGC_CTRL__UDEC_IT_MODE_MASK | 666 UVD_CGC_CTRL__UDEC_DB_MODE_MASK | 667 UVD_CGC_CTRL__UDEC_MP_MODE_MASK | 668 UVD_CGC_CTRL__SYS_MODE_MASK | 669 UVD_CGC_CTRL__UDEC_MODE_MASK | 670 UVD_CGC_CTRL__MPEG2_MODE_MASK | 671 UVD_CGC_CTRL__REGS_MODE_MASK | 672 UVD_CGC_CTRL__RBC_MODE_MASK | 673 UVD_CGC_CTRL__LMI_MC_MODE_MASK | 674 UVD_CGC_CTRL__LMI_UMC_MODE_MASK | 675 UVD_CGC_CTRL__IDCT_MODE_MASK | 676 UVD_CGC_CTRL__MPRD_MODE_MASK | 677 UVD_CGC_CTRL__MPC_MODE_MASK | 678 UVD_CGC_CTRL__LBSI_MODE_MASK | 679 UVD_CGC_CTRL__LRBBM_MODE_MASK | 680 UVD_CGC_CTRL__WCB_MODE_MASK | 681 UVD_CGC_CTRL__VCPU_MODE_MASK | 682 UVD_CGC_CTRL__MMSCH_MODE_MASK); 683 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 684 VCN, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect); 685 686 /* turn off clock gating */ 687 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 688 VCN, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect); 689 690 /* turn on SUVD clock gating */ 691 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 692 VCN, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); 693 694 /* turn on sw mode in UVD_SUVD_CGC_CTRL */ 695 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 696 VCN, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); 697 } 698 699 /** 700 * vcn_v2_5_enable_clock_gating - enable VCN clock gating 701 * 702 * @adev: amdgpu_device pointer 703 * 704 * Enable clock gating for VCN block 705 */ 706 static void vcn_v2_5_enable_clock_gating(struct amdgpu_device *adev) 707 { 708 uint32_t data = 0; 709 int i; 710 711 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 712 if (adev->vcn.harvest_config & (1 << i)) 713 continue; 714 /* enable UVD CGC */ 715 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL); 716 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 717 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 718 else 719 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 720 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 721 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 722 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data); 723 724 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL); 725 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK 726 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK 727 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK 728 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK 729 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK 730 | UVD_CGC_CTRL__SYS_MODE_MASK 731 | UVD_CGC_CTRL__UDEC_MODE_MASK 732 | UVD_CGC_CTRL__MPEG2_MODE_MASK 733 | UVD_CGC_CTRL__REGS_MODE_MASK 734 | UVD_CGC_CTRL__RBC_MODE_MASK 735 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 736 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 737 | UVD_CGC_CTRL__IDCT_MODE_MASK 738 | UVD_CGC_CTRL__MPRD_MODE_MASK 739 | UVD_CGC_CTRL__MPC_MODE_MASK 740 | UVD_CGC_CTRL__LBSI_MODE_MASK 741 | UVD_CGC_CTRL__LRBBM_MODE_MASK 742 | UVD_CGC_CTRL__WCB_MODE_MASK 743 | UVD_CGC_CTRL__VCPU_MODE_MASK); 744 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data); 745 746 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL); 747 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 748 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 749 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 750 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 751 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 752 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK 753 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK 754 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 755 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 756 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 757 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data); 758 } 759 } 760 761 static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect) 762 { 763 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 764 struct amdgpu_ring *ring; 765 uint32_t rb_bufsz, tmp; 766 767 /* disable register anti-hang mechanism */ 768 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1, 769 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 770 /* enable dynamic power gating mode */ 771 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS); 772 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; 773 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; 774 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp); 775 776 if (indirect) 777 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; 778 779 /* enable clock gating */ 780 vcn_v2_5_clock_gating_dpg_mode(adev, 0, inst_idx, indirect); 781 782 /* enable VCPU clock */ 783 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 784 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 785 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; 786 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 787 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect); 788 789 /* disable master interupt */ 790 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 791 VCN, 0, mmUVD_MASTINT_EN), 0, 0, indirect); 792 793 /* setup mmUVD_LMI_CTRL */ 794 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 795 UVD_LMI_CTRL__REQ_MODE_MASK | 796 UVD_LMI_CTRL__CRC_RESET_MASK | 797 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 798 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 799 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | 800 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 801 0x00100000L); 802 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 803 VCN, 0, mmUVD_LMI_CTRL), tmp, 0, indirect); 804 805 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 806 VCN, 0, mmUVD_MPC_CNTL), 807 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); 808 809 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 810 VCN, 0, mmUVD_MPC_SET_MUXA0), 811 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 812 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 813 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 814 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); 815 816 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 817 VCN, 0, mmUVD_MPC_SET_MUXB0), 818 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 819 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 820 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 821 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); 822 823 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 824 VCN, 0, mmUVD_MPC_SET_MUX), 825 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 826 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 827 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); 828 829 vcn_v2_5_mc_resume_dpg_mode(adev, inst_idx, indirect); 830 831 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 832 VCN, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect); 833 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 834 VCN, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect); 835 836 /* enable LMI MC and UMC channels */ 837 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 838 VCN, 0, mmUVD_LMI_CTRL2), 0, 0, indirect); 839 840 /* unblock VCPU register access */ 841 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 842 VCN, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect); 843 844 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 845 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 846 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 847 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect); 848 849 /* enable master interrupt */ 850 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 851 VCN, 0, mmUVD_MASTINT_EN), 852 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); 853 854 if (indirect) 855 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr, 856 (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr - 857 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr)); 858 859 ring = &adev->vcn.inst[inst_idx].ring_dec; 860 /* force RBC into idle state */ 861 rb_bufsz = order_base_2(ring->ring_size); 862 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz); 863 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1); 864 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1); 865 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); 866 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); 867 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp); 868 869 /* Stall DPG before WPTR/RPTR reset */ 870 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 871 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK, 872 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 873 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET; 874 875 /* set the write pointer delay */ 876 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0); 877 878 /* set the wb address */ 879 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR, 880 (upper_32_bits(ring->gpu_addr) >> 2)); 881 882 /* program the RB_BASE for ring buffer */ 883 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW, 884 lower_32_bits(ring->gpu_addr)); 885 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH, 886 upper_32_bits(ring->gpu_addr)); 887 888 /* Initialize the ring buffer's read and write pointers */ 889 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0); 890 891 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0); 892 893 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR); 894 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, 895 lower_32_bits(ring->wptr)); 896 897 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET; 898 /* Unstall DPG */ 899 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 900 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 901 902 return 0; 903 } 904 905 static int vcn_v2_5_start(struct amdgpu_device *adev) 906 { 907 struct amdgpu_ring *ring; 908 uint32_t rb_bufsz, tmp; 909 int i, j, k, r; 910 911 if (adev->pm.dpm_enabled) 912 amdgpu_dpm_enable_uvd(adev, true); 913 914 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 915 if (adev->vcn.harvest_config & (1 << i)) 916 continue; 917 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 918 r = vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram); 919 continue; 920 } 921 922 /* disable register anti-hang mechanism */ 923 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 0, 924 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 925 926 /* set uvd status busy */ 927 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY; 928 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp); 929 } 930 931 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 932 return 0; 933 934 /*SW clock gating */ 935 vcn_v2_5_disable_clock_gating(adev); 936 937 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 938 if (adev->vcn.harvest_config & (1 << i)) 939 continue; 940 /* enable VCPU clock */ 941 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 942 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK); 943 944 /* disable master interrupt */ 945 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0, 946 ~UVD_MASTINT_EN__VCPU_EN_MASK); 947 948 /* setup mmUVD_LMI_CTRL */ 949 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL); 950 tmp &= ~0xff; 951 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp | 0x8| 952 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 953 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 954 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 955 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); 956 957 /* setup mmUVD_MPC_CNTL */ 958 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL); 959 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; 960 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; 961 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp); 962 963 /* setup UVD_MPC_SET_MUXA0 */ 964 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0, 965 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 966 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 967 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 968 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); 969 970 /* setup UVD_MPC_SET_MUXB0 */ 971 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0, 972 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 973 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 974 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 975 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); 976 977 /* setup mmUVD_MPC_SET_MUX */ 978 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX, 979 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 980 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 981 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); 982 } 983 984 vcn_v2_5_mc_resume(adev); 985 986 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 987 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 988 if (adev->vcn.harvest_config & (1 << i)) 989 continue; 990 /* VCN global tiling registers */ 991 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG, 992 adev->gfx.config.gb_addr_config); 993 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG, 994 adev->gfx.config.gb_addr_config); 995 996 /* enable LMI MC and UMC channels */ 997 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0, 998 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); 999 1000 /* unblock VCPU register access */ 1001 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0, 1002 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1003 1004 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0, 1005 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1006 1007 for (k = 0; k < 10; ++k) { 1008 uint32_t status; 1009 1010 for (j = 0; j < 100; ++j) { 1011 status = RREG32_SOC15(VCN, i, mmUVD_STATUS); 1012 if (status & 2) 1013 break; 1014 if (amdgpu_emu_mode == 1) 1015 msleep(500); 1016 else 1017 mdelay(10); 1018 } 1019 r = 0; 1020 if (status & 2) 1021 break; 1022 1023 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n"); 1024 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 1025 UVD_VCPU_CNTL__BLK_RST_MASK, 1026 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1027 mdelay(10); 1028 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0, 1029 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1030 1031 mdelay(10); 1032 r = -1; 1033 } 1034 1035 if (r) { 1036 DRM_ERROR("VCN decode not responding, giving up!!!\n"); 1037 return r; 1038 } 1039 1040 /* enable master interrupt */ 1041 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 1042 UVD_MASTINT_EN__VCPU_EN_MASK, 1043 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1044 1045 /* clear the busy bit of VCN_STATUS */ 1046 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0, 1047 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); 1048 1049 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0); 1050 1051 ring = &adev->vcn.inst[i].ring_dec; 1052 /* force RBC into idle state */ 1053 rb_bufsz = order_base_2(ring->ring_size); 1054 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz); 1055 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1); 1056 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1); 1057 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); 1058 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); 1059 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp); 1060 1061 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET; 1062 /* program the RB_BASE for ring buffer */ 1063 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW, 1064 lower_32_bits(ring->gpu_addr)); 1065 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH, 1066 upper_32_bits(ring->gpu_addr)); 1067 1068 /* Initialize the ring buffer's read and write pointers */ 1069 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0); 1070 1071 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR); 1072 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR, 1073 lower_32_bits(ring->wptr)); 1074 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET; 1075 1076 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET; 1077 ring = &adev->vcn.inst[i].ring_enc[0]; 1078 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr)); 1079 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr)); 1080 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr); 1081 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); 1082 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4); 1083 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET; 1084 1085 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET; 1086 ring = &adev->vcn.inst[i].ring_enc[1]; 1087 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr)); 1088 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr)); 1089 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr); 1090 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); 1091 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4); 1092 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET; 1093 } 1094 1095 return 0; 1096 } 1097 1098 static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev, 1099 struct amdgpu_mm_table *table) 1100 { 1101 uint32_t data = 0, loop = 0, size = 0; 1102 uint64_t addr = table->gpu_addr; 1103 struct mmsch_v1_1_init_header *header = NULL; 1104 1105 header = (struct mmsch_v1_1_init_header *)table->cpu_addr; 1106 size = header->total_size; 1107 1108 /* 1109 * 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of 1110 * memory descriptor location 1111 */ 1112 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr)); 1113 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr)); 1114 1115 /* 2, update vmid of descriptor */ 1116 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID); 1117 data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; 1118 /* use domain0 for MM scheduler */ 1119 data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); 1120 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, data); 1121 1122 /* 3, notify mmsch about the size of this descriptor */ 1123 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size); 1124 1125 /* 4, set resp to zero */ 1126 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0); 1127 1128 /* 1129 * 5, kick off the initialization and wait until 1130 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero 1131 */ 1132 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001); 1133 1134 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP); 1135 loop = 10; 1136 while ((data & 0x10000002) != 0x10000002) { 1137 udelay(100); 1138 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP); 1139 loop--; 1140 if (!loop) 1141 break; 1142 } 1143 1144 if (!loop) { 1145 dev_err(adev->dev, 1146 "failed to init MMSCH, mmMMSCH_VF_MAILBOX_RESP = %x\n", 1147 data); 1148 return -EBUSY; 1149 } 1150 1151 return 0; 1152 } 1153 1154 static int vcn_v2_5_sriov_start(struct amdgpu_device *adev) 1155 { 1156 struct amdgpu_ring *ring; 1157 uint32_t offset, size, tmp, i, rb_bufsz; 1158 uint32_t table_size = 0; 1159 struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } }; 1160 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } }; 1161 struct mmsch_v1_0_cmd_end end = { { 0 } }; 1162 uint32_t *init_table = adev->virt.mm_table.cpu_addr; 1163 struct mmsch_v1_1_init_header *header = (struct mmsch_v1_1_init_header *)init_table; 1164 1165 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE; 1166 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; 1167 end.cmd_header.command_type = MMSCH_COMMAND__END; 1168 1169 header->version = MMSCH_VERSION; 1170 header->total_size = sizeof(struct mmsch_v1_1_init_header) >> 2; 1171 init_table += header->total_size; 1172 1173 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1174 header->eng[i].table_offset = header->total_size; 1175 header->eng[i].init_status = 0; 1176 header->eng[i].table_size = 0; 1177 1178 table_size = 0; 1179 1180 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT( 1181 SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 1182 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); 1183 1184 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); 1185 /* mc resume*/ 1186 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1187 MMSCH_V1_0_INSERT_DIRECT_WT( 1188 SOC15_REG_OFFSET(VCN, i, 1189 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1190 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); 1191 MMSCH_V1_0_INSERT_DIRECT_WT( 1192 SOC15_REG_OFFSET(VCN, i, 1193 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1194 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); 1195 offset = 0; 1196 MMSCH_V1_0_INSERT_DIRECT_WT( 1197 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 0); 1198 } else { 1199 MMSCH_V1_0_INSERT_DIRECT_WT( 1200 SOC15_REG_OFFSET(VCN, i, 1201 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1202 lower_32_bits(adev->vcn.inst[i].gpu_addr)); 1203 MMSCH_V1_0_INSERT_DIRECT_WT( 1204 SOC15_REG_OFFSET(VCN, i, 1205 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1206 upper_32_bits(adev->vcn.inst[i].gpu_addr)); 1207 offset = size; 1208 MMSCH_V1_0_INSERT_DIRECT_WT( 1209 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 1210 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 1211 } 1212 1213 MMSCH_V1_0_INSERT_DIRECT_WT( 1214 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE0), 1215 size); 1216 MMSCH_V1_0_INSERT_DIRECT_WT( 1217 SOC15_REG_OFFSET(VCN, i, 1218 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 1219 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset)); 1220 MMSCH_V1_0_INSERT_DIRECT_WT( 1221 SOC15_REG_OFFSET(VCN, i, 1222 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 1223 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset)); 1224 MMSCH_V1_0_INSERT_DIRECT_WT( 1225 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET1), 1226 0); 1227 MMSCH_V1_0_INSERT_DIRECT_WT( 1228 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE1), 1229 AMDGPU_VCN_STACK_SIZE); 1230 MMSCH_V1_0_INSERT_DIRECT_WT( 1231 SOC15_REG_OFFSET(VCN, i, 1232 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 1233 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + 1234 AMDGPU_VCN_STACK_SIZE)); 1235 MMSCH_V1_0_INSERT_DIRECT_WT( 1236 SOC15_REG_OFFSET(VCN, i, 1237 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 1238 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + 1239 AMDGPU_VCN_STACK_SIZE)); 1240 MMSCH_V1_0_INSERT_DIRECT_WT( 1241 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET2), 1242 0); 1243 MMSCH_V1_0_INSERT_DIRECT_WT( 1244 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE2), 1245 AMDGPU_VCN_CONTEXT_SIZE); 1246 1247 ring = &adev->vcn.inst[i].ring_enc[0]; 1248 ring->wptr = 0; 1249 1250 MMSCH_V1_0_INSERT_DIRECT_WT( 1251 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_LO), 1252 lower_32_bits(ring->gpu_addr)); 1253 MMSCH_V1_0_INSERT_DIRECT_WT( 1254 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_HI), 1255 upper_32_bits(ring->gpu_addr)); 1256 MMSCH_V1_0_INSERT_DIRECT_WT( 1257 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_SIZE), 1258 ring->ring_size / 4); 1259 1260 ring = &adev->vcn.inst[i].ring_dec; 1261 ring->wptr = 0; 1262 MMSCH_V1_0_INSERT_DIRECT_WT( 1263 SOC15_REG_OFFSET(VCN, i, 1264 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW), 1265 lower_32_bits(ring->gpu_addr)); 1266 MMSCH_V1_0_INSERT_DIRECT_WT( 1267 SOC15_REG_OFFSET(VCN, i, 1268 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH), 1269 upper_32_bits(ring->gpu_addr)); 1270 1271 /* force RBC into idle state */ 1272 rb_bufsz = order_base_2(ring->ring_size); 1273 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz); 1274 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1); 1275 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1); 1276 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); 1277 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); 1278 MMSCH_V1_0_INSERT_DIRECT_WT( 1279 SOC15_REG_OFFSET(VCN, i, mmUVD_RBC_RB_CNTL), tmp); 1280 1281 /* add end packet */ 1282 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end)); 1283 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4; 1284 init_table += sizeof(struct mmsch_v1_0_cmd_end) / 4; 1285 1286 /* refine header */ 1287 header->eng[i].table_size = table_size; 1288 header->total_size += table_size; 1289 } 1290 1291 return vcn_v2_5_mmsch_start(adev, &adev->virt.mm_table); 1292 } 1293 1294 static int vcn_v2_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx) 1295 { 1296 uint32_t tmp; 1297 1298 /* Wait for power status to be 1 */ 1299 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1, 1300 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1301 1302 /* wait for read ptr to be equal to write ptr */ 1303 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR); 1304 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF); 1305 1306 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2); 1307 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF); 1308 1309 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF; 1310 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF); 1311 1312 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1, 1313 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1314 1315 /* disable dynamic power gating mode */ 1316 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0, 1317 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); 1318 1319 return 0; 1320 } 1321 1322 static int vcn_v2_5_stop(struct amdgpu_device *adev) 1323 { 1324 uint32_t tmp; 1325 int i, r = 0; 1326 1327 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1328 if (adev->vcn.harvest_config & (1 << i)) 1329 continue; 1330 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 1331 r = vcn_v2_5_stop_dpg_mode(adev, i); 1332 continue; 1333 } 1334 1335 /* wait for vcn idle */ 1336 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7); 1337 if (r) 1338 return r; 1339 1340 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | 1341 UVD_LMI_STATUS__READ_CLEAN_MASK | 1342 UVD_LMI_STATUS__WRITE_CLEAN_MASK | 1343 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; 1344 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp); 1345 if (r) 1346 return r; 1347 1348 /* block LMI UMC channel */ 1349 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2); 1350 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; 1351 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp); 1352 1353 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK| 1354 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; 1355 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp); 1356 if (r) 1357 return r; 1358 1359 /* block VCPU register access */ 1360 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 1361 UVD_RB_ARB_CTRL__VCPU_DIS_MASK, 1362 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1363 1364 /* reset VCPU */ 1365 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 1366 UVD_VCPU_CNTL__BLK_RST_MASK, 1367 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1368 1369 /* disable VCPU clock */ 1370 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0, 1371 ~(UVD_VCPU_CNTL__CLK_EN_MASK)); 1372 1373 /* clear status */ 1374 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0); 1375 1376 vcn_v2_5_enable_clock_gating(adev); 1377 1378 /* enable register anti-hang mechanism */ 1379 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 1380 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, 1381 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1382 } 1383 1384 if (adev->pm.dpm_enabled) 1385 amdgpu_dpm_enable_uvd(adev, false); 1386 1387 return 0; 1388 } 1389 1390 static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev, 1391 int inst_idx, struct dpg_pause_state *new_state) 1392 { 1393 struct amdgpu_ring *ring; 1394 uint32_t reg_data = 0; 1395 int ret_code = 0; 1396 1397 /* pause/unpause if state is changed */ 1398 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) { 1399 DRM_DEBUG("dpg pause state changed %d -> %d", 1400 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based); 1401 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) & 1402 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK); 1403 1404 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) { 1405 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1, 1406 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1407 1408 if (!ret_code) { 1409 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 1410 1411 /* pause DPG */ 1412 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK; 1413 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data); 1414 1415 /* wait for ACK */ 1416 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE, 1417 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, 1418 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK); 1419 1420 /* Stall DPG before WPTR/RPTR reset */ 1421 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1422 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK, 1423 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 1424 1425 /* Restore */ 1426 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET; 1427 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 1428 ring->wptr = 0; 1429 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr); 1430 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); 1431 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4); 1432 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr)); 1433 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr)); 1434 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET; 1435 1436 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET; 1437 ring = &adev->vcn.inst[inst_idx].ring_enc[1]; 1438 ring->wptr = 0; 1439 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr); 1440 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); 1441 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4); 1442 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr)); 1443 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr)); 1444 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET; 1445 1446 /* Unstall DPG */ 1447 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1448 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK); 1449 1450 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1451 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1452 } 1453 } else { 1454 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK; 1455 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data); 1456 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1, 1457 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1458 } 1459 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based; 1460 } 1461 1462 return 0; 1463 } 1464 1465 /** 1466 * vcn_v2_5_dec_ring_get_rptr - get read pointer 1467 * 1468 * @ring: amdgpu_ring pointer 1469 * 1470 * Returns the current hardware read pointer 1471 */ 1472 static uint64_t vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring *ring) 1473 { 1474 struct amdgpu_device *adev = ring->adev; 1475 1476 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR); 1477 } 1478 1479 /** 1480 * vcn_v2_5_dec_ring_get_wptr - get write pointer 1481 * 1482 * @ring: amdgpu_ring pointer 1483 * 1484 * Returns the current hardware write pointer 1485 */ 1486 static uint64_t vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring *ring) 1487 { 1488 struct amdgpu_device *adev = ring->adev; 1489 1490 if (ring->use_doorbell) 1491 return adev->wb.wb[ring->wptr_offs]; 1492 else 1493 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR); 1494 } 1495 1496 /** 1497 * vcn_v2_5_dec_ring_set_wptr - set write pointer 1498 * 1499 * @ring: amdgpu_ring pointer 1500 * 1501 * Commits the write pointer to the hardware 1502 */ 1503 static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring) 1504 { 1505 struct amdgpu_device *adev = ring->adev; 1506 1507 if (ring->use_doorbell) { 1508 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 1509 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1510 } else { 1511 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr)); 1512 } 1513 } 1514 1515 static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = { 1516 .type = AMDGPU_RING_TYPE_VCN_DEC, 1517 .align_mask = 0xf, 1518 .secure_submission_supported = true, 1519 .vmhub = AMDGPU_MMHUB_1, 1520 .get_rptr = vcn_v2_5_dec_ring_get_rptr, 1521 .get_wptr = vcn_v2_5_dec_ring_get_wptr, 1522 .set_wptr = vcn_v2_5_dec_ring_set_wptr, 1523 .emit_frame_size = 1524 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 + 1525 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 + 1526 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */ 1527 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */ 1528 6, 1529 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */ 1530 .emit_ib = vcn_v2_0_dec_ring_emit_ib, 1531 .emit_fence = vcn_v2_0_dec_ring_emit_fence, 1532 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush, 1533 .test_ring = vcn_v2_0_dec_ring_test_ring, 1534 .test_ib = amdgpu_vcn_dec_ring_test_ib, 1535 .insert_nop = vcn_v2_0_dec_ring_insert_nop, 1536 .insert_start = vcn_v2_0_dec_ring_insert_start, 1537 .insert_end = vcn_v2_0_dec_ring_insert_end, 1538 .pad_ib = amdgpu_ring_generic_pad_ib, 1539 .begin_use = amdgpu_vcn_ring_begin_use, 1540 .end_use = amdgpu_vcn_ring_end_use, 1541 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg, 1542 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait, 1543 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1544 }; 1545 1546 static const struct amdgpu_ring_funcs vcn_v2_6_dec_ring_vm_funcs = { 1547 .type = AMDGPU_RING_TYPE_VCN_DEC, 1548 .align_mask = 0xf, 1549 .secure_submission_supported = true, 1550 .vmhub = AMDGPU_MMHUB_0, 1551 .get_rptr = vcn_v2_5_dec_ring_get_rptr, 1552 .get_wptr = vcn_v2_5_dec_ring_get_wptr, 1553 .set_wptr = vcn_v2_5_dec_ring_set_wptr, 1554 .emit_frame_size = 1555 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 + 1556 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 + 1557 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */ 1558 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */ 1559 6, 1560 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */ 1561 .emit_ib = vcn_v2_0_dec_ring_emit_ib, 1562 .emit_fence = vcn_v2_0_dec_ring_emit_fence, 1563 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush, 1564 .test_ring = vcn_v2_0_dec_ring_test_ring, 1565 .test_ib = amdgpu_vcn_dec_ring_test_ib, 1566 .insert_nop = vcn_v2_0_dec_ring_insert_nop, 1567 .insert_start = vcn_v2_0_dec_ring_insert_start, 1568 .insert_end = vcn_v2_0_dec_ring_insert_end, 1569 .pad_ib = amdgpu_ring_generic_pad_ib, 1570 .begin_use = amdgpu_vcn_ring_begin_use, 1571 .end_use = amdgpu_vcn_ring_end_use, 1572 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg, 1573 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait, 1574 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1575 }; 1576 1577 /** 1578 * vcn_v2_5_enc_ring_get_rptr - get enc read pointer 1579 * 1580 * @ring: amdgpu_ring pointer 1581 * 1582 * Returns the current hardware enc read pointer 1583 */ 1584 static uint64_t vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring *ring) 1585 { 1586 struct amdgpu_device *adev = ring->adev; 1587 1588 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) 1589 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR); 1590 else 1591 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2); 1592 } 1593 1594 /** 1595 * vcn_v2_5_enc_ring_get_wptr - get enc write pointer 1596 * 1597 * @ring: amdgpu_ring pointer 1598 * 1599 * Returns the current hardware enc write pointer 1600 */ 1601 static uint64_t vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring *ring) 1602 { 1603 struct amdgpu_device *adev = ring->adev; 1604 1605 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) { 1606 if (ring->use_doorbell) 1607 return adev->wb.wb[ring->wptr_offs]; 1608 else 1609 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR); 1610 } else { 1611 if (ring->use_doorbell) 1612 return adev->wb.wb[ring->wptr_offs]; 1613 else 1614 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2); 1615 } 1616 } 1617 1618 /** 1619 * vcn_v2_5_enc_ring_set_wptr - set enc write pointer 1620 * 1621 * @ring: amdgpu_ring pointer 1622 * 1623 * Commits the enc write pointer to the hardware 1624 */ 1625 static void vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring *ring) 1626 { 1627 struct amdgpu_device *adev = ring->adev; 1628 1629 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) { 1630 if (ring->use_doorbell) { 1631 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 1632 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1633 } else { 1634 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr)); 1635 } 1636 } else { 1637 if (ring->use_doorbell) { 1638 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 1639 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1640 } else { 1641 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr)); 1642 } 1643 } 1644 } 1645 1646 static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = { 1647 .type = AMDGPU_RING_TYPE_VCN_ENC, 1648 .align_mask = 0x3f, 1649 .nop = VCN_ENC_CMD_NO_OP, 1650 .vmhub = AMDGPU_MMHUB_1, 1651 .get_rptr = vcn_v2_5_enc_ring_get_rptr, 1652 .get_wptr = vcn_v2_5_enc_ring_get_wptr, 1653 .set_wptr = vcn_v2_5_enc_ring_set_wptr, 1654 .emit_frame_size = 1655 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 1656 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 1657 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 1658 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 1659 1, /* vcn_v2_0_enc_ring_insert_end */ 1660 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 1661 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 1662 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 1663 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush, 1664 .test_ring = amdgpu_vcn_enc_ring_test_ring, 1665 .test_ib = amdgpu_vcn_enc_ring_test_ib, 1666 .insert_nop = amdgpu_ring_insert_nop, 1667 .insert_end = vcn_v2_0_enc_ring_insert_end, 1668 .pad_ib = amdgpu_ring_generic_pad_ib, 1669 .begin_use = amdgpu_vcn_ring_begin_use, 1670 .end_use = amdgpu_vcn_ring_end_use, 1671 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg, 1672 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait, 1673 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1674 }; 1675 1676 static const struct amdgpu_ring_funcs vcn_v2_6_enc_ring_vm_funcs = { 1677 .type = AMDGPU_RING_TYPE_VCN_ENC, 1678 .align_mask = 0x3f, 1679 .nop = VCN_ENC_CMD_NO_OP, 1680 .vmhub = AMDGPU_MMHUB_0, 1681 .get_rptr = vcn_v2_5_enc_ring_get_rptr, 1682 .get_wptr = vcn_v2_5_enc_ring_get_wptr, 1683 .set_wptr = vcn_v2_5_enc_ring_set_wptr, 1684 .emit_frame_size = 1685 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 1686 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 1687 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 1688 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 1689 1, /* vcn_v2_0_enc_ring_insert_end */ 1690 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 1691 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 1692 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 1693 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush, 1694 .test_ring = amdgpu_vcn_enc_ring_test_ring, 1695 .test_ib = amdgpu_vcn_enc_ring_test_ib, 1696 .insert_nop = amdgpu_ring_insert_nop, 1697 .insert_end = vcn_v2_0_enc_ring_insert_end, 1698 .pad_ib = amdgpu_ring_generic_pad_ib, 1699 .begin_use = amdgpu_vcn_ring_begin_use, 1700 .end_use = amdgpu_vcn_ring_end_use, 1701 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg, 1702 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait, 1703 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1704 }; 1705 1706 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev) 1707 { 1708 int i; 1709 1710 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1711 if (adev->vcn.harvest_config & (1 << i)) 1712 continue; 1713 if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0)) 1714 adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs; 1715 else /* CHIP_ALDEBARAN */ 1716 adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_6_dec_ring_vm_funcs; 1717 adev->vcn.inst[i].ring_dec.me = i; 1718 DRM_INFO("VCN(%d) decode is enabled in VM mode\n", i); 1719 } 1720 } 1721 1722 static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev) 1723 { 1724 int i, j; 1725 1726 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) { 1727 if (adev->vcn.harvest_config & (1 << j)) 1728 continue; 1729 for (i = 0; i < adev->vcn.num_enc_rings; ++i) { 1730 if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0)) 1731 adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs; 1732 else /* CHIP_ALDEBARAN */ 1733 adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_6_enc_ring_vm_funcs; 1734 adev->vcn.inst[j].ring_enc[i].me = j; 1735 } 1736 DRM_INFO("VCN(%d) encode is enabled in VM mode\n", j); 1737 } 1738 } 1739 1740 static bool vcn_v2_5_is_idle(void *handle) 1741 { 1742 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1743 int i, ret = 1; 1744 1745 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1746 if (adev->vcn.harvest_config & (1 << i)) 1747 continue; 1748 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE); 1749 } 1750 1751 return ret; 1752 } 1753 1754 static int vcn_v2_5_wait_for_idle(void *handle) 1755 { 1756 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1757 int i, ret = 0; 1758 1759 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1760 if (adev->vcn.harvest_config & (1 << i)) 1761 continue; 1762 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 1763 UVD_STATUS__IDLE); 1764 if (ret) 1765 return ret; 1766 } 1767 1768 return ret; 1769 } 1770 1771 static int vcn_v2_5_set_clockgating_state(void *handle, 1772 enum amd_clockgating_state state) 1773 { 1774 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1775 bool enable = (state == AMD_CG_STATE_GATE); 1776 1777 if (amdgpu_sriov_vf(adev)) 1778 return 0; 1779 1780 if (enable) { 1781 if (!vcn_v2_5_is_idle(handle)) 1782 return -EBUSY; 1783 vcn_v2_5_enable_clock_gating(adev); 1784 } else { 1785 vcn_v2_5_disable_clock_gating(adev); 1786 } 1787 1788 return 0; 1789 } 1790 1791 static int vcn_v2_5_set_powergating_state(void *handle, 1792 enum amd_powergating_state state) 1793 { 1794 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1795 int ret; 1796 1797 if (amdgpu_sriov_vf(adev)) 1798 return 0; 1799 1800 if(state == adev->vcn.cur_state) 1801 return 0; 1802 1803 if (state == AMD_PG_STATE_GATE) 1804 ret = vcn_v2_5_stop(adev); 1805 else 1806 ret = vcn_v2_5_start(adev); 1807 1808 if(!ret) 1809 adev->vcn.cur_state = state; 1810 1811 return ret; 1812 } 1813 1814 static int vcn_v2_5_set_interrupt_state(struct amdgpu_device *adev, 1815 struct amdgpu_irq_src *source, 1816 unsigned type, 1817 enum amdgpu_interrupt_state state) 1818 { 1819 return 0; 1820 } 1821 1822 static int vcn_v2_5_process_interrupt(struct amdgpu_device *adev, 1823 struct amdgpu_irq_src *source, 1824 struct amdgpu_iv_entry *entry) 1825 { 1826 uint32_t ip_instance; 1827 1828 switch (entry->client_id) { 1829 case SOC15_IH_CLIENTID_VCN: 1830 ip_instance = 0; 1831 break; 1832 case SOC15_IH_CLIENTID_VCN1: 1833 ip_instance = 1; 1834 break; 1835 default: 1836 DRM_ERROR("Unhandled client id: %d\n", entry->client_id); 1837 return 0; 1838 } 1839 1840 DRM_DEBUG("IH: VCN TRAP\n"); 1841 1842 switch (entry->src_id) { 1843 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT: 1844 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec); 1845 break; 1846 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE: 1847 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]); 1848 break; 1849 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY: 1850 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]); 1851 break; 1852 default: 1853 DRM_ERROR("Unhandled interrupt: %d %d\n", 1854 entry->src_id, entry->src_data[0]); 1855 break; 1856 } 1857 1858 return 0; 1859 } 1860 1861 static const struct amdgpu_irq_src_funcs vcn_v2_5_irq_funcs = { 1862 .set = vcn_v2_5_set_interrupt_state, 1863 .process = vcn_v2_5_process_interrupt, 1864 }; 1865 1866 static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev) 1867 { 1868 int i; 1869 1870 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1871 if (adev->vcn.harvest_config & (1 << i)) 1872 continue; 1873 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1; 1874 adev->vcn.inst[i].irq.funcs = &vcn_v2_5_irq_funcs; 1875 } 1876 } 1877 1878 static const struct amd_ip_funcs vcn_v2_5_ip_funcs = { 1879 .name = "vcn_v2_5", 1880 .early_init = vcn_v2_5_early_init, 1881 .late_init = NULL, 1882 .sw_init = vcn_v2_5_sw_init, 1883 .sw_fini = vcn_v2_5_sw_fini, 1884 .hw_init = vcn_v2_5_hw_init, 1885 .hw_fini = vcn_v2_5_hw_fini, 1886 .suspend = vcn_v2_5_suspend, 1887 .resume = vcn_v2_5_resume, 1888 .is_idle = vcn_v2_5_is_idle, 1889 .wait_for_idle = vcn_v2_5_wait_for_idle, 1890 .check_soft_reset = NULL, 1891 .pre_soft_reset = NULL, 1892 .soft_reset = NULL, 1893 .post_soft_reset = NULL, 1894 .set_clockgating_state = vcn_v2_5_set_clockgating_state, 1895 .set_powergating_state = vcn_v2_5_set_powergating_state, 1896 }; 1897 1898 static const struct amd_ip_funcs vcn_v2_6_ip_funcs = { 1899 .name = "vcn_v2_6", 1900 .early_init = vcn_v2_5_early_init, 1901 .late_init = NULL, 1902 .sw_init = vcn_v2_5_sw_init, 1903 .sw_fini = vcn_v2_5_sw_fini, 1904 .hw_init = vcn_v2_5_hw_init, 1905 .hw_fini = vcn_v2_5_hw_fini, 1906 .suspend = vcn_v2_5_suspend, 1907 .resume = vcn_v2_5_resume, 1908 .is_idle = vcn_v2_5_is_idle, 1909 .wait_for_idle = vcn_v2_5_wait_for_idle, 1910 .check_soft_reset = NULL, 1911 .pre_soft_reset = NULL, 1912 .soft_reset = NULL, 1913 .post_soft_reset = NULL, 1914 .set_clockgating_state = vcn_v2_5_set_clockgating_state, 1915 .set_powergating_state = vcn_v2_5_set_powergating_state, 1916 }; 1917 1918 const struct amdgpu_ip_block_version vcn_v2_5_ip_block = 1919 { 1920 .type = AMD_IP_BLOCK_TYPE_VCN, 1921 .major = 2, 1922 .minor = 5, 1923 .rev = 0, 1924 .funcs = &vcn_v2_5_ip_funcs, 1925 }; 1926 1927 const struct amdgpu_ip_block_version vcn_v2_6_ip_block = 1928 { 1929 .type = AMD_IP_BLOCK_TYPE_VCN, 1930 .major = 2, 1931 .minor = 6, 1932 .rev = 0, 1933 .funcs = &vcn_v2_6_ip_funcs, 1934 }; 1935