13c556139SLikun Gao /* 23c556139SLikun Gao * Copyright 2020 Advanced Micro Devices, Inc. 33c556139SLikun Gao * 43c556139SLikun Gao * Permission is hereby granted, free of charge, to any person obtaining a 53c556139SLikun Gao * copy of this software and associated documentation files (the "Software"), 63c556139SLikun Gao * to deal in the Software without restriction, including without limitation 73c556139SLikun Gao * the rights to use, copy, modify, merge, publish, distribute, sublicense, 83c556139SLikun Gao * and/or sell copies of the Software, and to permit persons to whom the 93c556139SLikun Gao * Software is furnished to do so, subject to the following conditions: 103c556139SLikun Gao * 113c556139SLikun Gao * The above copyright notice and this permission notice shall be included in 123c556139SLikun Gao * all copies or substantial portions of the Software. 133c556139SLikun Gao * 143c556139SLikun Gao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 153c556139SLikun Gao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 163c556139SLikun Gao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 173c556139SLikun Gao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 183c556139SLikun Gao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 193c556139SLikun Gao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 203c556139SLikun Gao * OTHER DEALINGS IN THE SOFTWARE. 213c556139SLikun Gao * 223c556139SLikun Gao */ 233c556139SLikun Gao #include "amdgpu.h" 243c556139SLikun Gao #include "amdgpu_atombios.h" 253c556139SLikun Gao #include "hdp_v4_0.h" 263c556139SLikun Gao #include "amdgpu_ras.h" 273c556139SLikun Gao 283c556139SLikun Gao #include "hdp/hdp_4_0_offset.h" 293c556139SLikun Gao #include "hdp/hdp_4_0_sh_mask.h" 303c556139SLikun Gao #include <uapi/linux/kfd_ioctl.h> 313c556139SLikun Gao 323c556139SLikun Gao /* for Vega20 register name change */ 333c556139SLikun Gao #define mmHDP_MEM_POWER_CTRL 0x00d4 343c556139SLikun Gao #define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L 353c556139SLikun Gao #define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L 363c556139SLikun Gao #define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L 373c556139SLikun Gao #define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L 383c556139SLikun Gao #define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 393c556139SLikun Gao 403c556139SLikun Gao static void hdp_v4_0_flush_hdp(struct amdgpu_device *adev, 413c556139SLikun Gao struct amdgpu_ring *ring) 423c556139SLikun Gao { 433c556139SLikun Gao if (!ring || !ring->funcs->emit_wreg) 443c556139SLikun Gao WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); 453c556139SLikun Gao else 463c556139SLikun Gao amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); 473c556139SLikun Gao } 483c556139SLikun Gao 493c556139SLikun Gao static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev, 503c556139SLikun Gao struct amdgpu_ring *ring) 513c556139SLikun Gao { 521d789535SAlex Deucher if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 0)) 53d02692aeSHawking Zhang return; 54d02692aeSHawking Zhang 553c556139SLikun Gao if (!ring || !ring->funcs->emit_wreg) 563c556139SLikun Gao WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); 573c556139SLikun Gao else 583c556139SLikun Gao amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( 593c556139SLikun Gao HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); 603c556139SLikun Gao } 613c556139SLikun Gao 62ca81b26dSHawking Zhang static void hdp_v4_0_query_ras_error_count(struct amdgpu_device *adev, 63ca81b26dSHawking Zhang void *ras_error_status) 64ca81b26dSHawking Zhang { 65ca81b26dSHawking Zhang struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 66ca81b26dSHawking Zhang 67ca81b26dSHawking Zhang err_data->ue_count = 0; 68ca81b26dSHawking Zhang err_data->ce_count = 0; 69ca81b26dSHawking Zhang 70ca81b26dSHawking Zhang if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) 71ca81b26dSHawking Zhang return; 72ca81b26dSHawking Zhang 73ca81b26dSHawking Zhang /* HDP SRAM errors are uncorrectable ones (i.e. fatal errors) */ 74ca81b26dSHawking Zhang err_data->ue_count += RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); 75ca81b26dSHawking Zhang }; 76ca81b26dSHawking Zhang 773c556139SLikun Gao static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev) 783c556139SLikun Gao { 793c556139SLikun Gao if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) 803c556139SLikun Gao return; 81ca81b26dSHawking Zhang 821d789535SAlex Deucher if (adev->ip_versions[HDP_HWIP][0] >= IP_VERSION(4, 4, 0)) 83ca81b26dSHawking Zhang WREG32_SOC15(HDP, 0, mmHDP_EDC_CNT, 0); 84ca81b26dSHawking Zhang else 853c556139SLikun Gao /*read back hdp ras counter to reset it to 0 */ 863c556139SLikun Gao RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); 873c556139SLikun Gao } 883c556139SLikun Gao 893c556139SLikun Gao static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev, 903c556139SLikun Gao bool enable) 913c556139SLikun Gao { 923c556139SLikun Gao uint32_t def, data; 933c556139SLikun Gao 941d789535SAlex Deucher if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 0, 0) || 951d789535SAlex Deucher adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 0, 1) || 961d789535SAlex Deucher adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 1, 1) || 971d789535SAlex Deucher adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 1, 0)) { 983c556139SLikun Gao def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); 993c556139SLikun Gao 1003c556139SLikun Gao if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) 1013c556139SLikun Gao data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; 1023c556139SLikun Gao else 1033c556139SLikun Gao data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; 1043c556139SLikun Gao 1053c556139SLikun Gao if (def != data) 1063c556139SLikun Gao WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); 1073c556139SLikun Gao } else { 1083c556139SLikun Gao def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); 1093c556139SLikun Gao 1103c556139SLikun Gao if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) 1113c556139SLikun Gao data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | 1123c556139SLikun Gao HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | 1133c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | 1143c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; 1153c556139SLikun Gao else 1163c556139SLikun Gao data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | 1173c556139SLikun Gao HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | 1183c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | 1193c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); 1203c556139SLikun Gao 1213c556139SLikun Gao if (def != data) 1223c556139SLikun Gao WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); 1233c556139SLikun Gao } 1243c556139SLikun Gao } 1253c556139SLikun Gao 1263c556139SLikun Gao static void hdp_v4_0_get_clockgating_state(struct amdgpu_device *adev, 1273c556139SLikun Gao u32 *flags) 1283c556139SLikun Gao { 1293c556139SLikun Gao int data; 1303c556139SLikun Gao 1313c556139SLikun Gao /* AMD_CG_SUPPORT_HDP_LS */ 1323c556139SLikun Gao data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); 1333c556139SLikun Gao if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) 1343c556139SLikun Gao *flags |= AMD_CG_SUPPORT_HDP_LS; 1353c556139SLikun Gao } 1363c556139SLikun Gao 1373c556139SLikun Gao static void hdp_v4_0_init_registers(struct amdgpu_device *adev) 1383c556139SLikun Gao { 1391d789535SAlex Deucher switch (adev->ip_versions[HDP_HWIP][0]) { 14024be2d70SAlex Deucher case IP_VERSION(4, 2, 1): 1413c556139SLikun Gao WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); 1423c556139SLikun Gao break; 1433c556139SLikun Gao default: 1443c556139SLikun Gao break; 1453c556139SLikun Gao } 1463c556139SLikun Gao 1473c556139SLikun Gao WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1483c556139SLikun Gao 1493c556139SLikun Gao WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); 1503c556139SLikun Gao WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); 1513c556139SLikun Gao } 1523c556139SLikun Gao 1536d76e904Syipechai struct amdgpu_ras_block_hw_ops hdp_v4_0_ras_hw_ops = { 154ca81b26dSHawking Zhang .query_ras_error_count = hdp_v4_0_query_ras_error_count, 155ca81b26dSHawking Zhang .reset_ras_error_count = hdp_v4_0_reset_ras_error_count, 156ca81b26dSHawking Zhang }; 157ca81b26dSHawking Zhang 1586d76e904Syipechai struct amdgpu_hdp_ras hdp_v4_0_ras = { 1596d76e904Syipechai .ras_block = { 160*bdb3489cSyipechai .ras_comm = { 1616d76e904Syipechai .name = "hdp", 1626d76e904Syipechai .block = AMDGPU_RAS_BLOCK__HDP, 163*bdb3489cSyipechai }, 1646d76e904Syipechai .hw_ops = &hdp_v4_0_ras_hw_ops, 1656d76e904Syipechai .ras_late_init = amdgpu_hdp_ras_late_init, 1666d76e904Syipechai .ras_fini = amdgpu_hdp_ras_fini, 1676d76e904Syipechai }, 1686d76e904Syipechai }; 1696d76e904Syipechai 1703c556139SLikun Gao const struct amdgpu_hdp_funcs hdp_v4_0_funcs = { 1713c556139SLikun Gao .flush_hdp = hdp_v4_0_flush_hdp, 1723c556139SLikun Gao .invalidate_hdp = hdp_v4_0_invalidate_hdp, 1733c556139SLikun Gao .update_clock_gating = hdp_v4_0_update_clock_gating, 1743c556139SLikun Gao .get_clock_gating_state = hdp_v4_0_get_clockgating_state, 1753c556139SLikun Gao .init_registers = hdp_v4_0_init_registers, 1763c556139SLikun Gao }; 177