13c556139SLikun Gao /* 23c556139SLikun Gao * Copyright 2020 Advanced Micro Devices, Inc. 33c556139SLikun Gao * 43c556139SLikun Gao * Permission is hereby granted, free of charge, to any person obtaining a 53c556139SLikun Gao * copy of this software and associated documentation files (the "Software"), 63c556139SLikun Gao * to deal in the Software without restriction, including without limitation 73c556139SLikun Gao * the rights to use, copy, modify, merge, publish, distribute, sublicense, 83c556139SLikun Gao * and/or sell copies of the Software, and to permit persons to whom the 93c556139SLikun Gao * Software is furnished to do so, subject to the following conditions: 103c556139SLikun Gao * 113c556139SLikun Gao * The above copyright notice and this permission notice shall be included in 123c556139SLikun Gao * all copies or substantial portions of the Software. 133c556139SLikun Gao * 143c556139SLikun Gao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 153c556139SLikun Gao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 163c556139SLikun Gao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 173c556139SLikun Gao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 183c556139SLikun Gao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 193c556139SLikun Gao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 203c556139SLikun Gao * OTHER DEALINGS IN THE SOFTWARE. 213c556139SLikun Gao * 223c556139SLikun Gao */ 233c556139SLikun Gao #include "amdgpu.h" 243c556139SLikun Gao #include "amdgpu_atombios.h" 253c556139SLikun Gao #include "hdp_v4_0.h" 263c556139SLikun Gao #include "amdgpu_ras.h" 273c556139SLikun Gao 283c556139SLikun Gao #include "hdp/hdp_4_0_offset.h" 293c556139SLikun Gao #include "hdp/hdp_4_0_sh_mask.h" 303c556139SLikun Gao #include <uapi/linux/kfd_ioctl.h> 313c556139SLikun Gao 323c556139SLikun Gao /* for Vega20 register name change */ 333c556139SLikun Gao #define mmHDP_MEM_POWER_CTRL 0x00d4 343c556139SLikun Gao #define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L 353c556139SLikun Gao #define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L 363c556139SLikun Gao #define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L 373c556139SLikun Gao #define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L 383c556139SLikun Gao #define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 393c556139SLikun Gao 403c556139SLikun Gao static void hdp_v4_0_flush_hdp(struct amdgpu_device *adev, 413c556139SLikun Gao struct amdgpu_ring *ring) 423c556139SLikun Gao { 433c556139SLikun Gao if (!ring || !ring->funcs->emit_wreg) 443c556139SLikun Gao WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); 453c556139SLikun Gao else 463c556139SLikun Gao amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); 473c556139SLikun Gao } 483c556139SLikun Gao 493c556139SLikun Gao static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev, 503c556139SLikun Gao struct amdgpu_ring *ring) 513c556139SLikun Gao { 52*2024ccc8SLe Ma if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 0) || 53*2024ccc8SLe Ma adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 2)) 54d02692aeSHawking Zhang return; 55d02692aeSHawking Zhang 563c556139SLikun Gao if (!ring || !ring->funcs->emit_wreg) 573c556139SLikun Gao WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); 583c556139SLikun Gao else 593c556139SLikun Gao amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( 603c556139SLikun Gao HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); 613c556139SLikun Gao } 623c556139SLikun Gao 63ca81b26dSHawking Zhang static void hdp_v4_0_query_ras_error_count(struct amdgpu_device *adev, 64ca81b26dSHawking Zhang void *ras_error_status) 65ca81b26dSHawking Zhang { 66ca81b26dSHawking Zhang struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 67ca81b26dSHawking Zhang 68ca81b26dSHawking Zhang err_data->ue_count = 0; 69ca81b26dSHawking Zhang err_data->ce_count = 0; 70ca81b26dSHawking Zhang 71ca81b26dSHawking Zhang if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) 72ca81b26dSHawking Zhang return; 73ca81b26dSHawking Zhang 74ca81b26dSHawking Zhang /* HDP SRAM errors are uncorrectable ones (i.e. fatal errors) */ 75ca81b26dSHawking Zhang err_data->ue_count += RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); 76ca81b26dSHawking Zhang }; 77ca81b26dSHawking Zhang 783c556139SLikun Gao static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev) 793c556139SLikun Gao { 803c556139SLikun Gao if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) 813c556139SLikun Gao return; 82ca81b26dSHawking Zhang 831d789535SAlex Deucher if (adev->ip_versions[HDP_HWIP][0] >= IP_VERSION(4, 4, 0)) 84ca81b26dSHawking Zhang WREG32_SOC15(HDP, 0, mmHDP_EDC_CNT, 0); 85ca81b26dSHawking Zhang else 863c556139SLikun Gao /*read back hdp ras counter to reset it to 0 */ 873c556139SLikun Gao RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); 883c556139SLikun Gao } 893c556139SLikun Gao 903c556139SLikun Gao static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev, 913c556139SLikun Gao bool enable) 923c556139SLikun Gao { 933c556139SLikun Gao uint32_t def, data; 943c556139SLikun Gao 951d789535SAlex Deucher if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 0, 0) || 961d789535SAlex Deucher adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 0, 1) || 971d789535SAlex Deucher adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 1, 1) || 981d789535SAlex Deucher adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 1, 0)) { 993c556139SLikun Gao def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); 1003c556139SLikun Gao 1013c556139SLikun Gao if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) 1023c556139SLikun Gao data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; 1033c556139SLikun Gao else 1043c556139SLikun Gao data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; 1053c556139SLikun Gao 1063c556139SLikun Gao if (def != data) 1073c556139SLikun Gao WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); 1083c556139SLikun Gao } else { 1093c556139SLikun Gao def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); 1103c556139SLikun Gao 1113c556139SLikun Gao if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) 1123c556139SLikun Gao data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | 1133c556139SLikun Gao HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | 1143c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | 1153c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; 1163c556139SLikun Gao else 1173c556139SLikun Gao data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | 1183c556139SLikun Gao HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | 1193c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | 1203c556139SLikun Gao HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); 1213c556139SLikun Gao 1223c556139SLikun Gao if (def != data) 1233c556139SLikun Gao WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); 1243c556139SLikun Gao } 1253c556139SLikun Gao } 1263c556139SLikun Gao 1273c556139SLikun Gao static void hdp_v4_0_get_clockgating_state(struct amdgpu_device *adev, 12825faeddcSEvan Quan u64 *flags) 1293c556139SLikun Gao { 1303c556139SLikun Gao int data; 1313c556139SLikun Gao 1323c556139SLikun Gao /* AMD_CG_SUPPORT_HDP_LS */ 1333c556139SLikun Gao data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); 1343c556139SLikun Gao if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) 1353c556139SLikun Gao *flags |= AMD_CG_SUPPORT_HDP_LS; 1363c556139SLikun Gao } 1373c556139SLikun Gao 1383c556139SLikun Gao static void hdp_v4_0_init_registers(struct amdgpu_device *adev) 1393c556139SLikun Gao { 1401d789535SAlex Deucher switch (adev->ip_versions[HDP_HWIP][0]) { 14124be2d70SAlex Deucher case IP_VERSION(4, 2, 1): 1423c556139SLikun Gao WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); 1433c556139SLikun Gao break; 1443c556139SLikun Gao default: 1453c556139SLikun Gao break; 1463c556139SLikun Gao } 1473c556139SLikun Gao 1483c556139SLikun Gao WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1493c556139SLikun Gao 15045f0ff40SXiaogang Chen if (adev->ip_versions[HDP_HWIP][0] == IP_VERSION(4, 4, 0)) 15145f0ff40SXiaogang Chen WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, READ_BUFFER_WATERMARK, 2); 15245f0ff40SXiaogang Chen 1533c556139SLikun Gao WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); 1543c556139SLikun Gao WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); 1553c556139SLikun Gao } 1563c556139SLikun Gao 1576d76e904Syipechai struct amdgpu_ras_block_hw_ops hdp_v4_0_ras_hw_ops = { 158ca81b26dSHawking Zhang .query_ras_error_count = hdp_v4_0_query_ras_error_count, 159ca81b26dSHawking Zhang .reset_ras_error_count = hdp_v4_0_reset_ras_error_count, 160ca81b26dSHawking Zhang }; 161ca81b26dSHawking Zhang 1626d76e904Syipechai struct amdgpu_hdp_ras hdp_v4_0_ras = { 1636d76e904Syipechai .ras_block = { 164bdb3489cSyipechai .ras_comm = { 1656d76e904Syipechai .name = "hdp", 1666d76e904Syipechai .block = AMDGPU_RAS_BLOCK__HDP, 167634b56b0Syipechai .type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 168bdb3489cSyipechai }, 1696d76e904Syipechai .hw_ops = &hdp_v4_0_ras_hw_ops, 1706d76e904Syipechai }, 1716d76e904Syipechai }; 1726d76e904Syipechai 1733c556139SLikun Gao const struct amdgpu_hdp_funcs hdp_v4_0_funcs = { 1743c556139SLikun Gao .flush_hdp = hdp_v4_0_flush_hdp, 1753c556139SLikun Gao .invalidate_hdp = hdp_v4_0_invalidate_hdp, 1763c556139SLikun Gao .update_clock_gating = hdp_v4_0_update_clock_gating, 1773c556139SLikun Gao .get_clock_gating_state = hdp_v4_0_get_clockgating_state, 1783c556139SLikun Gao .init_registers = hdp_v4_0_init_registers, 1793c556139SLikun Gao }; 180