1fe3c9489SFeifei Xu /*
2fe3c9489SFeifei Xu * Copyright 2018 Advanced Micro Devices, Inc.
3fe3c9489SFeifei Xu *
4fe3c9489SFeifei Xu * Permission is hereby granted, free of charge, to any person obtaining a
5fe3c9489SFeifei Xu * copy of this software and associated documentation files (the "Software"),
6fe3c9489SFeifei Xu * to deal in the Software without restriction, including without limitation
7fe3c9489SFeifei Xu * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8fe3c9489SFeifei Xu * and/or sell copies of the Software, and to permit persons to whom the
9fe3c9489SFeifei Xu * Software is furnished to do so, subject to the following conditions:
10fe3c9489SFeifei Xu *
11fe3c9489SFeifei Xu * The above copyright notice and this permission notice shall be included in
12fe3c9489SFeifei Xu * all copies or substantial portions of the Software.
13fe3c9489SFeifei Xu *
14fe3c9489SFeifei Xu * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15fe3c9489SFeifei Xu * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16fe3c9489SFeifei Xu * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17fe3c9489SFeifei Xu * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18fe3c9489SFeifei Xu * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19fe3c9489SFeifei Xu * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20fe3c9489SFeifei Xu * OTHER DEALINGS IN THE SOFTWARE.
21fe3c9489SFeifei Xu *
22fe3c9489SFeifei Xu */
23fe3c9489SFeifei Xu #include "amdgpu.h"
24fe3c9489SFeifei Xu #include "amdgpu_atombios.h"
25fe3c9489SFeifei Xu #include "nbio_v7_4.h"
269ad1dc29SHawking Zhang #include "amdgpu_ras.h"
27fe3c9489SFeifei Xu
28fe3c9489SFeifei Xu #include "nbio/nbio_7_4_offset.h"
29fe3c9489SFeifei Xu #include "nbio/nbio_7_4_sh_mask.h"
30a0bb79e2SKent Russell #include "nbio/nbio_7_4_0_smn.h"
314e644fffSHawking Zhang #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
3288807dc8SOak Zeng #include <uapi/linux/kfd_ioctl.h>
33fe3c9489SFeifei Xu
349d015c0dSKenneth Feng #define smnPCIE_LC_CNTL 0x11140280
359d015c0dSKenneth Feng #define smnPCIE_LC_CNTL3 0x111402d4
369d015c0dSKenneth Feng #define smnPCIE_LC_CNTL6 0x111402ec
379d015c0dSKenneth Feng #define smnPCIE_LC_CNTL7 0x111402f0
38fe3c9489SFeifei Xu #define smnNBIF_MGCG_CTRL_LCLK 0x1013a21c
399d015c0dSKenneth Feng #define smnRCC_BIF_STRAP3 0x1012348c
409d015c0dSKenneth Feng #define RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK 0x0000FFFFL
419d015c0dSKenneth Feng #define RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK 0xFFFF0000L
429d015c0dSKenneth Feng #define smnRCC_BIF_STRAP5 0x10123494
439d015c0dSKenneth Feng #define RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK 0x0000FFFFL
449d015c0dSKenneth Feng #define smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2 0x1014008c
459d015c0dSKenneth Feng #define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK 0x0400L
469d015c0dSKenneth Feng #define smnBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP 0x10140324
479d015c0dSKenneth Feng #define smnPSWUSP0_PCIE_LC_CNTL2 0x111402c4
489d015c0dSKenneth Feng #define smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL 0x10123538
499d015c0dSKenneth Feng #define smnRCC_BIF_STRAP2 0x10123488
509d015c0dSKenneth Feng #define RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK 0x00004000L
519d015c0dSKenneth Feng #define RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT 0x0
529d015c0dSKenneth Feng #define RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT 0x10
539d015c0dSKenneth Feng #define RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT 0x0
54fe3c9489SFeifei Xu
550fe6a7b4SLe Ma /*
560fe6a7b4SLe Ma * These are nbio v7_4_1 registers mask. Temporarily define these here since
570fe6a7b4SLe Ma * nbio v7_4_1 header is incomplete.
580fe6a7b4SLe Ma */
59a0f9f854SAlex Deucher #define GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK 0x00001000L /* Don't use. Firmware uses this bit internally */
600fe6a7b4SLe Ma #define GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK 0x00002000L
610fe6a7b4SLe Ma #define GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK 0x00004000L
620fe6a7b4SLe Ma #define GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK 0x00008000L
630fe6a7b4SLe Ma #define GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK 0x00010000L
640fe6a7b4SLe Ma #define GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK 0x00020000L
65a0f9f854SAlex Deucher #define GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK 0x00040000L
66a0f9f854SAlex Deucher #define GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK 0x00080000L
67a0f9f854SAlex Deucher #define GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK 0x00100000L
680fe6a7b4SLe Ma
69989b6a05SJames Zhu #define mmBIF_MMSCH1_DOORBELL_RANGE 0x01dc
70989b6a05SJames Zhu #define mmBIF_MMSCH1_DOORBELL_RANGE_BASE_IDX 2
71989b6a05SJames Zhu //BIF_MMSCH1_DOORBELL_RANGE
72989b6a05SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE__OFFSET__SHIFT 0x2
73989b6a05SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE__SIZE__SHIFT 0x10
74989b6a05SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE__OFFSET_MASK 0x00000FFCL
75989b6a05SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE__SIZE_MASK 0x001F0000L
76989b6a05SJames Zhu
777ce29357SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE__OFFSET_MASK 0x00000FFCL
787ce29357SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE__SIZE_MASK 0x001F0000L
797ce29357SJames Zhu
807ce29357SJames Zhu #define mmBIF_MMSCH1_DOORBELL_RANGE_ALDE 0x01d8
817ce29357SJames Zhu #define mmBIF_MMSCH1_DOORBELL_RANGE_ALDE_BASE_IDX 2
827ce29357SJames Zhu //BIF_MMSCH1_DOORBELL_ALDE_RANGE
837ce29357SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE_ALDE__OFFSET__SHIFT 0x2
847ce29357SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE_ALDE__SIZE__SHIFT 0x10
857ce29357SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE_ALDE__OFFSET_MASK 0x00000FFCL
867ce29357SJames Zhu #define BIF_MMSCH1_DOORBELL_RANGE_ALDE__SIZE_MASK 0x001F0000L
877ce29357SJames Zhu
88f8a98f16SHawking Zhang #define mmRCC_DEV0_EPF0_STRAP0_ALDE 0x0015
89f8a98f16SHawking Zhang #define mmRCC_DEV0_EPF0_STRAP0_ALDE_BASE_IDX 2
90f8a98f16SHawking Zhang
91156872b0SJohn Clements #define mmBIF_DOORBELL_INT_CNTL_ALDE 0x00fe
9254e6badbSJohn Clements #define mmBIF_DOORBELL_INT_CNTL_ALDE_BASE_IDX 2
9354e6badbSJohn Clements #define BIF_DOORBELL_INT_CNTL_ALDE__DOORBELL_INTERRUPT_DISABLE__SHIFT 0x18
9454e6badbSJohn Clements #define BIF_DOORBELL_INT_CNTL_ALDE__DOORBELL_INTERRUPT_DISABLE_MASK 0x01000000L
9554e6badbSJohn Clements
96156872b0SJohn Clements #define mmBIF_INTR_CNTL_ALDE 0x0101
97156872b0SJohn Clements #define mmBIF_INTR_CNTL_ALDE_BASE_IDX 2
98156872b0SJohn Clements
9928f87950SLe Ma static void nbio_v7_4_query_ras_error_count(struct amdgpu_device *adev,
10028f87950SLe Ma void *ras_error_status);
10128f87950SLe Ma
nbio_v7_4_remap_hdp_registers(struct amdgpu_device * adev)10288807dc8SOak Zeng static void nbio_v7_4_remap_hdp_registers(struct amdgpu_device *adev)
10388807dc8SOak Zeng {
10488807dc8SOak Zeng WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL,
10588807dc8SOak Zeng adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
10688807dc8SOak Zeng WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL,
10788807dc8SOak Zeng adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
10888807dc8SOak Zeng }
10988807dc8SOak Zeng
nbio_v7_4_get_rev_id(struct amdgpu_device * adev)110fe3c9489SFeifei Xu static u32 nbio_v7_4_get_rev_id(struct amdgpu_device *adev)
111fe3c9489SFeifei Xu {
112f8a98f16SHawking Zhang u32 tmp;
113f8a98f16SHawking Zhang
114f8a98f16SHawking Zhang if (adev->asic_type == CHIP_ALDEBARAN)
115f8a98f16SHawking Zhang tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0_ALDE);
116f8a98f16SHawking Zhang else
117f8a98f16SHawking Zhang tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
118fe3c9489SFeifei Xu
119fe3c9489SFeifei Xu tmp &= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK;
120fe3c9489SFeifei Xu tmp >>= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT;
121fe3c9489SFeifei Xu
122fe3c9489SFeifei Xu return tmp;
123fe3c9489SFeifei Xu }
124fe3c9489SFeifei Xu
nbio_v7_4_mc_access_enable(struct amdgpu_device * adev,bool enable)125fe3c9489SFeifei Xu static void nbio_v7_4_mc_access_enable(struct amdgpu_device *adev, bool enable)
126fe3c9489SFeifei Xu {
127fe3c9489SFeifei Xu if (enable)
128fe3c9489SFeifei Xu WREG32_SOC15(NBIO, 0, mmBIF_FB_EN,
129fe3c9489SFeifei Xu BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
130fe3c9489SFeifei Xu else
131fe3c9489SFeifei Xu WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0);
132fe3c9489SFeifei Xu }
133fe3c9489SFeifei Xu
nbio_v7_4_get_memsize(struct amdgpu_device * adev)134fe3c9489SFeifei Xu static u32 nbio_v7_4_get_memsize(struct amdgpu_device *adev)
135fe3c9489SFeifei Xu {
136fe3c9489SFeifei Xu return RREG32_SOC15(NBIO, 0, mmRCC_CONFIG_MEMSIZE);
137fe3c9489SFeifei Xu }
138fe3c9489SFeifei Xu
nbio_v7_4_sdma_doorbell_range(struct amdgpu_device * adev,int instance,bool use_doorbell,int doorbell_index,int doorbell_size)139fe3c9489SFeifei Xu static void nbio_v7_4_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
1408987e2e2SOak Zeng bool use_doorbell, int doorbell_index, int doorbell_size)
141fe3c9489SFeifei Xu {
1423d81f67aSLe Ma u32 reg, doorbell_range;
143fe3c9489SFeifei Xu
144759eb38eSLe Ma if (instance < 2) {
1453d81f67aSLe Ma reg = instance +
1463d81f67aSLe Ma SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE);
147759eb38eSLe Ma } else {
1483d81f67aSLe Ma /*
1493d81f67aSLe Ma * These registers address of SDMA2~7 is not consecutive
1503d81f67aSLe Ma * from SDMA0~1. Need plus 4 dwords offset.
1513d81f67aSLe Ma *
1523d81f67aSLe Ma * BIF_SDMA0_DOORBELL_RANGE: 0x3bc0
1533d81f67aSLe Ma * BIF_SDMA1_DOORBELL_RANGE: 0x3bc4
1543d81f67aSLe Ma * BIF_SDMA2_DOORBELL_RANGE: 0x3bd8
155759eb38eSLe Ma + * BIF_SDMA4_DOORBELL_RANGE:
156759eb38eSLe Ma + * ARCTURUS: 0x3be0
157759eb38eSLe Ma + * ALDEBARAN: 0x3be4
1583d81f67aSLe Ma */
159759eb38eSLe Ma if (adev->asic_type == CHIP_ALDEBARAN && instance == 4)
160759eb38eSLe Ma reg = instance + 0x4 + 0x1 +
161759eb38eSLe Ma SOC15_REG_OFFSET(NBIO, 0,
162759eb38eSLe Ma mmBIF_SDMA0_DOORBELL_RANGE);
163759eb38eSLe Ma else
1643d81f67aSLe Ma reg = instance + 0x4 +
165759eb38eSLe Ma SOC15_REG_OFFSET(NBIO, 0,
166759eb38eSLe Ma mmBIF_SDMA0_DOORBELL_RANGE);
167759eb38eSLe Ma }
1683d81f67aSLe Ma
1693d81f67aSLe Ma doorbell_range = RREG32(reg);
170fe3c9489SFeifei Xu
171fe3c9489SFeifei Xu if (use_doorbell) {
172fe3c9489SFeifei Xu doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index);
1738987e2e2SOak Zeng doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, doorbell_size);
174fe3c9489SFeifei Xu } else
175fe3c9489SFeifei Xu doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0);
176fe3c9489SFeifei Xu
177fe3c9489SFeifei Xu WREG32(reg, doorbell_range);
178fe3c9489SFeifei Xu }
179fe3c9489SFeifei Xu
nbio_v7_4_vcn_doorbell_range(struct amdgpu_device * adev,bool use_doorbell,int doorbell_index,int instance)18039a5053fSLeo Liu static void nbio_v7_4_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell,
181989b6a05SJames Zhu int doorbell_index, int instance)
18239a5053fSLeo Liu {
183989b6a05SJames Zhu u32 reg;
184989b6a05SJames Zhu u32 doorbell_range;
18539a5053fSLeo Liu
1867ce29357SJames Zhu if (instance) {
1877ce29357SJames Zhu if (adev->asic_type == CHIP_ALDEBARAN)
1887ce29357SJames Zhu reg = SOC15_REG_OFFSET(NBIO, 0, mmBIF_MMSCH1_DOORBELL_RANGE_ALDE);
189989b6a05SJames Zhu else
1907ce29357SJames Zhu reg = SOC15_REG_OFFSET(NBIO, 0, mmBIF_MMSCH1_DOORBELL_RANGE);
1917ce29357SJames Zhu } else
192989b6a05SJames Zhu reg = SOC15_REG_OFFSET(NBIO, 0, mmBIF_MMSCH0_DOORBELL_RANGE);
193989b6a05SJames Zhu
194989b6a05SJames Zhu doorbell_range = RREG32(reg);
19539a5053fSLeo Liu
19639a5053fSLeo Liu if (use_doorbell) {
19739a5053fSLeo Liu doorbell_range = REG_SET_FIELD(doorbell_range,
19839a5053fSLeo Liu BIF_MMSCH0_DOORBELL_RANGE, OFFSET,
19939a5053fSLeo Liu doorbell_index);
20039a5053fSLeo Liu doorbell_range = REG_SET_FIELD(doorbell_range,
20139a5053fSLeo Liu BIF_MMSCH0_DOORBELL_RANGE, SIZE, 8);
20239a5053fSLeo Liu } else
20339a5053fSLeo Liu doorbell_range = REG_SET_FIELD(doorbell_range,
20439a5053fSLeo Liu BIF_MMSCH0_DOORBELL_RANGE, SIZE, 0);
20539a5053fSLeo Liu
20639a5053fSLeo Liu WREG32(reg, doorbell_range);
20739a5053fSLeo Liu }
20839a5053fSLeo Liu
nbio_v7_4_enable_doorbell_aperture(struct amdgpu_device * adev,bool enable)209fe3c9489SFeifei Xu static void nbio_v7_4_enable_doorbell_aperture(struct amdgpu_device *adev,
210fe3c9489SFeifei Xu bool enable)
211fe3c9489SFeifei Xu {
212fe3c9489SFeifei Xu WREG32_FIELD15(NBIO, 0, RCC_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, enable ? 1 : 0);
213fe3c9489SFeifei Xu }
214fe3c9489SFeifei Xu
nbio_v7_4_enable_doorbell_selfring_aperture(struct amdgpu_device * adev,bool enable)215fe3c9489SFeifei Xu static void nbio_v7_4_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
216fe3c9489SFeifei Xu bool enable)
217fe3c9489SFeifei Xu {
21812292519SJay Cornwall u32 tmp = 0;
219fe3c9489SFeifei Xu
22012292519SJay Cornwall if (enable) {
22112292519SJay Cornwall tmp = REG_SET_FIELD(tmp, DOORBELL_SELFRING_GPA_APER_CNTL, DOORBELL_SELFRING_GPA_APER_EN, 1) |
22212292519SJay Cornwall REG_SET_FIELD(tmp, DOORBELL_SELFRING_GPA_APER_CNTL, DOORBELL_SELFRING_GPA_APER_MODE, 1) |
22312292519SJay Cornwall REG_SET_FIELD(tmp, DOORBELL_SELFRING_GPA_APER_CNTL, DOORBELL_SELFRING_GPA_APER_SIZE, 0);
22412292519SJay Cornwall
22512292519SJay Cornwall WREG32_SOC15(NBIO, 0, mmDOORBELL_SELFRING_GPA_APER_BASE_LOW,
22612292519SJay Cornwall lower_32_bits(adev->doorbell.base));
22712292519SJay Cornwall WREG32_SOC15(NBIO, 0, mmDOORBELL_SELFRING_GPA_APER_BASE_HIGH,
22812292519SJay Cornwall upper_32_bits(adev->doorbell.base));
22912292519SJay Cornwall }
23012292519SJay Cornwall
23112292519SJay Cornwall WREG32_SOC15(NBIO, 0, mmDOORBELL_SELFRING_GPA_APER_CNTL, tmp);
232fe3c9489SFeifei Xu }
233fe3c9489SFeifei Xu
nbio_v7_4_ih_doorbell_range(struct amdgpu_device * adev,bool use_doorbell,int doorbell_index)234fe3c9489SFeifei Xu static void nbio_v7_4_ih_doorbell_range(struct amdgpu_device *adev,
235fe3c9489SFeifei Xu bool use_doorbell, int doorbell_index)
236fe3c9489SFeifei Xu {
237fe3c9489SFeifei Xu u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0 , mmBIF_IH_DOORBELL_RANGE);
238fe3c9489SFeifei Xu
239fe3c9489SFeifei Xu if (use_doorbell) {
240fe3c9489SFeifei Xu ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, OFFSET, doorbell_index);
241*318e431bSMukul Joshi ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 8);
242fe3c9489SFeifei Xu } else
243fe3c9489SFeifei Xu ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 0);
244fe3c9489SFeifei Xu
245fe3c9489SFeifei Xu WREG32_SOC15(NBIO, 0, mmBIF_IH_DOORBELL_RANGE, ih_doorbell_range);
246fe3c9489SFeifei Xu }
247fe3c9489SFeifei Xu
248fe3c9489SFeifei Xu
nbio_v7_4_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)249fe3c9489SFeifei Xu static void nbio_v7_4_update_medium_grain_clock_gating(struct amdgpu_device *adev,
250fe3c9489SFeifei Xu bool enable)
251fe3c9489SFeifei Xu {
252fe3c9489SFeifei Xu //TODO: Add support for v7.4
253fe3c9489SFeifei Xu }
254fe3c9489SFeifei Xu
nbio_v7_4_update_medium_grain_light_sleep(struct amdgpu_device * adev,bool enable)255fe3c9489SFeifei Xu static void nbio_v7_4_update_medium_grain_light_sleep(struct amdgpu_device *adev,
256fe3c9489SFeifei Xu bool enable)
257fe3c9489SFeifei Xu {
258fe3c9489SFeifei Xu uint32_t def, data;
259fe3c9489SFeifei Xu
260fe3c9489SFeifei Xu def = data = RREG32_PCIE(smnPCIE_CNTL2);
261fe3c9489SFeifei Xu if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
262fe3c9489SFeifei Xu data |= (PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
263fe3c9489SFeifei Xu PCIE_CNTL2__MST_MEM_LS_EN_MASK |
264fe3c9489SFeifei Xu PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
265fe3c9489SFeifei Xu } else {
266fe3c9489SFeifei Xu data &= ~(PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
267fe3c9489SFeifei Xu PCIE_CNTL2__MST_MEM_LS_EN_MASK |
268fe3c9489SFeifei Xu PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
269fe3c9489SFeifei Xu }
270fe3c9489SFeifei Xu
271fe3c9489SFeifei Xu if (def != data)
272fe3c9489SFeifei Xu WREG32_PCIE(smnPCIE_CNTL2, data);
273fe3c9489SFeifei Xu }
274fe3c9489SFeifei Xu
nbio_v7_4_get_clockgating_state(struct amdgpu_device * adev,u64 * flags)275fe3c9489SFeifei Xu static void nbio_v7_4_get_clockgating_state(struct amdgpu_device *adev,
27625faeddcSEvan Quan u64 *flags)
277fe3c9489SFeifei Xu {
278fe3c9489SFeifei Xu int data;
279fe3c9489SFeifei Xu
280fe3c9489SFeifei Xu /* AMD_CG_SUPPORT_BIF_MGCG */
281fe3c9489SFeifei Xu data = RREG32_PCIE(smnCPM_CONTROL);
282fe3c9489SFeifei Xu if (data & CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK)
283fe3c9489SFeifei Xu *flags |= AMD_CG_SUPPORT_BIF_MGCG;
284fe3c9489SFeifei Xu
285fe3c9489SFeifei Xu /* AMD_CG_SUPPORT_BIF_LS */
286fe3c9489SFeifei Xu data = RREG32_PCIE(smnPCIE_CNTL2);
287fe3c9489SFeifei Xu if (data & PCIE_CNTL2__SLV_MEM_LS_EN_MASK)
288fe3c9489SFeifei Xu *flags |= AMD_CG_SUPPORT_BIF_LS;
289fe3c9489SFeifei Xu }
290fe3c9489SFeifei Xu
nbio_v7_4_ih_control(struct amdgpu_device * adev)291fe3c9489SFeifei Xu static void nbio_v7_4_ih_control(struct amdgpu_device *adev)
292fe3c9489SFeifei Xu {
293fe3c9489SFeifei Xu u32 interrupt_cntl;
294fe3c9489SFeifei Xu
295fe3c9489SFeifei Xu /* setup interrupt control */
296fe3c9489SFeifei Xu WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
297fe3c9489SFeifei Xu interrupt_cntl = RREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL);
298fe3c9489SFeifei Xu /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
299fe3c9489SFeifei Xu * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
300fe3c9489SFeifei Xu */
301fe3c9489SFeifei Xu interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);
302fe3c9489SFeifei Xu /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
303fe3c9489SFeifei Xu interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);
304fe3c9489SFeifei Xu WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
305fe3c9489SFeifei Xu }
306fe3c9489SFeifei Xu
nbio_v7_4_get_hdp_flush_req_offset(struct amdgpu_device * adev)307fe3c9489SFeifei Xu static u32 nbio_v7_4_get_hdp_flush_req_offset(struct amdgpu_device *adev)
308fe3c9489SFeifei Xu {
309fe3c9489SFeifei Xu return SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ);
310fe3c9489SFeifei Xu }
311fe3c9489SFeifei Xu
nbio_v7_4_get_hdp_flush_done_offset(struct amdgpu_device * adev)312fe3c9489SFeifei Xu static u32 nbio_v7_4_get_hdp_flush_done_offset(struct amdgpu_device *adev)
313fe3c9489SFeifei Xu {
314fe3c9489SFeifei Xu return SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE);
315fe3c9489SFeifei Xu }
316fe3c9489SFeifei Xu
nbio_v7_4_get_pcie_index_offset(struct amdgpu_device * adev)317fe3c9489SFeifei Xu static u32 nbio_v7_4_get_pcie_index_offset(struct amdgpu_device *adev)
318fe3c9489SFeifei Xu {
319fe3c9489SFeifei Xu return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2);
320fe3c9489SFeifei Xu }
321fe3c9489SFeifei Xu
nbio_v7_4_get_pcie_data_offset(struct amdgpu_device * adev)322fe3c9489SFeifei Xu static u32 nbio_v7_4_get_pcie_data_offset(struct amdgpu_device *adev)
323fe3c9489SFeifei Xu {
324fe3c9489SFeifei Xu return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2);
325fe3c9489SFeifei Xu }
326fe3c9489SFeifei Xu
327bebc0762SHawking Zhang const struct nbio_hdp_flush_reg nbio_v7_4_hdp_flush_reg = {
328fe3c9489SFeifei Xu .ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK,
329fe3c9489SFeifei Xu .ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK,
330fe3c9489SFeifei Xu .ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK,
331fe3c9489SFeifei Xu .ref_and_mask_cp3 = GPU_HDP_FLUSH_DONE__CP3_MASK,
332fe3c9489SFeifei Xu .ref_and_mask_cp4 = GPU_HDP_FLUSH_DONE__CP4_MASK,
333fe3c9489SFeifei Xu .ref_and_mask_cp5 = GPU_HDP_FLUSH_DONE__CP5_MASK,
334fe3c9489SFeifei Xu .ref_and_mask_cp6 = GPU_HDP_FLUSH_DONE__CP6_MASK,
335fe3c9489SFeifei Xu .ref_and_mask_cp7 = GPU_HDP_FLUSH_DONE__CP7_MASK,
336fe3c9489SFeifei Xu .ref_and_mask_cp8 = GPU_HDP_FLUSH_DONE__CP8_MASK,
337fe3c9489SFeifei Xu .ref_and_mask_cp9 = GPU_HDP_FLUSH_DONE__CP9_MASK,
338df9feb1aSAlex Deucher .ref_and_mask_sdma0 = GPU_HDP_FLUSH_DONE__SDMA0_MASK,
339df9feb1aSAlex Deucher .ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK,
340df9feb1aSAlex Deucher };
341df9feb1aSAlex Deucher
nbio_v7_4_init_registers(struct amdgpu_device * adev)342fe3c9489SFeifei Xu static void nbio_v7_4_init_registers(struct amdgpu_device *adev)
343fe3c9489SFeifei Xu {
344bb7c3e9cSLijo Lazar uint32_t baco_cntl;
345bb7c3e9cSLijo Lazar
346e3993811SFelix Kuehling if (amdgpu_sriov_vf(adev))
347e3993811SFelix Kuehling adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
348e3993811SFelix Kuehling mmBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
349bb7c3e9cSLijo Lazar
350bb7c3e9cSLijo Lazar if (adev->ip_versions[NBIO_HWIP][0] == IP_VERSION(7, 4, 4) &&
351bb7c3e9cSLijo Lazar !amdgpu_sriov_vf(adev)) {
352bb7c3e9cSLijo Lazar baco_cntl = RREG32_SOC15(NBIO, 0, mmBACO_CNTL);
353bb7c3e9cSLijo Lazar if (baco_cntl &
354bb7c3e9cSLijo Lazar (BACO_CNTL__BACO_DUMMY_EN_MASK | BACO_CNTL__BACO_EN_MASK)) {
355bb7c3e9cSLijo Lazar baco_cntl &= ~(BACO_CNTL__BACO_DUMMY_EN_MASK |
356bb7c3e9cSLijo Lazar BACO_CNTL__BACO_EN_MASK);
357bb7c3e9cSLijo Lazar dev_dbg(adev->dev, "Unsetting baco dummy mode %x",
358bb7c3e9cSLijo Lazar baco_cntl);
359bb7c3e9cSLijo Lazar WREG32_SOC15(NBIO, 0, mmBACO_CNTL, baco_cntl);
360bb7c3e9cSLijo Lazar }
361bb7c3e9cSLijo Lazar }
362fe3c9489SFeifei Xu }
363fe3c9489SFeifei Xu
nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device * adev)3644241863aSHawking Zhang static void nbio_v7_4_handle_ras_controller_intr_no_bifring(struct amdgpu_device *adev)
3654241863aSHawking Zhang {
3664241863aSHawking Zhang uint32_t bif_doorbell_intr_cntl;
36728f87950SLe Ma struct ras_manager *obj = amdgpu_ras_find_obj(adev, adev->nbio.ras_if);
3683cd4f618SGuchun Chen struct ras_err_data err_data = {0, 0, 0, NULL};
369f75e94d8SGuchun Chen struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3704241863aSHawking Zhang
37154e6badbSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
37254e6badbSJohn Clements bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE);
37354e6badbSJohn Clements else
3744241863aSHawking Zhang bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL);
37554e6badbSJohn Clements
3764241863aSHawking Zhang if (REG_GET_FIELD(bif_doorbell_intr_cntl,
3774241863aSHawking Zhang BIF_DOORBELL_INT_CNTL, RAS_CNTLR_INTERRUPT_STATUS)) {
3784241863aSHawking Zhang /* driver has to clear the interrupt status when bif ring is disabled */
3794241863aSHawking Zhang bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl,
3804241863aSHawking Zhang BIF_DOORBELL_INT_CNTL,
3814241863aSHawking Zhang RAS_CNTLR_INTERRUPT_CLEAR, 1);
38254e6badbSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
38354e6badbSJohn Clements WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE, bif_doorbell_intr_cntl);
38454e6badbSJohn Clements else
3854241863aSHawking Zhang WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl);
3867c6e68c7SAndrey Grodzovsky
387f75e94d8SGuchun Chen if (!ras->disable_ras_err_cnt_harvest) {
38828f87950SLe Ma /*
389f75e94d8SGuchun Chen * clear error status after ras_controller_intr
390f75e94d8SGuchun Chen * according to hw team and count ue number
391f75e94d8SGuchun Chen * for query
39228f87950SLe Ma */
3933cd4f618SGuchun Chen nbio_v7_4_query_ras_error_count(adev, &err_data);
3943cd4f618SGuchun Chen
395f75e94d8SGuchun Chen /* logging on error cnt and printing for awareness */
3963cd4f618SGuchun Chen obj->err_data.ue_count += err_data.ue_count;
3973cd4f618SGuchun Chen obj->err_data.ce_count += err_data.ce_count;
3983cd4f618SGuchun Chen
3993cd4f618SGuchun Chen if (err_data.ce_count)
4006952e99cSGuchun Chen dev_info(adev->dev, "%ld correctable hardware "
4016952e99cSGuchun Chen "errors detected in %s block, "
4026952e99cSGuchun Chen "no user action is needed.\n",
4036952e99cSGuchun Chen obj->err_data.ce_count,
404640ae42eSJohn Clements get_ras_block_str(adev->nbio.ras_if));
4053cd4f618SGuchun Chen
4063cd4f618SGuchun Chen if (err_data.ue_count)
4076952e99cSGuchun Chen dev_info(adev->dev, "%ld uncorrectable hardware "
4086952e99cSGuchun Chen "errors detected in %s block\n",
4096952e99cSGuchun Chen obj->err_data.ue_count,
410640ae42eSJohn Clements get_ras_block_str(adev->nbio.ras_if));
411f75e94d8SGuchun Chen }
41228f87950SLe Ma
4136952e99cSGuchun Chen dev_info(adev->dev, "RAS controller interrupt triggered "
4146952e99cSGuchun Chen "by NBIF error\n");
4154a2d9356SLe Ma
4164a2d9356SLe Ma /* ras_controller_int is dedicated for nbif ras error,
4174a2d9356SLe Ma * not the global interrupt for sync flood
4184a2d9356SLe Ma */
41961934624SGuchun Chen amdgpu_ras_reset_gpu(adev);
4204241863aSHawking Zhang }
4214241863aSHawking Zhang }
4224241863aSHawking Zhang
nbio_v7_4_handle_ras_err_event_athub_intr_no_bifring(struct amdgpu_device * adev)4234241863aSHawking Zhang static void nbio_v7_4_handle_ras_err_event_athub_intr_no_bifring(struct amdgpu_device *adev)
4244241863aSHawking Zhang {
4254241863aSHawking Zhang uint32_t bif_doorbell_intr_cntl;
4264241863aSHawking Zhang
42754e6badbSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
42854e6badbSJohn Clements bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE);
42954e6badbSJohn Clements else
4304241863aSHawking Zhang bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL);
43154e6badbSJohn Clements
4324241863aSHawking Zhang if (REG_GET_FIELD(bif_doorbell_intr_cntl,
4334241863aSHawking Zhang BIF_DOORBELL_INT_CNTL, RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS)) {
4344241863aSHawking Zhang /* driver has to clear the interrupt status when bif ring is disabled */
4354241863aSHawking Zhang bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl,
4364241863aSHawking Zhang BIF_DOORBELL_INT_CNTL,
4374241863aSHawking Zhang RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR, 1);
43854e6badbSJohn Clements
43954e6badbSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
44054e6badbSJohn Clements WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL_ALDE, bif_doorbell_intr_cntl);
44154e6badbSJohn Clements else
4424241863aSHawking Zhang WREG32_SOC15(NBIO, 0, mmBIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl);
4437c6e68c7SAndrey Grodzovsky
4447c6e68c7SAndrey Grodzovsky amdgpu_ras_global_ras_isr(adev);
4454241863aSHawking Zhang }
4464241863aSHawking Zhang }
4474241863aSHawking Zhang
4484e644fffSHawking Zhang
nbio_v7_4_set_ras_controller_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)4494e644fffSHawking Zhang static int nbio_v7_4_set_ras_controller_irq_state(struct amdgpu_device *adev,
4504e644fffSHawking Zhang struct amdgpu_irq_src *src,
4514e644fffSHawking Zhang unsigned type,
4524e644fffSHawking Zhang enum amdgpu_interrupt_state state)
4534e644fffSHawking Zhang {
4544e644fffSHawking Zhang /* The ras_controller_irq enablement should be done in psp bl when it
4554e644fffSHawking Zhang * tries to enable ras feature. Driver only need to set the correct interrupt
4564e644fffSHawking Zhang * vector for bare-metal and sriov use case respectively
4574e644fffSHawking Zhang */
4584e644fffSHawking Zhang uint32_t bif_intr_cntl;
4594e644fffSHawking Zhang
460156872b0SJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
461156872b0SJohn Clements bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE);
462156872b0SJohn Clements else
4634e644fffSHawking Zhang bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL);
464156872b0SJohn Clements
4654e644fffSHawking Zhang if (state == AMDGPU_IRQ_STATE_ENABLE) {
4664e644fffSHawking Zhang /* set interrupt vector select bit to 0 to select
4674e644fffSHawking Zhang * vetcor 1 for bare metal case */
4684e644fffSHawking Zhang bif_intr_cntl = REG_SET_FIELD(bif_intr_cntl,
4694e644fffSHawking Zhang BIF_INTR_CNTL,
4704e644fffSHawking Zhang RAS_INTR_VEC_SEL, 0);
471156872b0SJohn Clements
472156872b0SJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
473156872b0SJohn Clements WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE, bif_intr_cntl);
474156872b0SJohn Clements else
4754e644fffSHawking Zhang WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL, bif_intr_cntl);
476156872b0SJohn Clements
4774e644fffSHawking Zhang }
4784e644fffSHawking Zhang
4794e644fffSHawking Zhang return 0;
4804e644fffSHawking Zhang }
4814e644fffSHawking Zhang
nbio_v7_4_process_ras_controller_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)4824e644fffSHawking Zhang static int nbio_v7_4_process_ras_controller_irq(struct amdgpu_device *adev,
4834e644fffSHawking Zhang struct amdgpu_irq_src *source,
4844e644fffSHawking Zhang struct amdgpu_iv_entry *entry)
4854e644fffSHawking Zhang {
4864e644fffSHawking Zhang /* By design, the ih cookie for ras_controller_irq should be written
4874e644fffSHawking Zhang * to BIFring instead of general iv ring. However, due to known bif ring
4884e644fffSHawking Zhang * hw bug, it has to be disabled. There is no chance the process function
4894e644fffSHawking Zhang * will be involked. Just left it as a dummy one.
4904e644fffSHawking Zhang */
4914e644fffSHawking Zhang return 0;
4924e644fffSHawking Zhang }
4934e644fffSHawking Zhang
nbio_v7_4_set_ras_err_event_athub_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)4944e644fffSHawking Zhang static int nbio_v7_4_set_ras_err_event_athub_irq_state(struct amdgpu_device *adev,
4954e644fffSHawking Zhang struct amdgpu_irq_src *src,
4964e644fffSHawking Zhang unsigned type,
4974e644fffSHawking Zhang enum amdgpu_interrupt_state state)
4984e644fffSHawking Zhang {
4994e644fffSHawking Zhang /* The ras_controller_irq enablement should be done in psp bl when it
5004e644fffSHawking Zhang * tries to enable ras feature. Driver only need to set the correct interrupt
5014e644fffSHawking Zhang * vector for bare-metal and sriov use case respectively
5024e644fffSHawking Zhang */
5034e644fffSHawking Zhang uint32_t bif_intr_cntl;
5044e644fffSHawking Zhang
505156872b0SJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
506156872b0SJohn Clements bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE);
507156872b0SJohn Clements else
5084e644fffSHawking Zhang bif_intr_cntl = RREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL);
509156872b0SJohn Clements
5104e644fffSHawking Zhang if (state == AMDGPU_IRQ_STATE_ENABLE) {
5114e644fffSHawking Zhang /* set interrupt vector select bit to 0 to select
5124e644fffSHawking Zhang * vetcor 1 for bare metal case */
5134e644fffSHawking Zhang bif_intr_cntl = REG_SET_FIELD(bif_intr_cntl,
5144e644fffSHawking Zhang BIF_INTR_CNTL,
5154e644fffSHawking Zhang RAS_INTR_VEC_SEL, 0);
516156872b0SJohn Clements
517156872b0SJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
518156872b0SJohn Clements WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL_ALDE, bif_intr_cntl);
519156872b0SJohn Clements else
5204e644fffSHawking Zhang WREG32_SOC15(NBIO, 0, mmBIF_INTR_CNTL, bif_intr_cntl);
5214e644fffSHawking Zhang }
5224e644fffSHawking Zhang
5234e644fffSHawking Zhang return 0;
5244e644fffSHawking Zhang }
5254e644fffSHawking Zhang
nbio_v7_4_process_err_event_athub_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)5264e644fffSHawking Zhang static int nbio_v7_4_process_err_event_athub_irq(struct amdgpu_device *adev,
5274e644fffSHawking Zhang struct amdgpu_irq_src *source,
5284e644fffSHawking Zhang struct amdgpu_iv_entry *entry)
5294e644fffSHawking Zhang {
5304e644fffSHawking Zhang /* By design, the ih cookie for err_event_athub_irq should be written
5314e644fffSHawking Zhang * to BIFring instead of general iv ring. However, due to known bif ring
5324e644fffSHawking Zhang * hw bug, it has to be disabled. There is no chance the process function
5334e644fffSHawking Zhang * will be involked. Just left it as a dummy one.
5344e644fffSHawking Zhang */
5354e644fffSHawking Zhang return 0;
5364e644fffSHawking Zhang }
5374e644fffSHawking Zhang
5384e644fffSHawking Zhang static const struct amdgpu_irq_src_funcs nbio_v7_4_ras_controller_irq_funcs = {
5394e644fffSHawking Zhang .set = nbio_v7_4_set_ras_controller_irq_state,
5404e644fffSHawking Zhang .process = nbio_v7_4_process_ras_controller_irq,
5414e644fffSHawking Zhang };
5424e644fffSHawking Zhang
5434e644fffSHawking Zhang static const struct amdgpu_irq_src_funcs nbio_v7_4_ras_err_event_athub_irq_funcs = {
5444e644fffSHawking Zhang .set = nbio_v7_4_set_ras_err_event_athub_irq_state,
5454e644fffSHawking Zhang .process = nbio_v7_4_process_err_event_athub_irq,
5464e644fffSHawking Zhang };
5474e644fffSHawking Zhang
nbio_v7_4_init_ras_controller_interrupt(struct amdgpu_device * adev)5484e644fffSHawking Zhang static int nbio_v7_4_init_ras_controller_interrupt (struct amdgpu_device *adev)
5494e644fffSHawking Zhang {
5504e644fffSHawking Zhang int r;
5514e644fffSHawking Zhang
5524e644fffSHawking Zhang /* init the irq funcs */
5534e644fffSHawking Zhang adev->nbio.ras_controller_irq.funcs =
5544e644fffSHawking Zhang &nbio_v7_4_ras_controller_irq_funcs;
5554e644fffSHawking Zhang adev->nbio.ras_controller_irq.num_types = 1;
5564e644fffSHawking Zhang
5574e644fffSHawking Zhang /* register ras controller interrupt */
5584e644fffSHawking Zhang r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_BIF,
5594e644fffSHawking Zhang NBIF_7_4__SRCID__RAS_CONTROLLER_INTERRUPT,
5604e644fffSHawking Zhang &adev->nbio.ras_controller_irq);
5614e644fffSHawking Zhang
5628831fa6eSGuchun Chen return r;
5634e644fffSHawking Zhang }
5644e644fffSHawking Zhang
nbio_v7_4_init_ras_err_event_athub_interrupt(struct amdgpu_device * adev)5654e644fffSHawking Zhang static int nbio_v7_4_init_ras_err_event_athub_interrupt (struct amdgpu_device *adev)
5664e644fffSHawking Zhang {
5674e644fffSHawking Zhang
5684e644fffSHawking Zhang int r;
5694e644fffSHawking Zhang
5704e644fffSHawking Zhang /* init the irq funcs */
5714e644fffSHawking Zhang adev->nbio.ras_err_event_athub_irq.funcs =
5724e644fffSHawking Zhang &nbio_v7_4_ras_err_event_athub_irq_funcs;
5734e644fffSHawking Zhang adev->nbio.ras_err_event_athub_irq.num_types = 1;
5744e644fffSHawking Zhang
5754e644fffSHawking Zhang /* register ras err event athub interrupt */
5764e644fffSHawking Zhang r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_BIF,
5774e644fffSHawking Zhang NBIF_7_4__SRCID__ERREVENT_ATHUB_INTERRUPT,
5784e644fffSHawking Zhang &adev->nbio.ras_err_event_athub_irq);
5794e644fffSHawking Zhang
5808831fa6eSGuchun Chen return r;
5814e644fffSHawking Zhang }
5824e644fffSHawking Zhang
5835c39d600SLe Ma #define smnPARITY_ERROR_STATUS_UNCORR_GRP2 0x13a20030
584226f4f5aSJohn Clements #define smnPARITY_ERROR_STATUS_UNCORR_GRP2_ALDE 0x13b20030
585226f4f5aSJohn Clements #define smnRAS_GLOBAL_STATUS_LO_ALDE 0x13b20020
5865c39d600SLe Ma
nbio_v7_4_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)58752652ef2SGuchun Chen static void nbio_v7_4_query_ras_error_count(struct amdgpu_device *adev,
58852652ef2SGuchun Chen void *ras_error_status)
58952652ef2SGuchun Chen {
5905c39d600SLe Ma uint32_t global_sts, central_sts, int_eoi, parity_sts;
5911a3f2e8cSGuchun Chen uint32_t corr, fatal, non_fatal;
5921a3f2e8cSGuchun Chen struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
5931a3f2e8cSGuchun Chen
594226f4f5aSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
595226f4f5aSJohn Clements global_sts = RREG32_PCIE(smnRAS_GLOBAL_STATUS_LO_ALDE);
596226f4f5aSJohn Clements else
5971a3f2e8cSGuchun Chen global_sts = RREG32_PCIE(smnRAS_GLOBAL_STATUS_LO);
598226f4f5aSJohn Clements
5991a3f2e8cSGuchun Chen corr = REG_GET_FIELD(global_sts, RAS_GLOBAL_STATUS_LO, ParityErrCorr);
6001a3f2e8cSGuchun Chen fatal = REG_GET_FIELD(global_sts, RAS_GLOBAL_STATUS_LO, ParityErrFatal);
6011a3f2e8cSGuchun Chen non_fatal = REG_GET_FIELD(global_sts, RAS_GLOBAL_STATUS_LO,
6021a3f2e8cSGuchun Chen ParityErrNonFatal);
603226f4f5aSJohn Clements
604226f4f5aSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
605226f4f5aSJohn Clements parity_sts = RREG32_PCIE(smnPARITY_ERROR_STATUS_UNCORR_GRP2_ALDE);
606226f4f5aSJohn Clements else
6075c39d600SLe Ma parity_sts = RREG32_PCIE(smnPARITY_ERROR_STATUS_UNCORR_GRP2);
6081a3f2e8cSGuchun Chen
6091a3f2e8cSGuchun Chen if (corr)
6101a3f2e8cSGuchun Chen err_data->ce_count++;
6111a3f2e8cSGuchun Chen if (fatal)
6121a3f2e8cSGuchun Chen err_data->ue_count++;
6131a3f2e8cSGuchun Chen
6141a3f2e8cSGuchun Chen if (corr || fatal || non_fatal) {
6151a3f2e8cSGuchun Chen central_sts = RREG32_PCIE(smnBIFL_RAS_CENTRAL_STATUS);
616226f4f5aSJohn Clements
6171a3f2e8cSGuchun Chen /* clear error status register */
618226f4f5aSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
619226f4f5aSJohn Clements WREG32_PCIE(smnRAS_GLOBAL_STATUS_LO_ALDE, global_sts);
620226f4f5aSJohn Clements else
6211a3f2e8cSGuchun Chen WREG32_PCIE(smnRAS_GLOBAL_STATUS_LO, global_sts);
6221a3f2e8cSGuchun Chen
6235c39d600SLe Ma if (fatal)
624226f4f5aSJohn Clements {
6255c39d600SLe Ma /* clear parity fatal error indication field */
626226f4f5aSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
627226f4f5aSJohn Clements WREG32_PCIE(smnPARITY_ERROR_STATUS_UNCORR_GRP2_ALDE, parity_sts);
628226f4f5aSJohn Clements else
629226f4f5aSJohn Clements WREG32_PCIE(smnPARITY_ERROR_STATUS_UNCORR_GRP2, parity_sts);
630226f4f5aSJohn Clements }
6315c39d600SLe Ma
6321a3f2e8cSGuchun Chen if (REG_GET_FIELD(central_sts, BIFL_RAS_CENTRAL_STATUS,
6331a3f2e8cSGuchun Chen BIFL_RasContller_Intr_Recv)) {
6341a3f2e8cSGuchun Chen /* clear interrupt status register */
6351a3f2e8cSGuchun Chen WREG32_PCIE(smnBIFL_RAS_CENTRAL_STATUS, central_sts);
6361a3f2e8cSGuchun Chen int_eoi = RREG32_PCIE(smnIOHC_INTERRUPT_EOI);
6371a3f2e8cSGuchun Chen int_eoi = REG_SET_FIELD(int_eoi,
6381a3f2e8cSGuchun Chen IOHC_INTERRUPT_EOI, SMI_EOI, 1);
6391a3f2e8cSGuchun Chen WREG32_PCIE(smnIOHC_INTERRUPT_EOI, int_eoi);
6401a3f2e8cSGuchun Chen }
6411a3f2e8cSGuchun Chen }
64252652ef2SGuchun Chen }
64352652ef2SGuchun Chen
nbio_v7_4_enable_doorbell_interrupt(struct amdgpu_device * adev,bool enable)644956f6705SLe Ma static void nbio_v7_4_enable_doorbell_interrupt(struct amdgpu_device *adev,
645956f6705SLe Ma bool enable)
646956f6705SLe Ma {
64754e6badbSJohn Clements if (adev->asic_type == CHIP_ALDEBARAN)
64854e6badbSJohn Clements WREG32_FIELD15(NBIO, 0, BIF_DOORBELL_INT_CNTL_ALDE,
64954e6badbSJohn Clements DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1);
65054e6badbSJohn Clements else
651956f6705SLe Ma WREG32_FIELD15(NBIO, 0, BIF_DOORBELL_INT_CNTL,
652956f6705SLe Ma DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1);
653956f6705SLe Ma }
654956f6705SLe Ma
6552e54fe5dSyipechai const struct amdgpu_ras_block_hw_ops nbio_v7_4_ras_hw_ops = {
6562e54fe5dSyipechai .query_ras_error_count = nbio_v7_4_query_ras_error_count,
6572e54fe5dSyipechai };
6582e54fe5dSyipechai
6592e54fe5dSyipechai struct amdgpu_nbio_ras nbio_v7_4_ras = {
6602e54fe5dSyipechai .ras_block = {
661bdb3489cSyipechai .ras_comm = {
6622e54fe5dSyipechai .name = "pcie_bif",
6632e54fe5dSyipechai .block = AMDGPU_RAS_BLOCK__PCIE_BIF,
66480ed77f9Syipechai .type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
665bdb3489cSyipechai },
6662e54fe5dSyipechai .hw_ops = &nbio_v7_4_ras_hw_ops,
6672e54fe5dSyipechai .ras_late_init = amdgpu_nbio_ras_late_init,
6682e54fe5dSyipechai },
6696e36f231SHawking Zhang .handle_ras_controller_intr_no_bifring = nbio_v7_4_handle_ras_controller_intr_no_bifring,
6706e36f231SHawking Zhang .handle_ras_err_event_athub_intr_no_bifring = nbio_v7_4_handle_ras_err_event_athub_intr_no_bifring,
6716e36f231SHawking Zhang .init_ras_controller_interrupt = nbio_v7_4_init_ras_controller_interrupt,
6726e36f231SHawking Zhang .init_ras_err_event_athub_interrupt = nbio_v7_4_init_ras_err_event_athub_interrupt,
6736e36f231SHawking Zhang };
6746e36f231SHawking Zhang
6752e54fe5dSyipechai
6766c204906SLijo Lazar #ifdef CONFIG_PCIEASPM
nbio_v7_4_program_ltr(struct amdgpu_device * adev)6779d015c0dSKenneth Feng static void nbio_v7_4_program_ltr(struct amdgpu_device *adev)
6789d015c0dSKenneth Feng {
6799d015c0dSKenneth Feng uint32_t def, data;
6809d015c0dSKenneth Feng
6819d015c0dSKenneth Feng WREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL, 0x75EB);
6829d015c0dSKenneth Feng
6839d015c0dSKenneth Feng def = data = RREG32_PCIE(smnRCC_BIF_STRAP2);
6849d015c0dSKenneth Feng data &= ~RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK;
6859d015c0dSKenneth Feng if (def != data)
6869d015c0dSKenneth Feng WREG32_PCIE(smnRCC_BIF_STRAP2, data);
6879d015c0dSKenneth Feng
6889d015c0dSKenneth Feng def = data = RREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL);
6899d015c0dSKenneth Feng data &= ~EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK;
6909d015c0dSKenneth Feng if (def != data)
6919d015c0dSKenneth Feng WREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL, data);
6929d015c0dSKenneth Feng
6939d015c0dSKenneth Feng def = data = RREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2);
6949d015c0dSKenneth Feng data |= BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK;
6959d015c0dSKenneth Feng if (def != data)
6969d015c0dSKenneth Feng WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data);
6979d015c0dSKenneth Feng }
6986c204906SLijo Lazar #endif
6999d015c0dSKenneth Feng
nbio_v7_4_program_aspm(struct amdgpu_device * adev)7009d015c0dSKenneth Feng static void nbio_v7_4_program_aspm(struct amdgpu_device *adev)
7019d015c0dSKenneth Feng {
7026c204906SLijo Lazar #ifdef CONFIG_PCIEASPM
7039d015c0dSKenneth Feng uint32_t def, data;
7049d015c0dSKenneth Feng
7056ff53495SLijo Lazar if (adev->ip_versions[NBIO_HWIP][0] == IP_VERSION(7, 4, 4))
7066ff53495SLijo Lazar return;
7076ff53495SLijo Lazar
7089d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPCIE_LC_CNTL);
7099d015c0dSKenneth Feng data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK;
7109d015c0dSKenneth Feng data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
7119d015c0dSKenneth Feng data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
7129d015c0dSKenneth Feng if (def != data)
7139d015c0dSKenneth Feng WREG32_PCIE(smnPCIE_LC_CNTL, data);
7149d015c0dSKenneth Feng
7159d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPCIE_LC_CNTL7);
7169d015c0dSKenneth Feng data |= PCIE_LC_CNTL7__LC_NBIF_ASPM_INPUT_EN_MASK;
7179d015c0dSKenneth Feng if (def != data)
7189d015c0dSKenneth Feng WREG32_PCIE(smnPCIE_LC_CNTL7, data);
7199d015c0dSKenneth Feng
7209d015c0dSKenneth Feng def = data = RREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK);
7219d015c0dSKenneth Feng data |= NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK_MASK;
7229d015c0dSKenneth Feng if (def != data)
7239d015c0dSKenneth Feng WREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK, data);
7249d015c0dSKenneth Feng
7259d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPCIE_LC_CNTL3);
7269d015c0dSKenneth Feng data |= PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK;
7279d015c0dSKenneth Feng if (def != data)
7289d015c0dSKenneth Feng WREG32_PCIE(smnPCIE_LC_CNTL3, data);
7299d015c0dSKenneth Feng
7309d015c0dSKenneth Feng def = data = RREG32_PCIE(smnRCC_BIF_STRAP3);
7319d015c0dSKenneth Feng data &= ~RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK;
7329d015c0dSKenneth Feng data &= ~RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK;
7339d015c0dSKenneth Feng if (def != data)
7349d015c0dSKenneth Feng WREG32_PCIE(smnRCC_BIF_STRAP3, data);
7359d015c0dSKenneth Feng
7369d015c0dSKenneth Feng def = data = RREG32_PCIE(smnRCC_BIF_STRAP5);
7379d015c0dSKenneth Feng data &= ~RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK;
7389d015c0dSKenneth Feng if (def != data)
7399d015c0dSKenneth Feng WREG32_PCIE(smnRCC_BIF_STRAP5, data);
7409d015c0dSKenneth Feng
7419d015c0dSKenneth Feng def = data = RREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2);
7429d015c0dSKenneth Feng data &= ~BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK;
7439d015c0dSKenneth Feng if (def != data)
7449d015c0dSKenneth Feng WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data);
7459d015c0dSKenneth Feng
7469d015c0dSKenneth Feng WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP, 0x10011001);
7479d015c0dSKenneth Feng
7489d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPSWUSP0_PCIE_LC_CNTL2);
7499d015c0dSKenneth Feng data |= PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
7509d015c0dSKenneth Feng PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
7519d015c0dSKenneth Feng data &= ~PSWUSP0_PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK;
7529d015c0dSKenneth Feng if (def != data)
7539d015c0dSKenneth Feng WREG32_PCIE(smnPSWUSP0_PCIE_LC_CNTL2, data);
7549d015c0dSKenneth Feng
7559d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPCIE_LC_CNTL6);
7569d015c0dSKenneth Feng data |= PCIE_LC_CNTL6__LC_L1_POWERDOWN_MASK |
7579d015c0dSKenneth Feng PCIE_LC_CNTL6__LC_RX_L0S_STANDBY_EN_MASK;
7589d015c0dSKenneth Feng if (def != data)
7599d015c0dSKenneth Feng WREG32_PCIE(smnPCIE_LC_CNTL6, data);
7609d015c0dSKenneth Feng
7616c204906SLijo Lazar /* Don't bother about LTR if LTR is not enabled
7626c204906SLijo Lazar * in the path */
7636c204906SLijo Lazar if (adev->pdev->ltr_path)
7649d015c0dSKenneth Feng nbio_v7_4_program_ltr(adev);
7659d015c0dSKenneth Feng
7669d015c0dSKenneth Feng def = data = RREG32_PCIE(smnRCC_BIF_STRAP3);
7679d015c0dSKenneth Feng data |= 0x5DE0 << RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT;
7689d015c0dSKenneth Feng data |= 0x0010 << RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT;
7699d015c0dSKenneth Feng if (def != data)
7709d015c0dSKenneth Feng WREG32_PCIE(smnRCC_BIF_STRAP3, data);
7719d015c0dSKenneth Feng
7729d015c0dSKenneth Feng def = data = RREG32_PCIE(smnRCC_BIF_STRAP5);
7739d015c0dSKenneth Feng data |= 0x0010 << RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT;
7749d015c0dSKenneth Feng if (def != data)
7759d015c0dSKenneth Feng WREG32_PCIE(smnRCC_BIF_STRAP5, data);
7769d015c0dSKenneth Feng
7779d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPCIE_LC_CNTL);
7789d015c0dSKenneth Feng data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
7799d015c0dSKenneth Feng data |= 0x9 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT;
7809d015c0dSKenneth Feng data |= 0x1 << PCIE_LC_CNTL__LC_PMI_TO_L1_DIS__SHIFT;
7819d015c0dSKenneth Feng if (def != data)
7829d015c0dSKenneth Feng WREG32_PCIE(smnPCIE_LC_CNTL, data);
7839d015c0dSKenneth Feng
7849d015c0dSKenneth Feng def = data = RREG32_PCIE(smnPCIE_LC_CNTL3);
7859d015c0dSKenneth Feng data &= ~PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK;
7869d015c0dSKenneth Feng if (def != data)
7879d015c0dSKenneth Feng WREG32_PCIE(smnPCIE_LC_CNTL3, data);
7886c204906SLijo Lazar #endif
7899d015c0dSKenneth Feng }
7909d015c0dSKenneth Feng
791fe3c9489SFeifei Xu const struct amdgpu_nbio_funcs nbio_v7_4_funcs = {
792fe3c9489SFeifei Xu .get_hdp_flush_req_offset = nbio_v7_4_get_hdp_flush_req_offset,
793fe3c9489SFeifei Xu .get_hdp_flush_done_offset = nbio_v7_4_get_hdp_flush_done_offset,
794fe3c9489SFeifei Xu .get_pcie_index_offset = nbio_v7_4_get_pcie_index_offset,
795fe3c9489SFeifei Xu .get_pcie_data_offset = nbio_v7_4_get_pcie_data_offset,
796fe3c9489SFeifei Xu .get_rev_id = nbio_v7_4_get_rev_id,
797fe3c9489SFeifei Xu .mc_access_enable = nbio_v7_4_mc_access_enable,
798fe3c9489SFeifei Xu .get_memsize = nbio_v7_4_get_memsize,
799fe3c9489SFeifei Xu .sdma_doorbell_range = nbio_v7_4_sdma_doorbell_range,
80039a5053fSLeo Liu .vcn_doorbell_range = nbio_v7_4_vcn_doorbell_range,
801fe3c9489SFeifei Xu .enable_doorbell_aperture = nbio_v7_4_enable_doorbell_aperture,
802fe3c9489SFeifei Xu .enable_doorbell_selfring_aperture = nbio_v7_4_enable_doorbell_selfring_aperture,
803fe3c9489SFeifei Xu .ih_doorbell_range = nbio_v7_4_ih_doorbell_range,
804956f6705SLe Ma .enable_doorbell_interrupt = nbio_v7_4_enable_doorbell_interrupt,
805fe3c9489SFeifei Xu .update_medium_grain_clock_gating = nbio_v7_4_update_medium_grain_clock_gating,
806fe3c9489SFeifei Xu .update_medium_grain_light_sleep = nbio_v7_4_update_medium_grain_light_sleep,
807fe3c9489SFeifei Xu .get_clockgating_state = nbio_v7_4_get_clockgating_state,
808fe3c9489SFeifei Xu .ih_control = nbio_v7_4_ih_control,
809fe3c9489SFeifei Xu .init_registers = nbio_v7_4_init_registers,
81088807dc8SOak Zeng .remap_hdp_registers = nbio_v7_4_remap_hdp_registers,
8119d015c0dSKenneth Feng .program_aspm = nbio_v7_4_program_aspm,
812fe3c9489SFeifei Xu };
813