1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/pci.h> 25 26 #include "amdgpu.h" 27 #include "amdgpu_ih.h" 28 #include "soc15.h" 29 30 #include "oss/osssys_4_0_offset.h" 31 #include "oss/osssys_4_0_sh_mask.h" 32 33 #include "soc15_common.h" 34 #include "vega10_ih.h" 35 36 #define MAX_REARM_RETRY 10 37 38 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev); 39 40 /** 41 * vega10_ih_init_register_offset - Initialize register offset for ih rings 42 * 43 * @adev: amdgpu_device pointer 44 * 45 * Initialize register offset ih rings (VEGA10). 46 */ 47 static void vega10_ih_init_register_offset(struct amdgpu_device *adev) 48 { 49 struct amdgpu_ih_regs *ih_regs; 50 51 if (adev->irq.ih.ring_size) { 52 ih_regs = &adev->irq.ih.ih_regs; 53 ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); 54 ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); 55 ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); 56 ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); 57 ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); 58 ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); 59 ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); 60 ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); 61 ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; 62 } 63 64 if (adev->irq.ih1.ring_size) { 65 ih_regs = &adev->irq.ih1.ih_regs; 66 ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); 67 ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); 68 ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); 69 ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); 70 ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); 71 ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); 72 ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; 73 } 74 75 if (adev->irq.ih2.ring_size) { 76 ih_regs = &adev->irq.ih2.ih_regs; 77 ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); 78 ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); 79 ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); 80 ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); 81 ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); 82 ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); 83 ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; 84 } 85 } 86 87 /** 88 * vega10_ih_enable_interrupts - Enable the interrupt ring buffer 89 * 90 * @adev: amdgpu_device pointer 91 * 92 * Enable the interrupt ring buffer (VEGA10). 93 */ 94 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev) 95 { 96 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); 97 98 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); 99 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); 100 if (amdgpu_sriov_vf(adev)) { 101 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { 102 DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); 103 return; 104 } 105 } else { 106 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); 107 } 108 adev->irq.ih.enabled = true; 109 110 if (adev->irq.ih1.ring_size) { 111 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); 112 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, 113 RB_ENABLE, 1); 114 if (amdgpu_sriov_vf(adev)) { 115 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, 116 ih_rb_cntl)) { 117 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); 118 return; 119 } 120 } else { 121 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); 122 } 123 adev->irq.ih1.enabled = true; 124 } 125 126 if (adev->irq.ih2.ring_size) { 127 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); 128 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, 129 RB_ENABLE, 1); 130 if (amdgpu_sriov_vf(adev)) { 131 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, 132 ih_rb_cntl)) { 133 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); 134 return; 135 } 136 } else { 137 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); 138 } 139 adev->irq.ih2.enabled = true; 140 } 141 142 if (adev->irq.ih_soft.ring_size) 143 adev->irq.ih_soft.enabled = true; 144 } 145 146 /** 147 * vega10_ih_disable_interrupts - Disable the interrupt ring buffer 148 * 149 * @adev: amdgpu_device pointer 150 * 151 * Disable the interrupt ring buffer (VEGA10). 152 */ 153 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev) 154 { 155 u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); 156 157 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); 158 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); 159 if (amdgpu_sriov_vf(adev)) { 160 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { 161 DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); 162 return; 163 } 164 } else { 165 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); 166 } 167 168 /* set rptr, wptr to 0 */ 169 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); 170 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); 171 adev->irq.ih.enabled = false; 172 adev->irq.ih.rptr = 0; 173 174 if (adev->irq.ih1.ring_size) { 175 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); 176 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, 177 RB_ENABLE, 0); 178 if (amdgpu_sriov_vf(adev)) { 179 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, 180 ih_rb_cntl)) { 181 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); 182 return; 183 } 184 } else { 185 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); 186 } 187 /* set rptr, wptr to 0 */ 188 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); 189 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); 190 adev->irq.ih1.enabled = false; 191 adev->irq.ih1.rptr = 0; 192 } 193 194 if (adev->irq.ih2.ring_size) { 195 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); 196 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, 197 RB_ENABLE, 0); 198 if (amdgpu_sriov_vf(adev)) { 199 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, 200 ih_rb_cntl)) { 201 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); 202 return; 203 } 204 } else { 205 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); 206 } 207 208 /* set rptr, wptr to 0 */ 209 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); 210 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); 211 adev->irq.ih2.enabled = false; 212 adev->irq.ih2.rptr = 0; 213 } 214 } 215 216 /** 217 * vega10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer 218 * 219 * @adev: amdgpu_device pointer 220 * @ih: amdgpu_ih_ring pointet 221 * @enable: true - enable the interrupts, false - disable the interrupts 222 * 223 * Toggle the interrupt ring buffer (VEGA10) 224 */ 225 static int vega10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, 226 struct amdgpu_ih_ring *ih, 227 bool enable) 228 { 229 struct amdgpu_ih_regs *ih_regs; 230 uint32_t tmp; 231 232 ih_regs = &ih->ih_regs; 233 234 tmp = RREG32(ih_regs->ih_rb_cntl); 235 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); 236 /* enable_intr field is only valid in ring0 */ 237 if (ih == &adev->irq.ih) 238 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); 239 if (amdgpu_sriov_vf(adev)) { 240 if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { 241 dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); 242 return -ETIMEDOUT; 243 } 244 } else { 245 WREG32(ih_regs->ih_rb_cntl, tmp); 246 } 247 248 if (enable) { 249 ih->enabled = true; 250 } else { 251 /* set rptr, wptr to 0 */ 252 WREG32(ih_regs->ih_rb_rptr, 0); 253 WREG32(ih_regs->ih_rb_wptr, 0); 254 ih->enabled = false; 255 ih->rptr = 0; 256 } 257 258 return 0; 259 } 260 261 static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) 262 { 263 int rb_bufsz = order_base_2(ih->ring_size / 4); 264 265 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, 266 MC_SPACE, ih->use_bus_addr ? 1 : 4); 267 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, 268 WPTR_OVERFLOW_CLEAR, 1); 269 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, 270 WPTR_OVERFLOW_ENABLE, 1); 271 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); 272 /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register 273 * value is written to memory 274 */ 275 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, 276 WPTR_WRITEBACK_ENABLE, 1); 277 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); 278 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); 279 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); 280 281 return ih_rb_cntl; 282 } 283 284 static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) 285 { 286 u32 ih_doorbell_rtpr = 0; 287 288 if (ih->use_doorbell) { 289 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, 290 IH_DOORBELL_RPTR, OFFSET, 291 ih->doorbell_index); 292 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, 293 IH_DOORBELL_RPTR, 294 ENABLE, 1); 295 } else { 296 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, 297 IH_DOORBELL_RPTR, 298 ENABLE, 0); 299 } 300 return ih_doorbell_rtpr; 301 } 302 303 /** 304 * vega10_ih_enable_ring - enable an ih ring buffer 305 * 306 * @adev: amdgpu_device pointer 307 * @ih: amdgpu_ih_ring pointer 308 * 309 * Enable an ih ring buffer (VEGA10) 310 */ 311 static int vega10_ih_enable_ring(struct amdgpu_device *adev, 312 struct amdgpu_ih_ring *ih) 313 { 314 struct amdgpu_ih_regs *ih_regs; 315 uint32_t tmp; 316 317 ih_regs = &ih->ih_regs; 318 319 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ 320 WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); 321 WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); 322 323 tmp = RREG32(ih_regs->ih_rb_cntl); 324 tmp = vega10_ih_rb_cntl(ih, tmp); 325 if (ih == &adev->irq.ih) 326 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); 327 if (ih == &adev->irq.ih1) { 328 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); 329 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); 330 } 331 if (amdgpu_sriov_vf(adev)) { 332 if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { 333 dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); 334 return -ETIMEDOUT; 335 } 336 } else { 337 WREG32(ih_regs->ih_rb_cntl, tmp); 338 } 339 340 if (ih == &adev->irq.ih) { 341 /* set the ih ring 0 writeback address whether it's enabled or not */ 342 WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); 343 WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); 344 } 345 346 /* set rptr, wptr to 0 */ 347 WREG32(ih_regs->ih_rb_wptr, 0); 348 WREG32(ih_regs->ih_rb_rptr, 0); 349 350 WREG32(ih_regs->ih_doorbell_rptr, vega10_ih_doorbell_rptr(ih)); 351 352 return 0; 353 } 354 355 /** 356 * vega10_ih_irq_init - init and enable the interrupt ring 357 * 358 * @adev: amdgpu_device pointer 359 * 360 * Allocate a ring buffer for the interrupt controller, 361 * enable the RLC, disable interrupts, enable the IH 362 * ring buffer and enable it (VI). 363 * Called at device load and reume. 364 * Returns 0 for success, errors for failure. 365 */ 366 static int vega10_ih_irq_init(struct amdgpu_device *adev) 367 { 368 struct amdgpu_ih_ring *ih; 369 u32 ih_rb_cntl, ih_chicken; 370 int ret = 0; 371 u32 tmp; 372 373 /* disable irqs */ 374 vega10_ih_disable_interrupts(adev); 375 376 adev->nbio.funcs->ih_control(adev); 377 378 ih = &adev->irq.ih; 379 /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ 380 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); 381 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); 382 383 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); 384 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); 385 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 386 !!adev->irq.msi_enabled); 387 if (amdgpu_sriov_vf(adev)) { 388 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { 389 DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); 390 return -ETIMEDOUT; 391 } 392 } else { 393 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); 394 } 395 396 if ((adev->asic_type == CHIP_ARCTURUS && 397 adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || 398 adev->asic_type == CHIP_RENOIR) { 399 ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); 400 if (adev->irq.ih.use_bus_addr) { 401 ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, 402 MC_SPACE_GPA_ENABLE, 1); 403 } else { 404 ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, 405 MC_SPACE_FBPA_ENABLE, 1); 406 } 407 WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); 408 } 409 410 /* set the writeback address whether it's enabled or not */ 411 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, 412 lower_32_bits(ih->wptr_addr)); 413 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, 414 upper_32_bits(ih->wptr_addr) & 0xFFFF); 415 416 /* set rptr, wptr to 0 */ 417 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); 418 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); 419 420 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, 421 vega10_ih_doorbell_rptr(ih)); 422 423 ih = &adev->irq.ih1; 424 if (ih->ring_size) { 425 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); 426 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, 427 (ih->gpu_addr >> 40) & 0xff); 428 429 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); 430 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); 431 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, 432 WPTR_OVERFLOW_ENABLE, 0); 433 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, 434 RB_FULL_DRAIN_ENABLE, 1); 435 if (amdgpu_sriov_vf(adev)) { 436 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, 437 ih_rb_cntl)) { 438 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); 439 return -ETIMEDOUT; 440 } 441 } else { 442 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); 443 } 444 445 /* set rptr, wptr to 0 */ 446 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); 447 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); 448 449 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1, 450 vega10_ih_doorbell_rptr(ih)); 451 } 452 453 ih = &adev->irq.ih2; 454 if (ih->ring_size) { 455 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); 456 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, 457 (ih->gpu_addr >> 40) & 0xff); 458 459 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); 460 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); 461 462 if (amdgpu_sriov_vf(adev)) { 463 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, 464 ih_rb_cntl)) { 465 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); 466 return -ETIMEDOUT; 467 } 468 } else { 469 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); 470 } 471 472 /* set rptr, wptr to 0 */ 473 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); 474 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); 475 476 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2, 477 vega10_ih_doorbell_rptr(ih)); 478 } 479 480 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); 481 tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, 482 CLIENT18_IS_STORM_CLIENT, 1); 483 WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp); 484 485 tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL); 486 tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); 487 WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp); 488 489 pci_set_master(adev->pdev); 490 491 /* enable interrupts */ 492 vega10_ih_enable_interrupts(adev); 493 494 return ret; 495 } 496 497 /** 498 * vega10_ih_irq_disable - disable interrupts 499 * 500 * @adev: amdgpu_device pointer 501 * 502 * Disable interrupts on the hw (VEGA10). 503 */ 504 static void vega10_ih_irq_disable(struct amdgpu_device *adev) 505 { 506 vega10_ih_disable_interrupts(adev); 507 508 /* Wait and acknowledge irq */ 509 mdelay(1); 510 } 511 512 /** 513 * vega10_ih_get_wptr - get the IH ring buffer wptr 514 * 515 * @adev: amdgpu_device pointer 516 * @ih: IH ring buffer to fetch wptr 517 * 518 * Get the IH ring buffer wptr from either the register 519 * or the writeback memory buffer (VEGA10). Also check for 520 * ring buffer overflow and deal with it. 521 * Returns the value of the wptr. 522 */ 523 static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, 524 struct amdgpu_ih_ring *ih) 525 { 526 u32 wptr, reg, tmp; 527 528 wptr = le32_to_cpu(*ih->wptr_cpu); 529 530 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) 531 goto out; 532 533 /* Double check that the overflow wasn't already cleared. */ 534 535 if (ih == &adev->irq.ih) 536 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); 537 else if (ih == &adev->irq.ih1) 538 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); 539 else if (ih == &adev->irq.ih2) 540 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); 541 else 542 BUG(); 543 544 wptr = RREG32_NO_KIQ(reg); 545 if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) 546 goto out; 547 548 wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); 549 550 /* When a ring buffer overflow happen start parsing interrupt 551 * from the last not overwritten vector (wptr + 32). Hopefully 552 * this should allow us to catchup. 553 */ 554 tmp = (wptr + 32) & ih->ptr_mask; 555 dev_warn(adev->dev, "IH ring buffer overflow " 556 "(0x%08X, 0x%08X, 0x%08X)\n", 557 wptr, ih->rptr, tmp); 558 ih->rptr = tmp; 559 560 if (ih == &adev->irq.ih) 561 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); 562 else if (ih == &adev->irq.ih1) 563 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); 564 else if (ih == &adev->irq.ih2) 565 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); 566 else 567 BUG(); 568 569 tmp = RREG32_NO_KIQ(reg); 570 tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); 571 WREG32_NO_KIQ(reg, tmp); 572 573 out: 574 return (wptr & ih->ptr_mask); 575 } 576 577 /** 578 * vega10_ih_decode_iv - decode an interrupt vector 579 * 580 * @adev: amdgpu_device pointer 581 * @ih: IH ring buffer to decode 582 * @entry: IV entry to place decoded information into 583 * 584 * Decodes the interrupt vector at the current rptr 585 * position and also advance the position. 586 */ 587 static void vega10_ih_decode_iv(struct amdgpu_device *adev, 588 struct amdgpu_ih_ring *ih, 589 struct amdgpu_iv_entry *entry) 590 { 591 /* wptr/rptr are in bytes! */ 592 u32 ring_index = ih->rptr >> 2; 593 uint32_t dw[8]; 594 595 dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); 596 dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); 597 dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); 598 dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); 599 dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); 600 dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); 601 dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); 602 dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); 603 604 entry->client_id = dw[0] & 0xff; 605 entry->src_id = (dw[0] >> 8) & 0xff; 606 entry->ring_id = (dw[0] >> 16) & 0xff; 607 entry->vmid = (dw[0] >> 24) & 0xf; 608 entry->vmid_src = (dw[0] >> 31); 609 entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); 610 entry->timestamp_src = dw[2] >> 31; 611 entry->pasid = dw[3] & 0xffff; 612 entry->pasid_src = dw[3] >> 31; 613 entry->src_data[0] = dw[4]; 614 entry->src_data[1] = dw[5]; 615 entry->src_data[2] = dw[6]; 616 entry->src_data[3] = dw[7]; 617 618 /* wptr/rptr are in bytes! */ 619 ih->rptr += 32; 620 } 621 622 /** 623 * vega10_ih_irq_rearm - rearm IRQ if lost 624 * 625 * @adev: amdgpu_device pointer 626 * @ih: IH ring to match 627 * 628 */ 629 static void vega10_ih_irq_rearm(struct amdgpu_device *adev, 630 struct amdgpu_ih_ring *ih) 631 { 632 uint32_t reg_rptr = 0; 633 uint32_t v = 0; 634 uint32_t i = 0; 635 636 if (ih == &adev->irq.ih) 637 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); 638 else if (ih == &adev->irq.ih1) 639 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); 640 else if (ih == &adev->irq.ih2) 641 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); 642 else 643 return; 644 645 /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */ 646 for (i = 0; i < MAX_REARM_RETRY; i++) { 647 v = RREG32_NO_KIQ(reg_rptr); 648 if ((v < ih->ring_size) && (v != ih->rptr)) 649 WDOORBELL32(ih->doorbell_index, ih->rptr); 650 else 651 break; 652 } 653 } 654 655 /** 656 * vega10_ih_set_rptr - set the IH ring buffer rptr 657 * 658 * @adev: amdgpu_device pointer 659 * @ih: IH ring buffer to set rptr 660 * 661 * Set the IH ring buffer rptr. 662 */ 663 static void vega10_ih_set_rptr(struct amdgpu_device *adev, 664 struct amdgpu_ih_ring *ih) 665 { 666 if (ih->use_doorbell) { 667 /* XXX check if swapping is necessary on BE */ 668 *ih->rptr_cpu = ih->rptr; 669 WDOORBELL32(ih->doorbell_index, ih->rptr); 670 671 if (amdgpu_sriov_vf(adev)) 672 vega10_ih_irq_rearm(adev, ih); 673 } else if (ih == &adev->irq.ih) { 674 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); 675 } else if (ih == &adev->irq.ih1) { 676 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); 677 } else if (ih == &adev->irq.ih2) { 678 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); 679 } 680 } 681 682 /** 683 * vega10_ih_self_irq - dispatch work for ring 1 and 2 684 * 685 * @adev: amdgpu_device pointer 686 * @source: irq source 687 * @entry: IV with WPTR update 688 * 689 * Update the WPTR from the IV and schedule work to handle the entries. 690 */ 691 static int vega10_ih_self_irq(struct amdgpu_device *adev, 692 struct amdgpu_irq_src *source, 693 struct amdgpu_iv_entry *entry) 694 { 695 uint32_t wptr = cpu_to_le32(entry->src_data[0]); 696 697 switch (entry->ring_id) { 698 case 1: 699 *adev->irq.ih1.wptr_cpu = wptr; 700 schedule_work(&adev->irq.ih1_work); 701 break; 702 case 2: 703 *adev->irq.ih2.wptr_cpu = wptr; 704 schedule_work(&adev->irq.ih2_work); 705 break; 706 default: break; 707 } 708 return 0; 709 } 710 711 static const struct amdgpu_irq_src_funcs vega10_ih_self_irq_funcs = { 712 .process = vega10_ih_self_irq, 713 }; 714 715 static void vega10_ih_set_self_irq_funcs(struct amdgpu_device *adev) 716 { 717 adev->irq.self_irq.num_types = 0; 718 adev->irq.self_irq.funcs = &vega10_ih_self_irq_funcs; 719 } 720 721 static int vega10_ih_early_init(void *handle) 722 { 723 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 724 725 vega10_ih_set_interrupt_funcs(adev); 726 vega10_ih_set_self_irq_funcs(adev); 727 return 0; 728 } 729 730 static int vega10_ih_sw_init(void *handle) 731 { 732 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 733 int r; 734 735 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0, 736 &adev->irq.self_irq); 737 if (r) 738 return r; 739 740 r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true); 741 if (r) 742 return r; 743 744 adev->irq.ih.use_doorbell = true; 745 adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; 746 747 r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); 748 if (r) 749 return r; 750 751 adev->irq.ih1.use_doorbell = true; 752 adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; 753 754 r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); 755 if (r) 756 return r; 757 758 adev->irq.ih2.use_doorbell = true; 759 adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; 760 761 r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); 762 if (r) 763 return r; 764 765 r = amdgpu_irq_init(adev); 766 767 return r; 768 } 769 770 static int vega10_ih_sw_fini(void *handle) 771 { 772 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 773 774 amdgpu_irq_fini(adev); 775 amdgpu_ih_ring_fini(adev, &adev->irq.ih2); 776 amdgpu_ih_ring_fini(adev, &adev->irq.ih1); 777 amdgpu_ih_ring_fini(adev, &adev->irq.ih); 778 779 return 0; 780 } 781 782 static int vega10_ih_hw_init(void *handle) 783 { 784 int r; 785 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 786 787 r = vega10_ih_irq_init(adev); 788 if (r) 789 return r; 790 791 return 0; 792 } 793 794 static int vega10_ih_hw_fini(void *handle) 795 { 796 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 797 798 vega10_ih_irq_disable(adev); 799 800 return 0; 801 } 802 803 static int vega10_ih_suspend(void *handle) 804 { 805 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 806 807 return vega10_ih_hw_fini(adev); 808 } 809 810 static int vega10_ih_resume(void *handle) 811 { 812 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 813 814 return vega10_ih_hw_init(adev); 815 } 816 817 static bool vega10_ih_is_idle(void *handle) 818 { 819 /* todo */ 820 return true; 821 } 822 823 static int vega10_ih_wait_for_idle(void *handle) 824 { 825 /* todo */ 826 return -ETIMEDOUT; 827 } 828 829 static int vega10_ih_soft_reset(void *handle) 830 { 831 /* todo */ 832 833 return 0; 834 } 835 836 static void vega10_ih_update_clockgating_state(struct amdgpu_device *adev, 837 bool enable) 838 { 839 uint32_t data, def, field_val; 840 841 if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) { 842 def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); 843 field_val = enable ? 0 : 1; 844 /** 845 * Vega10 does not have IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE 846 * and IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field. 847 */ 848 if (adev->asic_type > CHIP_VEGA10) { 849 data = REG_SET_FIELD(data, IH_CLK_CTRL, 850 IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val); 851 data = REG_SET_FIELD(data, IH_CLK_CTRL, 852 IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val); 853 } 854 855 data = REG_SET_FIELD(data, IH_CLK_CTRL, 856 DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); 857 data = REG_SET_FIELD(data, IH_CLK_CTRL, 858 OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val); 859 data = REG_SET_FIELD(data, IH_CLK_CTRL, 860 LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val); 861 data = REG_SET_FIELD(data, IH_CLK_CTRL, 862 DYN_CLK_SOFT_OVERRIDE, field_val); 863 data = REG_SET_FIELD(data, IH_CLK_CTRL, 864 REG_CLK_SOFT_OVERRIDE, field_val); 865 if (def != data) 866 WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data); 867 } 868 } 869 870 static int vega10_ih_set_clockgating_state(void *handle, 871 enum amd_clockgating_state state) 872 { 873 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 874 875 vega10_ih_update_clockgating_state(adev, 876 state == AMD_CG_STATE_GATE); 877 return 0; 878 879 } 880 881 static int vega10_ih_set_powergating_state(void *handle, 882 enum amd_powergating_state state) 883 { 884 return 0; 885 } 886 887 const struct amd_ip_funcs vega10_ih_ip_funcs = { 888 .name = "vega10_ih", 889 .early_init = vega10_ih_early_init, 890 .late_init = NULL, 891 .sw_init = vega10_ih_sw_init, 892 .sw_fini = vega10_ih_sw_fini, 893 .hw_init = vega10_ih_hw_init, 894 .hw_fini = vega10_ih_hw_fini, 895 .suspend = vega10_ih_suspend, 896 .resume = vega10_ih_resume, 897 .is_idle = vega10_ih_is_idle, 898 .wait_for_idle = vega10_ih_wait_for_idle, 899 .soft_reset = vega10_ih_soft_reset, 900 .set_clockgating_state = vega10_ih_set_clockgating_state, 901 .set_powergating_state = vega10_ih_set_powergating_state, 902 }; 903 904 static const struct amdgpu_ih_funcs vega10_ih_funcs = { 905 .get_wptr = vega10_ih_get_wptr, 906 .decode_iv = vega10_ih_decode_iv, 907 .set_rptr = vega10_ih_set_rptr 908 }; 909 910 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev) 911 { 912 adev->irq.ih_funcs = &vega10_ih_funcs; 913 } 914 915 const struct amdgpu_ip_block_version vega10_ih_ip_block = 916 { 917 .type = AMD_IP_BLOCK_TYPE_IH, 918 .major = 4, 919 .minor = 0, 920 .rev = 0, 921 .funcs = &vega10_ih_ip_funcs, 922 }; 923