1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "mmhub_v1_8.h" 25 26 #include "mmhub/mmhub_1_8_0_offset.h" 27 #include "mmhub/mmhub_1_8_0_sh_mask.h" 28 #include "vega10_enum.h" 29 30 #include "soc15_common.h" 31 #include "soc15.h" 32 #include "amdgpu_ras.h" 33 34 #define regVM_L2_CNTL3_DEFAULT 0x80100007 35 #define regVM_L2_CNTL4_DEFAULT 0x000000c1 36 37 static u64 mmhub_v1_8_get_fb_location(struct amdgpu_device *adev) 38 { 39 u64 base = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE); 40 u64 top = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP); 41 42 base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 43 base <<= 24; 44 45 top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK; 46 top <<= 24; 47 48 adev->gmc.fb_start = base; 49 adev->gmc.fb_end = top; 50 51 return base; 52 } 53 54 static void mmhub_v1_8_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 55 uint64_t page_table_base) 56 { 57 struct amdgpu_vmhub *hub; 58 u32 inst_mask; 59 int i; 60 61 inst_mask = adev->aid_mask; 62 for_each_inst(i, inst_mask) { 63 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 64 WREG32_SOC15_OFFSET(MMHUB, i, 65 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 66 hub->ctx_addr_distance * vmid, 67 lower_32_bits(page_table_base)); 68 69 WREG32_SOC15_OFFSET(MMHUB, i, 70 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 71 hub->ctx_addr_distance * vmid, 72 upper_32_bits(page_table_base)); 73 } 74 } 75 76 static void mmhub_v1_8_init_gart_aperture_regs(struct amdgpu_device *adev) 77 { 78 uint64_t pt_base; 79 u32 inst_mask; 80 int i; 81 82 if (adev->gmc.pdb0_bo) 83 pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo); 84 else 85 pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 86 87 mmhub_v1_8_setup_vm_pt_regs(adev, 0, pt_base); 88 89 /* If use GART for FB translation, vmid0 page table covers both 90 * vram and system memory (gart) 91 */ 92 inst_mask = adev->aid_mask; 93 for_each_inst(i, inst_mask) { 94 if (adev->gmc.pdb0_bo) { 95 WREG32_SOC15(MMHUB, i, 96 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 97 (u32)(adev->gmc.fb_start >> 12)); 98 WREG32_SOC15(MMHUB, i, 99 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 100 (u32)(adev->gmc.fb_start >> 44)); 101 102 WREG32_SOC15(MMHUB, i, 103 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 104 (u32)(adev->gmc.gart_end >> 12)); 105 WREG32_SOC15(MMHUB, i, 106 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 107 (u32)(adev->gmc.gart_end >> 44)); 108 109 } else { 110 WREG32_SOC15(MMHUB, i, 111 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 112 (u32)(adev->gmc.gart_start >> 12)); 113 WREG32_SOC15(MMHUB, i, 114 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 115 (u32)(adev->gmc.gart_start >> 44)); 116 117 WREG32_SOC15(MMHUB, i, 118 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 119 (u32)(adev->gmc.gart_end >> 12)); 120 WREG32_SOC15(MMHUB, i, 121 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 122 (u32)(adev->gmc.gart_end >> 44)); 123 } 124 } 125 } 126 127 static void mmhub_v1_8_init_system_aperture_regs(struct amdgpu_device *adev) 128 { 129 uint32_t tmp, inst_mask; 130 uint64_t value; 131 int i; 132 133 inst_mask = adev->aid_mask; 134 for_each_inst(i, inst_mask) { 135 /* Program the AGP BAR */ 136 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BASE, 0); 137 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 138 adev->gmc.agp_start >> 24); 139 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 140 adev->gmc.agp_end >> 24); 141 142 if (amdgpu_sriov_vf(adev)) 143 return; 144 145 /* Program the system aperture low logical page number. */ 146 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 147 min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); 148 149 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 150 max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); 151 152 /* In the case squeezing vram into GART aperture, we don't use 153 * FB aperture and AGP aperture. Disable them. 154 */ 155 if (adev->gmc.pdb0_bo) { 156 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 0xFFFFFF); 157 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 0); 158 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_TOP, 0); 159 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_BASE, 160 0x00FFFFFF); 161 WREG32_SOC15(MMHUB, i, 162 regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 163 0x3FFFFFFF); 164 WREG32_SOC15(MMHUB, i, 165 regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0); 166 } 167 168 /* Set default page address. */ 169 value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr); 170 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 171 (u32)(value >> 12)); 172 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 173 (u32)(value >> 44)); 174 175 /* Program "protection fault". */ 176 WREG32_SOC15(MMHUB, i, 177 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 178 (u32)(adev->dummy_page_addr >> 12)); 179 WREG32_SOC15(MMHUB, i, 180 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 181 (u32)((u64)adev->dummy_page_addr >> 44)); 182 183 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2); 184 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2, 185 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 186 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2, tmp); 187 } 188 } 189 190 static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev) 191 { 192 uint32_t tmp, inst_mask; 193 int i; 194 195 /* Setup TLB control */ 196 inst_mask = adev->aid_mask; 197 for_each_inst(i, inst_mask) { 198 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 199 200 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 201 1); 202 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 203 SYSTEM_ACCESS_MODE, 3); 204 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 205 ENABLE_ADVANCED_DRIVER_MODEL, 1); 206 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 207 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 208 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 209 MTYPE, MTYPE_UC);/* XXX for emulation. */ 210 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 211 212 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 213 } 214 } 215 216 static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev) 217 { 218 uint32_t tmp, inst_mask; 219 int i; 220 221 if (amdgpu_sriov_vf(adev)) 222 return; 223 224 /* Setup L2 cache */ 225 inst_mask = adev->aid_mask; 226 for_each_inst(i, inst_mask) { 227 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL); 228 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); 229 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 230 ENABLE_L2_FRAGMENT_PROCESSING, 1); 231 /* XXX for emulation, Refer to closed source code.*/ 232 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 233 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 234 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 235 0); 236 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 237 CONTEXT1_IDENTITY_ACCESS_MODE, 1); 238 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 239 IDENTITY_MODE_FRAGMENT_SIZE, 0); 240 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL, tmp); 241 242 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL2); 243 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 244 1); 245 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 246 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL2, tmp); 247 248 tmp = regVM_L2_CNTL3_DEFAULT; 249 if (adev->gmc.translate_further) { 250 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12); 251 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 252 L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 253 } else { 254 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9); 255 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 256 L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 257 } 258 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL3, tmp); 259 260 tmp = regVM_L2_CNTL4_DEFAULT; 261 /* For AMD APP APUs setup WC memory */ 262 if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) { 263 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 264 VMC_TAP_PDE_REQUEST_PHYSICAL, 1); 265 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 266 VMC_TAP_PTE_REQUEST_PHYSICAL, 1); 267 } else { 268 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 269 VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 270 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 271 VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 272 } 273 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL4, tmp); 274 } 275 } 276 277 static void mmhub_v1_8_enable_system_domain(struct amdgpu_device *adev) 278 { 279 uint32_t tmp, inst_mask; 280 int i; 281 282 inst_mask = adev->aid_mask; 283 for_each_inst(i, inst_mask) { 284 tmp = RREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL); 285 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 286 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 287 adev->gmc.vmid0_page_table_depth); 288 tmp = REG_SET_FIELD(tmp, 289 VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE, 290 adev->gmc.vmid0_page_table_block_size); 291 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, 292 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 293 WREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL, tmp); 294 } 295 } 296 297 static void mmhub_v1_8_disable_identity_aperture(struct amdgpu_device *adev) 298 { 299 u32 inst_mask; 300 int i; 301 302 if (amdgpu_sriov_vf(adev)) 303 return; 304 305 inst_mask = adev->aid_mask; 306 for_each_inst(i, inst_mask) { 307 WREG32_SOC15(MMHUB, i, 308 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 309 0XFFFFFFFF); 310 WREG32_SOC15(MMHUB, i, 311 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 312 0x0000000F); 313 314 WREG32_SOC15(MMHUB, i, 315 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 316 0); 317 WREG32_SOC15(MMHUB, i, 318 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 319 0); 320 321 WREG32_SOC15(MMHUB, i, 322 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 323 WREG32_SOC15(MMHUB, i, 324 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 325 } 326 } 327 328 static void mmhub_v1_8_setup_vmid_config(struct amdgpu_device *adev) 329 { 330 struct amdgpu_vmhub *hub; 331 unsigned int num_level, block_size; 332 uint32_t tmp, inst_mask; 333 int i, j; 334 335 num_level = adev->vm_manager.num_level; 336 block_size = adev->vm_manager.block_size; 337 if (adev->gmc.translate_further) 338 num_level -= 1; 339 else 340 block_size -= 9; 341 342 inst_mask = adev->aid_mask; 343 for_each_inst(j, inst_mask) { 344 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 345 for (i = 0; i <= 14; i++) { 346 tmp = RREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 347 i); 348 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 349 ENABLE_CONTEXT, 1); 350 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 351 PAGE_TABLE_DEPTH, num_level); 352 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 353 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 354 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 355 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 356 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 357 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 358 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 359 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 360 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 361 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 362 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 363 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 364 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 365 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 366 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 367 PAGE_TABLE_BLOCK_SIZE, 368 block_size); 369 /* On 9.4.3, XNACK can be enabled in the SQ 370 * per-process. Retry faults need to be enabled for 371 * that to work. 372 */ 373 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 374 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1); 375 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 376 i * hub->ctx_distance, tmp); 377 WREG32_SOC15_OFFSET(MMHUB, j, 378 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 379 i * hub->ctx_addr_distance, 0); 380 WREG32_SOC15_OFFSET(MMHUB, j, 381 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 382 i * hub->ctx_addr_distance, 0); 383 WREG32_SOC15_OFFSET(MMHUB, j, 384 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 385 i * hub->ctx_addr_distance, 386 lower_32_bits(adev->vm_manager.max_pfn - 1)); 387 WREG32_SOC15_OFFSET(MMHUB, j, 388 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 389 i * hub->ctx_addr_distance, 390 upper_32_bits(adev->vm_manager.max_pfn - 1)); 391 } 392 } 393 } 394 395 static void mmhub_v1_8_program_invalidation(struct amdgpu_device *adev) 396 { 397 struct amdgpu_vmhub *hub; 398 u32 i, j, inst_mask; 399 400 inst_mask = adev->aid_mask; 401 for_each_inst(j, inst_mask) { 402 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 403 for (i = 0; i < 18; ++i) { 404 WREG32_SOC15_OFFSET(MMHUB, j, 405 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 406 i * hub->eng_addr_distance, 0xffffffff); 407 WREG32_SOC15_OFFSET(MMHUB, j, 408 regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 409 i * hub->eng_addr_distance, 0x1f); 410 } 411 } 412 } 413 414 static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev) 415 { 416 /* GART Enable. */ 417 mmhub_v1_8_init_gart_aperture_regs(adev); 418 mmhub_v1_8_init_system_aperture_regs(adev); 419 mmhub_v1_8_init_tlb_regs(adev); 420 mmhub_v1_8_init_cache_regs(adev); 421 422 mmhub_v1_8_enable_system_domain(adev); 423 mmhub_v1_8_disable_identity_aperture(adev); 424 mmhub_v1_8_setup_vmid_config(adev); 425 mmhub_v1_8_program_invalidation(adev); 426 427 return 0; 428 } 429 430 static void mmhub_v1_8_gart_disable(struct amdgpu_device *adev) 431 { 432 struct amdgpu_vmhub *hub; 433 u32 tmp; 434 u32 i, j, inst_mask; 435 436 /* Disable all tables */ 437 inst_mask = adev->aid_mask; 438 for_each_inst(j, inst_mask) { 439 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 440 for (i = 0; i < 16; i++) 441 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT0_CNTL, 442 i * hub->ctx_distance, 0); 443 444 /* Setup TLB control */ 445 tmp = RREG32_SOC15(MMHUB, j, regMC_VM_MX_L1_TLB_CNTL); 446 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 447 0); 448 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 449 ENABLE_ADVANCED_DRIVER_MODEL, 0); 450 WREG32_SOC15(MMHUB, j, regMC_VM_MX_L1_TLB_CNTL, tmp); 451 452 if (!amdgpu_sriov_vf(adev)) { 453 /* Setup L2 cache */ 454 tmp = RREG32_SOC15(MMHUB, j, regVM_L2_CNTL); 455 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 456 0); 457 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL, tmp); 458 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL3, 0); 459 } 460 } 461 } 462 463 /** 464 * mmhub_v1_8_set_fault_enable_default - update GART/VM fault handling 465 * 466 * @adev: amdgpu_device pointer 467 * @value: true redirects VM faults to the default page 468 */ 469 static void mmhub_v1_8_set_fault_enable_default(struct amdgpu_device *adev, bool value) 470 { 471 u32 tmp, inst_mask; 472 int i; 473 474 if (amdgpu_sriov_vf(adev)) 475 return; 476 477 inst_mask = adev->aid_mask; 478 for_each_inst(i, inst_mask) { 479 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL); 480 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 481 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 482 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 483 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 484 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 485 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 486 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 487 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 488 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 489 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 490 value); 491 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 492 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 493 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 494 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 495 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 496 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 497 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 498 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 499 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 500 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 501 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 502 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 503 if (!value) { 504 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 505 CRASH_ON_NO_RETRY_FAULT, 1); 506 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 507 CRASH_ON_RETRY_FAULT, 1); 508 } 509 510 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL, tmp); 511 } 512 } 513 514 static void mmhub_v1_8_init(struct amdgpu_device *adev) 515 { 516 struct amdgpu_vmhub *hub; 517 u32 inst_mask; 518 int i; 519 520 inst_mask = adev->aid_mask; 521 for_each_inst(i, inst_mask) { 522 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 523 524 hub->ctx0_ptb_addr_lo32 = SOC15_REG_OFFSET(MMHUB, i, 525 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 526 hub->ctx0_ptb_addr_hi32 = SOC15_REG_OFFSET(MMHUB, i, 527 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 528 hub->vm_inv_eng0_req = 529 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_REQ); 530 hub->vm_inv_eng0_ack = 531 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_ACK); 532 hub->vm_context0_cntl = 533 SOC15_REG_OFFSET(MMHUB, i, regVM_CONTEXT0_CNTL); 534 hub->vm_l2_pro_fault_status = SOC15_REG_OFFSET(MMHUB, i, 535 regVM_L2_PROTECTION_FAULT_STATUS); 536 hub->vm_l2_pro_fault_cntl = SOC15_REG_OFFSET(MMHUB, i, 537 regVM_L2_PROTECTION_FAULT_CNTL); 538 539 hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL; 540 hub->ctx_addr_distance = 541 regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 542 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 543 hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - 544 regVM_INVALIDATE_ENG0_REQ; 545 hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 546 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 547 } 548 } 549 550 static int mmhub_v1_8_set_clockgating(struct amdgpu_device *adev, 551 enum amd_clockgating_state state) 552 { 553 return 0; 554 } 555 556 static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags) 557 { 558 559 } 560 561 const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = { 562 .get_fb_location = mmhub_v1_8_get_fb_location, 563 .init = mmhub_v1_8_init, 564 .gart_enable = mmhub_v1_8_gart_enable, 565 .set_fault_enable_default = mmhub_v1_8_set_fault_enable_default, 566 .gart_disable = mmhub_v1_8_gart_disable, 567 .setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs, 568 .set_clockgating = mmhub_v1_8_set_clockgating, 569 .get_clockgating = mmhub_v1_8_get_clockgating, 570 }; 571 572 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = { 573 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_CE_ERR_STATUS_LO, regMMEA0_CE_ERR_STATUS_HI), 574 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, 575 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_CE_ERR_STATUS_LO, regMMEA1_CE_ERR_STATUS_HI), 576 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, 577 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_CE_ERR_STATUS_LO, regMMEA2_CE_ERR_STATUS_HI), 578 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, 579 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_CE_ERR_STATUS_LO, regMMEA3_CE_ERR_STATUS_HI), 580 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, 581 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_CE_ERR_STATUS_LO, regMMEA4_CE_ERR_STATUS_HI), 582 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, 583 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_CE_ERR_STATUS_LO, regMM_CANE_CE_ERR_STATUS_HI), 584 1, 0, "MM_CANE"}, 585 }; 586 587 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ue_reg_list[] = { 588 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_UE_ERR_STATUS_LO, regMMEA0_UE_ERR_STATUS_HI), 589 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, 590 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_UE_ERR_STATUS_LO, regMMEA1_UE_ERR_STATUS_HI), 591 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, 592 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_UE_ERR_STATUS_LO, regMMEA2_UE_ERR_STATUS_HI), 593 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, 594 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_UE_ERR_STATUS_LO, regMMEA3_UE_ERR_STATUS_HI), 595 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, 596 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_UE_ERR_STATUS_LO, regMMEA4_UE_ERR_STATUS_HI), 597 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, 598 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_UE_ERR_STATUS_LO, regMM_CANE_UE_ERR_STATUS_HI), 599 1, 0, "MM_CANE"}, 600 }; 601 602 static const struct amdgpu_ras_memory_id_entry mmhub_v1_8_ras_memory_list[] = { 603 {AMDGPU_MMHUB_WGMI_PAGEMEM, "MMEA_WGMI_PAGEMEM"}, 604 {AMDGPU_MMHUB_RGMI_PAGEMEM, "MMEA_RGMI_PAGEMEM"}, 605 {AMDGPU_MMHUB_WDRAM_PAGEMEM, "MMEA_WDRAM_PAGEMEM"}, 606 {AMDGPU_MMHUB_RDRAM_PAGEMEM, "MMEA_RDRAM_PAGEMEM"}, 607 {AMDGPU_MMHUB_WIO_CMDMEM, "MMEA_WIO_CMDMEM"}, 608 {AMDGPU_MMHUB_RIO_CMDMEM, "MMEA_RIO_CMDMEM"}, 609 {AMDGPU_MMHUB_WGMI_CMDMEM, "MMEA_WGMI_CMDMEM"}, 610 {AMDGPU_MMHUB_RGMI_CMDMEM, "MMEA_RGMI_CMDMEM"}, 611 {AMDGPU_MMHUB_WDRAM_CMDMEM, "MMEA_WDRAM_CMDMEM"}, 612 {AMDGPU_MMHUB_RDRAM_CMDMEM, "MMEA_RDRAM_CMDMEM"}, 613 {AMDGPU_MMHUB_MAM_DMEM0, "MMEA_MAM_DMEM0"}, 614 {AMDGPU_MMHUB_MAM_DMEM1, "MMEA_MAM_DMEM1"}, 615 {AMDGPU_MMHUB_MAM_DMEM2, "MMEA_MAM_DMEM2"}, 616 {AMDGPU_MMHUB_MAM_DMEM3, "MMEA_MAM_DMEM3"}, 617 {AMDGPU_MMHUB_WRET_TAGMEM, "MMEA_WRET_TAGMEM"}, 618 {AMDGPU_MMHUB_RRET_TAGMEM, "MMEA_RRET_TAGMEM"}, 619 {AMDGPU_MMHUB_WIO_DATAMEM, "MMEA_WIO_DATAMEM"}, 620 {AMDGPU_MMHUB_WGMI_DATAMEM, "MMEA_WGMI_DATAMEM"}, 621 {AMDGPU_MMHUB_WDRAM_DATAMEM, "MMEA_WDRAM_DATAMEM"}, 622 }; 623 624 static void mmhub_v1_8_inst_query_ras_error_count(struct amdgpu_device *adev, 625 uint32_t mmhub_inst, 626 void *ras_err_status) 627 { 628 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 629 630 amdgpu_ras_inst_query_ras_error_count(adev, 631 mmhub_v1_8_ce_reg_list, 632 ARRAY_SIZE(mmhub_v1_8_ce_reg_list), 633 mmhub_v1_8_ras_memory_list, 634 ARRAY_SIZE(mmhub_v1_8_ras_memory_list), 635 mmhub_inst, 636 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE, 637 &err_data->ce_count); 638 amdgpu_ras_inst_query_ras_error_count(adev, 639 mmhub_v1_8_ue_reg_list, 640 ARRAY_SIZE(mmhub_v1_8_ue_reg_list), 641 mmhub_v1_8_ras_memory_list, 642 ARRAY_SIZE(mmhub_v1_8_ras_memory_list), 643 mmhub_inst, 644 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 645 &err_data->ue_count); 646 } 647 648 static void mmhub_v1_8_query_ras_error_count(struct amdgpu_device *adev, 649 void *ras_err_status) 650 { 651 uint32_t inst_mask; 652 uint32_t i; 653 654 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 655 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 656 return; 657 } 658 659 inst_mask = adev->aid_mask; 660 for_each_inst(i, inst_mask) 661 mmhub_v1_8_inst_query_ras_error_count(adev, i, ras_err_status); 662 } 663 664 static void mmhub_v1_8_inst_reset_ras_error_count(struct amdgpu_device *adev, 665 uint32_t mmhub_inst) 666 { 667 amdgpu_ras_inst_reset_ras_error_count(adev, 668 mmhub_v1_8_ce_reg_list, 669 ARRAY_SIZE(mmhub_v1_8_ce_reg_list), 670 mmhub_inst); 671 amdgpu_ras_inst_reset_ras_error_count(adev, 672 mmhub_v1_8_ue_reg_list, 673 ARRAY_SIZE(mmhub_v1_8_ue_reg_list), 674 mmhub_inst); 675 } 676 677 static void mmhub_v1_8_reset_ras_error_count(struct amdgpu_device *adev) 678 { 679 uint32_t inst_mask; 680 uint32_t i; 681 682 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 683 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 684 return; 685 } 686 687 inst_mask = adev->aid_mask; 688 for_each_inst(i, inst_mask) 689 mmhub_v1_8_inst_reset_ras_error_count(adev, i); 690 } 691 692 static const u32 mmhub_v1_8_mmea_err_status_reg[] __maybe_unused = { 693 regMMEA0_ERR_STATUS, 694 regMMEA1_ERR_STATUS, 695 regMMEA2_ERR_STATUS, 696 regMMEA3_ERR_STATUS, 697 regMMEA4_ERR_STATUS, 698 }; 699 700 static void mmhub_v1_8_inst_query_ras_err_status(struct amdgpu_device *adev, 701 uint32_t mmhub_inst) 702 { 703 uint32_t reg_value; 704 uint32_t mmea_err_status_addr_dist; 705 uint32_t i; 706 707 /* query mmea ras err status */ 708 mmea_err_status_addr_dist = regMMEA1_ERR_STATUS - regMMEA0_ERR_STATUS; 709 for (i = 0; i < ARRAY_SIZE(mmhub_v1_8_mmea_err_status_reg); i++) { 710 reg_value = RREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 711 regMMEA0_ERR_STATUS, 712 i * mmea_err_status_addr_dist); 713 if (REG_GET_FIELD(reg_value, MMEA0_ERR_STATUS, SDP_RDRSP_STATUS) || 714 REG_GET_FIELD(reg_value, MMEA0_ERR_STATUS, SDP_WRRSP_STATUS) || 715 REG_GET_FIELD(reg_value, MMEA0_ERR_STATUS, SDP_RDRSP_DATAPARITY_ERROR)) { 716 dev_warn(adev->dev, 717 "Detected MMEA%d err in MMHUB%d, status: 0x%x\n", 718 i, mmhub_inst, reg_value); 719 } 720 } 721 722 /* query mm_cane ras err status */ 723 reg_value = RREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ERR_STATUS); 724 if (REG_GET_FIELD(reg_value, MM_CANE_ERR_STATUS, SDPM_RDRSP_STATUS) || 725 REG_GET_FIELD(reg_value, MM_CANE_ERR_STATUS, SDPM_WRRSP_STATUS) || 726 REG_GET_FIELD(reg_value, MM_CANE_ERR_STATUS, SDPM_RDRSP_DATAPARITY_ERROR)) { 727 dev_warn(adev->dev, 728 "Detected MM CANE err in MMHUB%d, status: 0x%x\n", 729 mmhub_inst, reg_value); 730 } 731 } 732 733 static void mmhub_v1_8_query_ras_error_status(struct amdgpu_device *adev) 734 { 735 uint32_t inst_mask; 736 uint32_t i; 737 738 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 739 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 740 return; 741 } 742 743 inst_mask = adev->aid_mask; 744 for_each_inst(i, inst_mask) 745 mmhub_v1_8_inst_query_ras_err_status(adev, i); 746 } 747 748 static void mmhub_v1_8_inst_reset_ras_err_status(struct amdgpu_device *adev, 749 uint32_t mmhub_inst) 750 { 751 uint32_t mmea_cgtt_clk_cntl_addr_dist; 752 uint32_t mmea_err_status_addr_dist; 753 uint32_t reg_value; 754 uint32_t i; 755 756 /* reset mmea ras err status */ 757 mmea_cgtt_clk_cntl_addr_dist = regMMEA1_CGTT_CLK_CTRL - regMMEA0_CGTT_CLK_CTRL; 758 mmea_err_status_addr_dist = regMMEA1_ERR_STATUS - regMMEA0_ERR_STATUS; 759 for (i = 0; i < ARRAY_SIZE(mmhub_v1_8_mmea_err_status_reg); i++) { 760 /* force clk branch on for response path 761 * set MMEA0_CGTT_CLK_CTRL.SOFT_OVERRIDE_RETURN = 1 762 */ 763 reg_value = RREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 764 regMMEA0_CGTT_CLK_CTRL, 765 i * mmea_cgtt_clk_cntl_addr_dist); 766 reg_value = REG_SET_FIELD(reg_value, MMEA0_CGTT_CLK_CTRL, 767 SOFT_OVERRIDE_RETURN, 1); 768 WREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 769 regMMEA0_CGTT_CLK_CTRL, 770 i * mmea_cgtt_clk_cntl_addr_dist, 771 reg_value); 772 773 /* set MMEA0_ERR_STATUS.CLEAR_ERROR_STATUS = 1 */ 774 reg_value = RREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 775 regMMEA0_ERR_STATUS, 776 i * mmea_err_status_addr_dist); 777 reg_value = REG_SET_FIELD(reg_value, MMEA0_ERR_STATUS, 778 CLEAR_ERROR_STATUS, 1); 779 WREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 780 regMMEA0_ERR_STATUS, 781 i * mmea_err_status_addr_dist, 782 reg_value); 783 784 /* set MMEA0_CGTT_CLK_CTRL.SOFT_OVERRIDE_RETURN = 0 */ 785 reg_value = RREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 786 regMMEA0_CGTT_CLK_CTRL, 787 i * mmea_cgtt_clk_cntl_addr_dist); 788 reg_value = REG_SET_FIELD(reg_value, MMEA0_CGTT_CLK_CTRL, 789 SOFT_OVERRIDE_RETURN, 0); 790 WREG32_SOC15_OFFSET(MMHUB, mmhub_inst, 791 regMMEA0_CGTT_CLK_CTRL, 792 i * mmea_cgtt_clk_cntl_addr_dist, 793 reg_value); 794 } 795 796 /* reset mm_cane ras err status 797 * force clk branch on for response path 798 * set MM_CANE_ICG_CTRL.SOFT_OVERRIDE_ATRET = 1 799 */ 800 reg_value = RREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ICG_CTRL); 801 reg_value = REG_SET_FIELD(reg_value, MM_CANE_ICG_CTRL, 802 SOFT_OVERRIDE_ATRET, 1); 803 WREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ICG_CTRL, reg_value); 804 805 /* set MM_CANE_ERR_STATUS.CLEAR_ERROR_STATUS = 1 */ 806 reg_value = RREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ERR_STATUS); 807 reg_value = REG_SET_FIELD(reg_value, MM_CANE_ERR_STATUS, 808 CLEAR_ERROR_STATUS, 1); 809 WREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ERR_STATUS, reg_value); 810 811 /* set MM_CANE_ICG_CTRL.SOFT_OVERRIDE_ATRET = 0 */ 812 reg_value = RREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ICG_CTRL); 813 reg_value = REG_SET_FIELD(reg_value, MM_CANE_ICG_CTRL, 814 SOFT_OVERRIDE_ATRET, 0); 815 WREG32_SOC15(MMHUB, mmhub_inst, regMM_CANE_ICG_CTRL, reg_value); 816 } 817 818 static void mmhub_v1_8_reset_ras_error_status(struct amdgpu_device *adev) 819 { 820 uint32_t inst_mask; 821 uint32_t i; 822 823 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 824 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 825 return; 826 } 827 828 inst_mask = adev->aid_mask; 829 for_each_inst(i, inst_mask) 830 mmhub_v1_8_inst_reset_ras_err_status(adev, i); 831 } 832 833 static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = { 834 .query_ras_error_count = mmhub_v1_8_query_ras_error_count, 835 .reset_ras_error_count = mmhub_v1_8_reset_ras_error_count, 836 .query_ras_error_status = mmhub_v1_8_query_ras_error_status, 837 .reset_ras_error_status = mmhub_v1_8_reset_ras_error_status, 838 }; 839 840 struct amdgpu_mmhub_ras mmhub_v1_8_ras = { 841 .ras_block = { 842 .hw_ops = &mmhub_v1_8_ras_hw_ops, 843 }, 844 }; 845