1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include <drm/drmP.h> 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include "radeon_audio.h" 31 #include "cikd.h" 32 #include "atom.h" 33 #include "cik_blit_shaders.h" 34 #include "radeon_ucode.h" 35 #include "clearstate_ci.h" 36 37 #define SH_MEM_CONFIG_GFX_DEFAULT \ 38 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) 39 40 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 44 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 45 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); 46 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 47 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 48 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 49 50 MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); 51 MODULE_FIRMWARE("radeon/bonaire_me.bin"); 52 MODULE_FIRMWARE("radeon/bonaire_ce.bin"); 53 MODULE_FIRMWARE("radeon/bonaire_mec.bin"); 54 MODULE_FIRMWARE("radeon/bonaire_mc.bin"); 55 MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); 56 MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); 57 MODULE_FIRMWARE("radeon/bonaire_smc.bin"); 58 MODULE_FIRMWARE("radeon/bonaire_k_smc.bin"); 59 60 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 61 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 62 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 63 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 64 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 65 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); 66 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 67 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 68 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 69 70 MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); 71 MODULE_FIRMWARE("radeon/hawaii_me.bin"); 72 MODULE_FIRMWARE("radeon/hawaii_ce.bin"); 73 MODULE_FIRMWARE("radeon/hawaii_mec.bin"); 74 MODULE_FIRMWARE("radeon/hawaii_mc.bin"); 75 MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); 76 MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); 77 MODULE_FIRMWARE("radeon/hawaii_smc.bin"); 78 MODULE_FIRMWARE("radeon/hawaii_k_smc.bin"); 79 80 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 81 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 82 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 83 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 84 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 85 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 86 87 MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); 88 MODULE_FIRMWARE("radeon/kaveri_me.bin"); 89 MODULE_FIRMWARE("radeon/kaveri_ce.bin"); 90 MODULE_FIRMWARE("radeon/kaveri_mec.bin"); 91 MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); 92 MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); 93 MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); 94 95 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 96 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 97 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 98 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 99 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 100 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 101 102 MODULE_FIRMWARE("radeon/kabini_pfp.bin"); 103 MODULE_FIRMWARE("radeon/kabini_me.bin"); 104 MODULE_FIRMWARE("radeon/kabini_ce.bin"); 105 MODULE_FIRMWARE("radeon/kabini_mec.bin"); 106 MODULE_FIRMWARE("radeon/kabini_rlc.bin"); 107 MODULE_FIRMWARE("radeon/kabini_sdma.bin"); 108 109 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); 110 MODULE_FIRMWARE("radeon/MULLINS_me.bin"); 111 MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); 112 MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); 113 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); 114 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); 115 116 MODULE_FIRMWARE("radeon/mullins_pfp.bin"); 117 MODULE_FIRMWARE("radeon/mullins_me.bin"); 118 MODULE_FIRMWARE("radeon/mullins_ce.bin"); 119 MODULE_FIRMWARE("radeon/mullins_mec.bin"); 120 MODULE_FIRMWARE("radeon/mullins_rlc.bin"); 121 MODULE_FIRMWARE("radeon/mullins_sdma.bin"); 122 123 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 124 extern void r600_ih_ring_fini(struct radeon_device *rdev); 125 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 126 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 127 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 128 extern void sumo_rlc_fini(struct radeon_device *rdev); 129 extern int sumo_rlc_init(struct radeon_device *rdev); 130 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); 131 extern void si_rlc_reset(struct radeon_device *rdev); 132 extern void si_init_uvd_internal_cg(struct radeon_device *rdev); 133 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 134 extern int cik_sdma_resume(struct radeon_device *rdev); 135 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable); 136 extern void cik_sdma_fini(struct radeon_device *rdev); 137 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable); 138 static void cik_rlc_stop(struct radeon_device *rdev); 139 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 140 static void cik_program_aspm(struct radeon_device *rdev); 141 static void cik_init_pg(struct radeon_device *rdev); 142 static void cik_init_cg(struct radeon_device *rdev); 143 static void cik_fini_pg(struct radeon_device *rdev); 144 static void cik_fini_cg(struct radeon_device *rdev); 145 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 146 bool enable); 147 148 /** 149 * cik_get_allowed_info_register - fetch the register for the info ioctl 150 * 151 * @rdev: radeon_device pointer 152 * @reg: register offset in bytes 153 * @val: register value 154 * 155 * Returns 0 for success or -EINVAL for an invalid register 156 * 157 */ 158 int cik_get_allowed_info_register(struct radeon_device *rdev, 159 u32 reg, u32 *val) 160 { 161 switch (reg) { 162 case GRBM_STATUS: 163 case GRBM_STATUS2: 164 case GRBM_STATUS_SE0: 165 case GRBM_STATUS_SE1: 166 case GRBM_STATUS_SE2: 167 case GRBM_STATUS_SE3: 168 case SRBM_STATUS: 169 case SRBM_STATUS2: 170 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET): 171 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET): 172 case UVD_STATUS: 173 /* TODO VCE */ 174 *val = RREG32(reg); 175 return 0; 176 default: 177 return -EINVAL; 178 } 179 } 180 181 /* 182 * Indirect registers accessor 183 */ 184 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg) 185 { 186 unsigned long flags; 187 u32 r; 188 189 spin_lock_irqsave(&rdev->didt_idx_lock, flags); 190 WREG32(CIK_DIDT_IND_INDEX, (reg)); 191 r = RREG32(CIK_DIDT_IND_DATA); 192 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); 193 return r; 194 } 195 196 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v) 197 { 198 unsigned long flags; 199 200 spin_lock_irqsave(&rdev->didt_idx_lock, flags); 201 WREG32(CIK_DIDT_IND_INDEX, (reg)); 202 WREG32(CIK_DIDT_IND_DATA, (v)); 203 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); 204 } 205 206 /* get temperature in millidegrees */ 207 int ci_get_temp(struct radeon_device *rdev) 208 { 209 u32 temp; 210 int actual_temp = 0; 211 212 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 213 CTF_TEMP_SHIFT; 214 215 if (temp & 0x200) 216 actual_temp = 255; 217 else 218 actual_temp = temp & 0x1ff; 219 220 actual_temp = actual_temp * 1000; 221 222 return actual_temp; 223 } 224 225 /* get temperature in millidegrees */ 226 int kv_get_temp(struct radeon_device *rdev) 227 { 228 u32 temp; 229 int actual_temp = 0; 230 231 temp = RREG32_SMC(0xC0300E0C); 232 233 if (temp) 234 actual_temp = (temp / 8) - 49; 235 else 236 actual_temp = 0; 237 238 actual_temp = actual_temp * 1000; 239 240 return actual_temp; 241 } 242 243 /* 244 * Indirect registers accessor 245 */ 246 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 247 { 248 unsigned long flags; 249 u32 r; 250 251 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 252 WREG32(PCIE_INDEX, reg); 253 (void)RREG32(PCIE_INDEX); 254 r = RREG32(PCIE_DATA); 255 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 256 return r; 257 } 258 259 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 260 { 261 unsigned long flags; 262 263 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 264 WREG32(PCIE_INDEX, reg); 265 (void)RREG32(PCIE_INDEX); 266 WREG32(PCIE_DATA, v); 267 (void)RREG32(PCIE_DATA); 268 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 269 } 270 271 static const u32 spectre_rlc_save_restore_register_list[] = 272 { 273 (0x0e00 << 16) | (0xc12c >> 2), 274 0x00000000, 275 (0x0e00 << 16) | (0xc140 >> 2), 276 0x00000000, 277 (0x0e00 << 16) | (0xc150 >> 2), 278 0x00000000, 279 (0x0e00 << 16) | (0xc15c >> 2), 280 0x00000000, 281 (0x0e00 << 16) | (0xc168 >> 2), 282 0x00000000, 283 (0x0e00 << 16) | (0xc170 >> 2), 284 0x00000000, 285 (0x0e00 << 16) | (0xc178 >> 2), 286 0x00000000, 287 (0x0e00 << 16) | (0xc204 >> 2), 288 0x00000000, 289 (0x0e00 << 16) | (0xc2b4 >> 2), 290 0x00000000, 291 (0x0e00 << 16) | (0xc2b8 >> 2), 292 0x00000000, 293 (0x0e00 << 16) | (0xc2bc >> 2), 294 0x00000000, 295 (0x0e00 << 16) | (0xc2c0 >> 2), 296 0x00000000, 297 (0x0e00 << 16) | (0x8228 >> 2), 298 0x00000000, 299 (0x0e00 << 16) | (0x829c >> 2), 300 0x00000000, 301 (0x0e00 << 16) | (0x869c >> 2), 302 0x00000000, 303 (0x0600 << 16) | (0x98f4 >> 2), 304 0x00000000, 305 (0x0e00 << 16) | (0x98f8 >> 2), 306 0x00000000, 307 (0x0e00 << 16) | (0x9900 >> 2), 308 0x00000000, 309 (0x0e00 << 16) | (0xc260 >> 2), 310 0x00000000, 311 (0x0e00 << 16) | (0x90e8 >> 2), 312 0x00000000, 313 (0x0e00 << 16) | (0x3c000 >> 2), 314 0x00000000, 315 (0x0e00 << 16) | (0x3c00c >> 2), 316 0x00000000, 317 (0x0e00 << 16) | (0x8c1c >> 2), 318 0x00000000, 319 (0x0e00 << 16) | (0x9700 >> 2), 320 0x00000000, 321 (0x0e00 << 16) | (0xcd20 >> 2), 322 0x00000000, 323 (0x4e00 << 16) | (0xcd20 >> 2), 324 0x00000000, 325 (0x5e00 << 16) | (0xcd20 >> 2), 326 0x00000000, 327 (0x6e00 << 16) | (0xcd20 >> 2), 328 0x00000000, 329 (0x7e00 << 16) | (0xcd20 >> 2), 330 0x00000000, 331 (0x8e00 << 16) | (0xcd20 >> 2), 332 0x00000000, 333 (0x9e00 << 16) | (0xcd20 >> 2), 334 0x00000000, 335 (0xae00 << 16) | (0xcd20 >> 2), 336 0x00000000, 337 (0xbe00 << 16) | (0xcd20 >> 2), 338 0x00000000, 339 (0x0e00 << 16) | (0x89bc >> 2), 340 0x00000000, 341 (0x0e00 << 16) | (0x8900 >> 2), 342 0x00000000, 343 0x3, 344 (0x0e00 << 16) | (0xc130 >> 2), 345 0x00000000, 346 (0x0e00 << 16) | (0xc134 >> 2), 347 0x00000000, 348 (0x0e00 << 16) | (0xc1fc >> 2), 349 0x00000000, 350 (0x0e00 << 16) | (0xc208 >> 2), 351 0x00000000, 352 (0x0e00 << 16) | (0xc264 >> 2), 353 0x00000000, 354 (0x0e00 << 16) | (0xc268 >> 2), 355 0x00000000, 356 (0x0e00 << 16) | (0xc26c >> 2), 357 0x00000000, 358 (0x0e00 << 16) | (0xc270 >> 2), 359 0x00000000, 360 (0x0e00 << 16) | (0xc274 >> 2), 361 0x00000000, 362 (0x0e00 << 16) | (0xc278 >> 2), 363 0x00000000, 364 (0x0e00 << 16) | (0xc27c >> 2), 365 0x00000000, 366 (0x0e00 << 16) | (0xc280 >> 2), 367 0x00000000, 368 (0x0e00 << 16) | (0xc284 >> 2), 369 0x00000000, 370 (0x0e00 << 16) | (0xc288 >> 2), 371 0x00000000, 372 (0x0e00 << 16) | (0xc28c >> 2), 373 0x00000000, 374 (0x0e00 << 16) | (0xc290 >> 2), 375 0x00000000, 376 (0x0e00 << 16) | (0xc294 >> 2), 377 0x00000000, 378 (0x0e00 << 16) | (0xc298 >> 2), 379 0x00000000, 380 (0x0e00 << 16) | (0xc29c >> 2), 381 0x00000000, 382 (0x0e00 << 16) | (0xc2a0 >> 2), 383 0x00000000, 384 (0x0e00 << 16) | (0xc2a4 >> 2), 385 0x00000000, 386 (0x0e00 << 16) | (0xc2a8 >> 2), 387 0x00000000, 388 (0x0e00 << 16) | (0xc2ac >> 2), 389 0x00000000, 390 (0x0e00 << 16) | (0xc2b0 >> 2), 391 0x00000000, 392 (0x0e00 << 16) | (0x301d0 >> 2), 393 0x00000000, 394 (0x0e00 << 16) | (0x30238 >> 2), 395 0x00000000, 396 (0x0e00 << 16) | (0x30250 >> 2), 397 0x00000000, 398 (0x0e00 << 16) | (0x30254 >> 2), 399 0x00000000, 400 (0x0e00 << 16) | (0x30258 >> 2), 401 0x00000000, 402 (0x0e00 << 16) | (0x3025c >> 2), 403 0x00000000, 404 (0x4e00 << 16) | (0xc900 >> 2), 405 0x00000000, 406 (0x5e00 << 16) | (0xc900 >> 2), 407 0x00000000, 408 (0x6e00 << 16) | (0xc900 >> 2), 409 0x00000000, 410 (0x7e00 << 16) | (0xc900 >> 2), 411 0x00000000, 412 (0x8e00 << 16) | (0xc900 >> 2), 413 0x00000000, 414 (0x9e00 << 16) | (0xc900 >> 2), 415 0x00000000, 416 (0xae00 << 16) | (0xc900 >> 2), 417 0x00000000, 418 (0xbe00 << 16) | (0xc900 >> 2), 419 0x00000000, 420 (0x4e00 << 16) | (0xc904 >> 2), 421 0x00000000, 422 (0x5e00 << 16) | (0xc904 >> 2), 423 0x00000000, 424 (0x6e00 << 16) | (0xc904 >> 2), 425 0x00000000, 426 (0x7e00 << 16) | (0xc904 >> 2), 427 0x00000000, 428 (0x8e00 << 16) | (0xc904 >> 2), 429 0x00000000, 430 (0x9e00 << 16) | (0xc904 >> 2), 431 0x00000000, 432 (0xae00 << 16) | (0xc904 >> 2), 433 0x00000000, 434 (0xbe00 << 16) | (0xc904 >> 2), 435 0x00000000, 436 (0x4e00 << 16) | (0xc908 >> 2), 437 0x00000000, 438 (0x5e00 << 16) | (0xc908 >> 2), 439 0x00000000, 440 (0x6e00 << 16) | (0xc908 >> 2), 441 0x00000000, 442 (0x7e00 << 16) | (0xc908 >> 2), 443 0x00000000, 444 (0x8e00 << 16) | (0xc908 >> 2), 445 0x00000000, 446 (0x9e00 << 16) | (0xc908 >> 2), 447 0x00000000, 448 (0xae00 << 16) | (0xc908 >> 2), 449 0x00000000, 450 (0xbe00 << 16) | (0xc908 >> 2), 451 0x00000000, 452 (0x4e00 << 16) | (0xc90c >> 2), 453 0x00000000, 454 (0x5e00 << 16) | (0xc90c >> 2), 455 0x00000000, 456 (0x6e00 << 16) | (0xc90c >> 2), 457 0x00000000, 458 (0x7e00 << 16) | (0xc90c >> 2), 459 0x00000000, 460 (0x8e00 << 16) | (0xc90c >> 2), 461 0x00000000, 462 (0x9e00 << 16) | (0xc90c >> 2), 463 0x00000000, 464 (0xae00 << 16) | (0xc90c >> 2), 465 0x00000000, 466 (0xbe00 << 16) | (0xc90c >> 2), 467 0x00000000, 468 (0x4e00 << 16) | (0xc910 >> 2), 469 0x00000000, 470 (0x5e00 << 16) | (0xc910 >> 2), 471 0x00000000, 472 (0x6e00 << 16) | (0xc910 >> 2), 473 0x00000000, 474 (0x7e00 << 16) | (0xc910 >> 2), 475 0x00000000, 476 (0x8e00 << 16) | (0xc910 >> 2), 477 0x00000000, 478 (0x9e00 << 16) | (0xc910 >> 2), 479 0x00000000, 480 (0xae00 << 16) | (0xc910 >> 2), 481 0x00000000, 482 (0xbe00 << 16) | (0xc910 >> 2), 483 0x00000000, 484 (0x0e00 << 16) | (0xc99c >> 2), 485 0x00000000, 486 (0x0e00 << 16) | (0x9834 >> 2), 487 0x00000000, 488 (0x0000 << 16) | (0x30f00 >> 2), 489 0x00000000, 490 (0x0001 << 16) | (0x30f00 >> 2), 491 0x00000000, 492 (0x0000 << 16) | (0x30f04 >> 2), 493 0x00000000, 494 (0x0001 << 16) | (0x30f04 >> 2), 495 0x00000000, 496 (0x0000 << 16) | (0x30f08 >> 2), 497 0x00000000, 498 (0x0001 << 16) | (0x30f08 >> 2), 499 0x00000000, 500 (0x0000 << 16) | (0x30f0c >> 2), 501 0x00000000, 502 (0x0001 << 16) | (0x30f0c >> 2), 503 0x00000000, 504 (0x0600 << 16) | (0x9b7c >> 2), 505 0x00000000, 506 (0x0e00 << 16) | (0x8a14 >> 2), 507 0x00000000, 508 (0x0e00 << 16) | (0x8a18 >> 2), 509 0x00000000, 510 (0x0600 << 16) | (0x30a00 >> 2), 511 0x00000000, 512 (0x0e00 << 16) | (0x8bf0 >> 2), 513 0x00000000, 514 (0x0e00 << 16) | (0x8bcc >> 2), 515 0x00000000, 516 (0x0e00 << 16) | (0x8b24 >> 2), 517 0x00000000, 518 (0x0e00 << 16) | (0x30a04 >> 2), 519 0x00000000, 520 (0x0600 << 16) | (0x30a10 >> 2), 521 0x00000000, 522 (0x0600 << 16) | (0x30a14 >> 2), 523 0x00000000, 524 (0x0600 << 16) | (0x30a18 >> 2), 525 0x00000000, 526 (0x0600 << 16) | (0x30a2c >> 2), 527 0x00000000, 528 (0x0e00 << 16) | (0xc700 >> 2), 529 0x00000000, 530 (0x0e00 << 16) | (0xc704 >> 2), 531 0x00000000, 532 (0x0e00 << 16) | (0xc708 >> 2), 533 0x00000000, 534 (0x0e00 << 16) | (0xc768 >> 2), 535 0x00000000, 536 (0x0400 << 16) | (0xc770 >> 2), 537 0x00000000, 538 (0x0400 << 16) | (0xc774 >> 2), 539 0x00000000, 540 (0x0400 << 16) | (0xc778 >> 2), 541 0x00000000, 542 (0x0400 << 16) | (0xc77c >> 2), 543 0x00000000, 544 (0x0400 << 16) | (0xc780 >> 2), 545 0x00000000, 546 (0x0400 << 16) | (0xc784 >> 2), 547 0x00000000, 548 (0x0400 << 16) | (0xc788 >> 2), 549 0x00000000, 550 (0x0400 << 16) | (0xc78c >> 2), 551 0x00000000, 552 (0x0400 << 16) | (0xc798 >> 2), 553 0x00000000, 554 (0x0400 << 16) | (0xc79c >> 2), 555 0x00000000, 556 (0x0400 << 16) | (0xc7a0 >> 2), 557 0x00000000, 558 (0x0400 << 16) | (0xc7a4 >> 2), 559 0x00000000, 560 (0x0400 << 16) | (0xc7a8 >> 2), 561 0x00000000, 562 (0x0400 << 16) | (0xc7ac >> 2), 563 0x00000000, 564 (0x0400 << 16) | (0xc7b0 >> 2), 565 0x00000000, 566 (0x0400 << 16) | (0xc7b4 >> 2), 567 0x00000000, 568 (0x0e00 << 16) | (0x9100 >> 2), 569 0x00000000, 570 (0x0e00 << 16) | (0x3c010 >> 2), 571 0x00000000, 572 (0x0e00 << 16) | (0x92a8 >> 2), 573 0x00000000, 574 (0x0e00 << 16) | (0x92ac >> 2), 575 0x00000000, 576 (0x0e00 << 16) | (0x92b4 >> 2), 577 0x00000000, 578 (0x0e00 << 16) | (0x92b8 >> 2), 579 0x00000000, 580 (0x0e00 << 16) | (0x92bc >> 2), 581 0x00000000, 582 (0x0e00 << 16) | (0x92c0 >> 2), 583 0x00000000, 584 (0x0e00 << 16) | (0x92c4 >> 2), 585 0x00000000, 586 (0x0e00 << 16) | (0x92c8 >> 2), 587 0x00000000, 588 (0x0e00 << 16) | (0x92cc >> 2), 589 0x00000000, 590 (0x0e00 << 16) | (0x92d0 >> 2), 591 0x00000000, 592 (0x0e00 << 16) | (0x8c00 >> 2), 593 0x00000000, 594 (0x0e00 << 16) | (0x8c04 >> 2), 595 0x00000000, 596 (0x0e00 << 16) | (0x8c20 >> 2), 597 0x00000000, 598 (0x0e00 << 16) | (0x8c38 >> 2), 599 0x00000000, 600 (0x0e00 << 16) | (0x8c3c >> 2), 601 0x00000000, 602 (0x0e00 << 16) | (0xae00 >> 2), 603 0x00000000, 604 (0x0e00 << 16) | (0x9604 >> 2), 605 0x00000000, 606 (0x0e00 << 16) | (0xac08 >> 2), 607 0x00000000, 608 (0x0e00 << 16) | (0xac0c >> 2), 609 0x00000000, 610 (0x0e00 << 16) | (0xac10 >> 2), 611 0x00000000, 612 (0x0e00 << 16) | (0xac14 >> 2), 613 0x00000000, 614 (0x0e00 << 16) | (0xac58 >> 2), 615 0x00000000, 616 (0x0e00 << 16) | (0xac68 >> 2), 617 0x00000000, 618 (0x0e00 << 16) | (0xac6c >> 2), 619 0x00000000, 620 (0x0e00 << 16) | (0xac70 >> 2), 621 0x00000000, 622 (0x0e00 << 16) | (0xac74 >> 2), 623 0x00000000, 624 (0x0e00 << 16) | (0xac78 >> 2), 625 0x00000000, 626 (0x0e00 << 16) | (0xac7c >> 2), 627 0x00000000, 628 (0x0e00 << 16) | (0xac80 >> 2), 629 0x00000000, 630 (0x0e00 << 16) | (0xac84 >> 2), 631 0x00000000, 632 (0x0e00 << 16) | (0xac88 >> 2), 633 0x00000000, 634 (0x0e00 << 16) | (0xac8c >> 2), 635 0x00000000, 636 (0x0e00 << 16) | (0x970c >> 2), 637 0x00000000, 638 (0x0e00 << 16) | (0x9714 >> 2), 639 0x00000000, 640 (0x0e00 << 16) | (0x9718 >> 2), 641 0x00000000, 642 (0x0e00 << 16) | (0x971c >> 2), 643 0x00000000, 644 (0x0e00 << 16) | (0x31068 >> 2), 645 0x00000000, 646 (0x4e00 << 16) | (0x31068 >> 2), 647 0x00000000, 648 (0x5e00 << 16) | (0x31068 >> 2), 649 0x00000000, 650 (0x6e00 << 16) | (0x31068 >> 2), 651 0x00000000, 652 (0x7e00 << 16) | (0x31068 >> 2), 653 0x00000000, 654 (0x8e00 << 16) | (0x31068 >> 2), 655 0x00000000, 656 (0x9e00 << 16) | (0x31068 >> 2), 657 0x00000000, 658 (0xae00 << 16) | (0x31068 >> 2), 659 0x00000000, 660 (0xbe00 << 16) | (0x31068 >> 2), 661 0x00000000, 662 (0x0e00 << 16) | (0xcd10 >> 2), 663 0x00000000, 664 (0x0e00 << 16) | (0xcd14 >> 2), 665 0x00000000, 666 (0x0e00 << 16) | (0x88b0 >> 2), 667 0x00000000, 668 (0x0e00 << 16) | (0x88b4 >> 2), 669 0x00000000, 670 (0x0e00 << 16) | (0x88b8 >> 2), 671 0x00000000, 672 (0x0e00 << 16) | (0x88bc >> 2), 673 0x00000000, 674 (0x0400 << 16) | (0x89c0 >> 2), 675 0x00000000, 676 (0x0e00 << 16) | (0x88c4 >> 2), 677 0x00000000, 678 (0x0e00 << 16) | (0x88c8 >> 2), 679 0x00000000, 680 (0x0e00 << 16) | (0x88d0 >> 2), 681 0x00000000, 682 (0x0e00 << 16) | (0x88d4 >> 2), 683 0x00000000, 684 (0x0e00 << 16) | (0x88d8 >> 2), 685 0x00000000, 686 (0x0e00 << 16) | (0x8980 >> 2), 687 0x00000000, 688 (0x0e00 << 16) | (0x30938 >> 2), 689 0x00000000, 690 (0x0e00 << 16) | (0x3093c >> 2), 691 0x00000000, 692 (0x0e00 << 16) | (0x30940 >> 2), 693 0x00000000, 694 (0x0e00 << 16) | (0x89a0 >> 2), 695 0x00000000, 696 (0x0e00 << 16) | (0x30900 >> 2), 697 0x00000000, 698 (0x0e00 << 16) | (0x30904 >> 2), 699 0x00000000, 700 (0x0e00 << 16) | (0x89b4 >> 2), 701 0x00000000, 702 (0x0e00 << 16) | (0x3c210 >> 2), 703 0x00000000, 704 (0x0e00 << 16) | (0x3c214 >> 2), 705 0x00000000, 706 (0x0e00 << 16) | (0x3c218 >> 2), 707 0x00000000, 708 (0x0e00 << 16) | (0x8904 >> 2), 709 0x00000000, 710 0x5, 711 (0x0e00 << 16) | (0x8c28 >> 2), 712 (0x0e00 << 16) | (0x8c2c >> 2), 713 (0x0e00 << 16) | (0x8c30 >> 2), 714 (0x0e00 << 16) | (0x8c34 >> 2), 715 (0x0e00 << 16) | (0x9600 >> 2), 716 }; 717 718 static const u32 kalindi_rlc_save_restore_register_list[] = 719 { 720 (0x0e00 << 16) | (0xc12c >> 2), 721 0x00000000, 722 (0x0e00 << 16) | (0xc140 >> 2), 723 0x00000000, 724 (0x0e00 << 16) | (0xc150 >> 2), 725 0x00000000, 726 (0x0e00 << 16) | (0xc15c >> 2), 727 0x00000000, 728 (0x0e00 << 16) | (0xc168 >> 2), 729 0x00000000, 730 (0x0e00 << 16) | (0xc170 >> 2), 731 0x00000000, 732 (0x0e00 << 16) | (0xc204 >> 2), 733 0x00000000, 734 (0x0e00 << 16) | (0xc2b4 >> 2), 735 0x00000000, 736 (0x0e00 << 16) | (0xc2b8 >> 2), 737 0x00000000, 738 (0x0e00 << 16) | (0xc2bc >> 2), 739 0x00000000, 740 (0x0e00 << 16) | (0xc2c0 >> 2), 741 0x00000000, 742 (0x0e00 << 16) | (0x8228 >> 2), 743 0x00000000, 744 (0x0e00 << 16) | (0x829c >> 2), 745 0x00000000, 746 (0x0e00 << 16) | (0x869c >> 2), 747 0x00000000, 748 (0x0600 << 16) | (0x98f4 >> 2), 749 0x00000000, 750 (0x0e00 << 16) | (0x98f8 >> 2), 751 0x00000000, 752 (0x0e00 << 16) | (0x9900 >> 2), 753 0x00000000, 754 (0x0e00 << 16) | (0xc260 >> 2), 755 0x00000000, 756 (0x0e00 << 16) | (0x90e8 >> 2), 757 0x00000000, 758 (0x0e00 << 16) | (0x3c000 >> 2), 759 0x00000000, 760 (0x0e00 << 16) | (0x3c00c >> 2), 761 0x00000000, 762 (0x0e00 << 16) | (0x8c1c >> 2), 763 0x00000000, 764 (0x0e00 << 16) | (0x9700 >> 2), 765 0x00000000, 766 (0x0e00 << 16) | (0xcd20 >> 2), 767 0x00000000, 768 (0x4e00 << 16) | (0xcd20 >> 2), 769 0x00000000, 770 (0x5e00 << 16) | (0xcd20 >> 2), 771 0x00000000, 772 (0x6e00 << 16) | (0xcd20 >> 2), 773 0x00000000, 774 (0x7e00 << 16) | (0xcd20 >> 2), 775 0x00000000, 776 (0x0e00 << 16) | (0x89bc >> 2), 777 0x00000000, 778 (0x0e00 << 16) | (0x8900 >> 2), 779 0x00000000, 780 0x3, 781 (0x0e00 << 16) | (0xc130 >> 2), 782 0x00000000, 783 (0x0e00 << 16) | (0xc134 >> 2), 784 0x00000000, 785 (0x0e00 << 16) | (0xc1fc >> 2), 786 0x00000000, 787 (0x0e00 << 16) | (0xc208 >> 2), 788 0x00000000, 789 (0x0e00 << 16) | (0xc264 >> 2), 790 0x00000000, 791 (0x0e00 << 16) | (0xc268 >> 2), 792 0x00000000, 793 (0x0e00 << 16) | (0xc26c >> 2), 794 0x00000000, 795 (0x0e00 << 16) | (0xc270 >> 2), 796 0x00000000, 797 (0x0e00 << 16) | (0xc274 >> 2), 798 0x00000000, 799 (0x0e00 << 16) | (0xc28c >> 2), 800 0x00000000, 801 (0x0e00 << 16) | (0xc290 >> 2), 802 0x00000000, 803 (0x0e00 << 16) | (0xc294 >> 2), 804 0x00000000, 805 (0x0e00 << 16) | (0xc298 >> 2), 806 0x00000000, 807 (0x0e00 << 16) | (0xc2a0 >> 2), 808 0x00000000, 809 (0x0e00 << 16) | (0xc2a4 >> 2), 810 0x00000000, 811 (0x0e00 << 16) | (0xc2a8 >> 2), 812 0x00000000, 813 (0x0e00 << 16) | (0xc2ac >> 2), 814 0x00000000, 815 (0x0e00 << 16) | (0x301d0 >> 2), 816 0x00000000, 817 (0x0e00 << 16) | (0x30238 >> 2), 818 0x00000000, 819 (0x0e00 << 16) | (0x30250 >> 2), 820 0x00000000, 821 (0x0e00 << 16) | (0x30254 >> 2), 822 0x00000000, 823 (0x0e00 << 16) | (0x30258 >> 2), 824 0x00000000, 825 (0x0e00 << 16) | (0x3025c >> 2), 826 0x00000000, 827 (0x4e00 << 16) | (0xc900 >> 2), 828 0x00000000, 829 (0x5e00 << 16) | (0xc900 >> 2), 830 0x00000000, 831 (0x6e00 << 16) | (0xc900 >> 2), 832 0x00000000, 833 (0x7e00 << 16) | (0xc900 >> 2), 834 0x00000000, 835 (0x4e00 << 16) | (0xc904 >> 2), 836 0x00000000, 837 (0x5e00 << 16) | (0xc904 >> 2), 838 0x00000000, 839 (0x6e00 << 16) | (0xc904 >> 2), 840 0x00000000, 841 (0x7e00 << 16) | (0xc904 >> 2), 842 0x00000000, 843 (0x4e00 << 16) | (0xc908 >> 2), 844 0x00000000, 845 (0x5e00 << 16) | (0xc908 >> 2), 846 0x00000000, 847 (0x6e00 << 16) | (0xc908 >> 2), 848 0x00000000, 849 (0x7e00 << 16) | (0xc908 >> 2), 850 0x00000000, 851 (0x4e00 << 16) | (0xc90c >> 2), 852 0x00000000, 853 (0x5e00 << 16) | (0xc90c >> 2), 854 0x00000000, 855 (0x6e00 << 16) | (0xc90c >> 2), 856 0x00000000, 857 (0x7e00 << 16) | (0xc90c >> 2), 858 0x00000000, 859 (0x4e00 << 16) | (0xc910 >> 2), 860 0x00000000, 861 (0x5e00 << 16) | (0xc910 >> 2), 862 0x00000000, 863 (0x6e00 << 16) | (0xc910 >> 2), 864 0x00000000, 865 (0x7e00 << 16) | (0xc910 >> 2), 866 0x00000000, 867 (0x0e00 << 16) | (0xc99c >> 2), 868 0x00000000, 869 (0x0e00 << 16) | (0x9834 >> 2), 870 0x00000000, 871 (0x0000 << 16) | (0x30f00 >> 2), 872 0x00000000, 873 (0x0000 << 16) | (0x30f04 >> 2), 874 0x00000000, 875 (0x0000 << 16) | (0x30f08 >> 2), 876 0x00000000, 877 (0x0000 << 16) | (0x30f0c >> 2), 878 0x00000000, 879 (0x0600 << 16) | (0x9b7c >> 2), 880 0x00000000, 881 (0x0e00 << 16) | (0x8a14 >> 2), 882 0x00000000, 883 (0x0e00 << 16) | (0x8a18 >> 2), 884 0x00000000, 885 (0x0600 << 16) | (0x30a00 >> 2), 886 0x00000000, 887 (0x0e00 << 16) | (0x8bf0 >> 2), 888 0x00000000, 889 (0x0e00 << 16) | (0x8bcc >> 2), 890 0x00000000, 891 (0x0e00 << 16) | (0x8b24 >> 2), 892 0x00000000, 893 (0x0e00 << 16) | (0x30a04 >> 2), 894 0x00000000, 895 (0x0600 << 16) | (0x30a10 >> 2), 896 0x00000000, 897 (0x0600 << 16) | (0x30a14 >> 2), 898 0x00000000, 899 (0x0600 << 16) | (0x30a18 >> 2), 900 0x00000000, 901 (0x0600 << 16) | (0x30a2c >> 2), 902 0x00000000, 903 (0x0e00 << 16) | (0xc700 >> 2), 904 0x00000000, 905 (0x0e00 << 16) | (0xc704 >> 2), 906 0x00000000, 907 (0x0e00 << 16) | (0xc708 >> 2), 908 0x00000000, 909 (0x0e00 << 16) | (0xc768 >> 2), 910 0x00000000, 911 (0x0400 << 16) | (0xc770 >> 2), 912 0x00000000, 913 (0x0400 << 16) | (0xc774 >> 2), 914 0x00000000, 915 (0x0400 << 16) | (0xc798 >> 2), 916 0x00000000, 917 (0x0400 << 16) | (0xc79c >> 2), 918 0x00000000, 919 (0x0e00 << 16) | (0x9100 >> 2), 920 0x00000000, 921 (0x0e00 << 16) | (0x3c010 >> 2), 922 0x00000000, 923 (0x0e00 << 16) | (0x8c00 >> 2), 924 0x00000000, 925 (0x0e00 << 16) | (0x8c04 >> 2), 926 0x00000000, 927 (0x0e00 << 16) | (0x8c20 >> 2), 928 0x00000000, 929 (0x0e00 << 16) | (0x8c38 >> 2), 930 0x00000000, 931 (0x0e00 << 16) | (0x8c3c >> 2), 932 0x00000000, 933 (0x0e00 << 16) | (0xae00 >> 2), 934 0x00000000, 935 (0x0e00 << 16) | (0x9604 >> 2), 936 0x00000000, 937 (0x0e00 << 16) | (0xac08 >> 2), 938 0x00000000, 939 (0x0e00 << 16) | (0xac0c >> 2), 940 0x00000000, 941 (0x0e00 << 16) | (0xac10 >> 2), 942 0x00000000, 943 (0x0e00 << 16) | (0xac14 >> 2), 944 0x00000000, 945 (0x0e00 << 16) | (0xac58 >> 2), 946 0x00000000, 947 (0x0e00 << 16) | (0xac68 >> 2), 948 0x00000000, 949 (0x0e00 << 16) | (0xac6c >> 2), 950 0x00000000, 951 (0x0e00 << 16) | (0xac70 >> 2), 952 0x00000000, 953 (0x0e00 << 16) | (0xac74 >> 2), 954 0x00000000, 955 (0x0e00 << 16) | (0xac78 >> 2), 956 0x00000000, 957 (0x0e00 << 16) | (0xac7c >> 2), 958 0x00000000, 959 (0x0e00 << 16) | (0xac80 >> 2), 960 0x00000000, 961 (0x0e00 << 16) | (0xac84 >> 2), 962 0x00000000, 963 (0x0e00 << 16) | (0xac88 >> 2), 964 0x00000000, 965 (0x0e00 << 16) | (0xac8c >> 2), 966 0x00000000, 967 (0x0e00 << 16) | (0x970c >> 2), 968 0x00000000, 969 (0x0e00 << 16) | (0x9714 >> 2), 970 0x00000000, 971 (0x0e00 << 16) | (0x9718 >> 2), 972 0x00000000, 973 (0x0e00 << 16) | (0x971c >> 2), 974 0x00000000, 975 (0x0e00 << 16) | (0x31068 >> 2), 976 0x00000000, 977 (0x4e00 << 16) | (0x31068 >> 2), 978 0x00000000, 979 (0x5e00 << 16) | (0x31068 >> 2), 980 0x00000000, 981 (0x6e00 << 16) | (0x31068 >> 2), 982 0x00000000, 983 (0x7e00 << 16) | (0x31068 >> 2), 984 0x00000000, 985 (0x0e00 << 16) | (0xcd10 >> 2), 986 0x00000000, 987 (0x0e00 << 16) | (0xcd14 >> 2), 988 0x00000000, 989 (0x0e00 << 16) | (0x88b0 >> 2), 990 0x00000000, 991 (0x0e00 << 16) | (0x88b4 >> 2), 992 0x00000000, 993 (0x0e00 << 16) | (0x88b8 >> 2), 994 0x00000000, 995 (0x0e00 << 16) | (0x88bc >> 2), 996 0x00000000, 997 (0x0400 << 16) | (0x89c0 >> 2), 998 0x00000000, 999 (0x0e00 << 16) | (0x88c4 >> 2), 1000 0x00000000, 1001 (0x0e00 << 16) | (0x88c8 >> 2), 1002 0x00000000, 1003 (0x0e00 << 16) | (0x88d0 >> 2), 1004 0x00000000, 1005 (0x0e00 << 16) | (0x88d4 >> 2), 1006 0x00000000, 1007 (0x0e00 << 16) | (0x88d8 >> 2), 1008 0x00000000, 1009 (0x0e00 << 16) | (0x8980 >> 2), 1010 0x00000000, 1011 (0x0e00 << 16) | (0x30938 >> 2), 1012 0x00000000, 1013 (0x0e00 << 16) | (0x3093c >> 2), 1014 0x00000000, 1015 (0x0e00 << 16) | (0x30940 >> 2), 1016 0x00000000, 1017 (0x0e00 << 16) | (0x89a0 >> 2), 1018 0x00000000, 1019 (0x0e00 << 16) | (0x30900 >> 2), 1020 0x00000000, 1021 (0x0e00 << 16) | (0x30904 >> 2), 1022 0x00000000, 1023 (0x0e00 << 16) | (0x89b4 >> 2), 1024 0x00000000, 1025 (0x0e00 << 16) | (0x3e1fc >> 2), 1026 0x00000000, 1027 (0x0e00 << 16) | (0x3c210 >> 2), 1028 0x00000000, 1029 (0x0e00 << 16) | (0x3c214 >> 2), 1030 0x00000000, 1031 (0x0e00 << 16) | (0x3c218 >> 2), 1032 0x00000000, 1033 (0x0e00 << 16) | (0x8904 >> 2), 1034 0x00000000, 1035 0x5, 1036 (0x0e00 << 16) | (0x8c28 >> 2), 1037 (0x0e00 << 16) | (0x8c2c >> 2), 1038 (0x0e00 << 16) | (0x8c30 >> 2), 1039 (0x0e00 << 16) | (0x8c34 >> 2), 1040 (0x0e00 << 16) | (0x9600 >> 2), 1041 }; 1042 1043 static const u32 bonaire_golden_spm_registers[] = 1044 { 1045 0x30800, 0xe0ffffff, 0xe0000000 1046 }; 1047 1048 static const u32 bonaire_golden_common_registers[] = 1049 { 1050 0xc770, 0xffffffff, 0x00000800, 1051 0xc774, 0xffffffff, 0x00000800, 1052 0xc798, 0xffffffff, 0x00007fbf, 1053 0xc79c, 0xffffffff, 0x00007faf 1054 }; 1055 1056 static const u32 bonaire_golden_registers[] = 1057 { 1058 0x3354, 0x00000333, 0x00000333, 1059 0x3350, 0x000c0fc0, 0x00040200, 1060 0x9a10, 0x00010000, 0x00058208, 1061 0x3c000, 0xffff1fff, 0x00140000, 1062 0x3c200, 0xfdfc0fff, 0x00000100, 1063 0x3c234, 0x40000000, 0x40000200, 1064 0x9830, 0xffffffff, 0x00000000, 1065 0x9834, 0xf00fffff, 0x00000400, 1066 0x9838, 0x0002021c, 0x00020200, 1067 0xc78, 0x00000080, 0x00000000, 1068 0x5bb0, 0x000000f0, 0x00000070, 1069 0x5bc0, 0xf0311fff, 0x80300000, 1070 0x98f8, 0x73773777, 0x12010001, 1071 0x350c, 0x00810000, 0x408af000, 1072 0x7030, 0x31000111, 0x00000011, 1073 0x2f48, 0x73773777, 0x12010001, 1074 0x220c, 0x00007fb6, 0x0021a1b1, 1075 0x2210, 0x00007fb6, 0x002021b1, 1076 0x2180, 0x00007fb6, 0x00002191, 1077 0x2218, 0x00007fb6, 0x002121b1, 1078 0x221c, 0x00007fb6, 0x002021b1, 1079 0x21dc, 0x00007fb6, 0x00002191, 1080 0x21e0, 0x00007fb6, 0x00002191, 1081 0x3628, 0x0000003f, 0x0000000a, 1082 0x362c, 0x0000003f, 0x0000000a, 1083 0x2ae4, 0x00073ffe, 0x000022a2, 1084 0x240c, 0x000007ff, 0x00000000, 1085 0x8a14, 0xf000003f, 0x00000007, 1086 0x8bf0, 0x00002001, 0x00000001, 1087 0x8b24, 0xffffffff, 0x00ffffff, 1088 0x30a04, 0x0000ff0f, 0x00000000, 1089 0x28a4c, 0x07ffffff, 0x06000000, 1090 0x4d8, 0x00000fff, 0x00000100, 1091 0x3e78, 0x00000001, 0x00000002, 1092 0x9100, 0x03000000, 0x0362c688, 1093 0x8c00, 0x000000ff, 0x00000001, 1094 0xe40, 0x00001fff, 0x00001fff, 1095 0x9060, 0x0000007f, 0x00000020, 1096 0x9508, 0x00010000, 0x00010000, 1097 0xac14, 0x000003ff, 0x000000f3, 1098 0xac0c, 0xffffffff, 0x00001032 1099 }; 1100 1101 static const u32 bonaire_mgcg_cgcg_init[] = 1102 { 1103 0xc420, 0xffffffff, 0xfffffffc, 1104 0x30800, 0xffffffff, 0xe0000000, 1105 0x3c2a0, 0xffffffff, 0x00000100, 1106 0x3c208, 0xffffffff, 0x00000100, 1107 0x3c2c0, 0xffffffff, 0xc0000100, 1108 0x3c2c8, 0xffffffff, 0xc0000100, 1109 0x3c2c4, 0xffffffff, 0xc0000100, 1110 0x55e4, 0xffffffff, 0x00600100, 1111 0x3c280, 0xffffffff, 0x00000100, 1112 0x3c214, 0xffffffff, 0x06000100, 1113 0x3c220, 0xffffffff, 0x00000100, 1114 0x3c218, 0xffffffff, 0x06000100, 1115 0x3c204, 0xffffffff, 0x00000100, 1116 0x3c2e0, 0xffffffff, 0x00000100, 1117 0x3c224, 0xffffffff, 0x00000100, 1118 0x3c200, 0xffffffff, 0x00000100, 1119 0x3c230, 0xffffffff, 0x00000100, 1120 0x3c234, 0xffffffff, 0x00000100, 1121 0x3c250, 0xffffffff, 0x00000100, 1122 0x3c254, 0xffffffff, 0x00000100, 1123 0x3c258, 0xffffffff, 0x00000100, 1124 0x3c25c, 0xffffffff, 0x00000100, 1125 0x3c260, 0xffffffff, 0x00000100, 1126 0x3c27c, 0xffffffff, 0x00000100, 1127 0x3c278, 0xffffffff, 0x00000100, 1128 0x3c210, 0xffffffff, 0x06000100, 1129 0x3c290, 0xffffffff, 0x00000100, 1130 0x3c274, 0xffffffff, 0x00000100, 1131 0x3c2b4, 0xffffffff, 0x00000100, 1132 0x3c2b0, 0xffffffff, 0x00000100, 1133 0x3c270, 0xffffffff, 0x00000100, 1134 0x30800, 0xffffffff, 0xe0000000, 1135 0x3c020, 0xffffffff, 0x00010000, 1136 0x3c024, 0xffffffff, 0x00030002, 1137 0x3c028, 0xffffffff, 0x00040007, 1138 0x3c02c, 0xffffffff, 0x00060005, 1139 0x3c030, 0xffffffff, 0x00090008, 1140 0x3c034, 0xffffffff, 0x00010000, 1141 0x3c038, 0xffffffff, 0x00030002, 1142 0x3c03c, 0xffffffff, 0x00040007, 1143 0x3c040, 0xffffffff, 0x00060005, 1144 0x3c044, 0xffffffff, 0x00090008, 1145 0x3c048, 0xffffffff, 0x00010000, 1146 0x3c04c, 0xffffffff, 0x00030002, 1147 0x3c050, 0xffffffff, 0x00040007, 1148 0x3c054, 0xffffffff, 0x00060005, 1149 0x3c058, 0xffffffff, 0x00090008, 1150 0x3c05c, 0xffffffff, 0x00010000, 1151 0x3c060, 0xffffffff, 0x00030002, 1152 0x3c064, 0xffffffff, 0x00040007, 1153 0x3c068, 0xffffffff, 0x00060005, 1154 0x3c06c, 0xffffffff, 0x00090008, 1155 0x3c070, 0xffffffff, 0x00010000, 1156 0x3c074, 0xffffffff, 0x00030002, 1157 0x3c078, 0xffffffff, 0x00040007, 1158 0x3c07c, 0xffffffff, 0x00060005, 1159 0x3c080, 0xffffffff, 0x00090008, 1160 0x3c084, 0xffffffff, 0x00010000, 1161 0x3c088, 0xffffffff, 0x00030002, 1162 0x3c08c, 0xffffffff, 0x00040007, 1163 0x3c090, 0xffffffff, 0x00060005, 1164 0x3c094, 0xffffffff, 0x00090008, 1165 0x3c098, 0xffffffff, 0x00010000, 1166 0x3c09c, 0xffffffff, 0x00030002, 1167 0x3c0a0, 0xffffffff, 0x00040007, 1168 0x3c0a4, 0xffffffff, 0x00060005, 1169 0x3c0a8, 0xffffffff, 0x00090008, 1170 0x3c000, 0xffffffff, 0x96e00200, 1171 0x8708, 0xffffffff, 0x00900100, 1172 0xc424, 0xffffffff, 0x0020003f, 1173 0x38, 0xffffffff, 0x0140001c, 1174 0x3c, 0x000f0000, 0x000f0000, 1175 0x220, 0xffffffff, 0xC060000C, 1176 0x224, 0xc0000fff, 0x00000100, 1177 0xf90, 0xffffffff, 0x00000100, 1178 0xf98, 0x00000101, 0x00000000, 1179 0x20a8, 0xffffffff, 0x00000104, 1180 0x55e4, 0xff000fff, 0x00000100, 1181 0x30cc, 0xc0000fff, 0x00000104, 1182 0xc1e4, 0x00000001, 0x00000001, 1183 0xd00c, 0xff000ff0, 0x00000100, 1184 0xd80c, 0xff000ff0, 0x00000100 1185 }; 1186 1187 static const u32 spectre_golden_spm_registers[] = 1188 { 1189 0x30800, 0xe0ffffff, 0xe0000000 1190 }; 1191 1192 static const u32 spectre_golden_common_registers[] = 1193 { 1194 0xc770, 0xffffffff, 0x00000800, 1195 0xc774, 0xffffffff, 0x00000800, 1196 0xc798, 0xffffffff, 0x00007fbf, 1197 0xc79c, 0xffffffff, 0x00007faf 1198 }; 1199 1200 static const u32 spectre_golden_registers[] = 1201 { 1202 0x3c000, 0xffff1fff, 0x96940200, 1203 0x3c00c, 0xffff0001, 0xff000000, 1204 0x3c200, 0xfffc0fff, 0x00000100, 1205 0x6ed8, 0x00010101, 0x00010000, 1206 0x9834, 0xf00fffff, 0x00000400, 1207 0x9838, 0xfffffffc, 0x00020200, 1208 0x5bb0, 0x000000f0, 0x00000070, 1209 0x5bc0, 0xf0311fff, 0x80300000, 1210 0x98f8, 0x73773777, 0x12010001, 1211 0x9b7c, 0x00ff0000, 0x00fc0000, 1212 0x2f48, 0x73773777, 0x12010001, 1213 0x8a14, 0xf000003f, 0x00000007, 1214 0x8b24, 0xffffffff, 0x00ffffff, 1215 0x28350, 0x3f3f3fff, 0x00000082, 1216 0x28354, 0x0000003f, 0x00000000, 1217 0x3e78, 0x00000001, 0x00000002, 1218 0x913c, 0xffff03df, 0x00000004, 1219 0xc768, 0x00000008, 0x00000008, 1220 0x8c00, 0x000008ff, 0x00000800, 1221 0x9508, 0x00010000, 0x00010000, 1222 0xac0c, 0xffffffff, 0x54763210, 1223 0x214f8, 0x01ff01ff, 0x00000002, 1224 0x21498, 0x007ff800, 0x00200000, 1225 0x2015c, 0xffffffff, 0x00000f40, 1226 0x30934, 0xffffffff, 0x00000001 1227 }; 1228 1229 static const u32 spectre_mgcg_cgcg_init[] = 1230 { 1231 0xc420, 0xffffffff, 0xfffffffc, 1232 0x30800, 0xffffffff, 0xe0000000, 1233 0x3c2a0, 0xffffffff, 0x00000100, 1234 0x3c208, 0xffffffff, 0x00000100, 1235 0x3c2c0, 0xffffffff, 0x00000100, 1236 0x3c2c8, 0xffffffff, 0x00000100, 1237 0x3c2c4, 0xffffffff, 0x00000100, 1238 0x55e4, 0xffffffff, 0x00600100, 1239 0x3c280, 0xffffffff, 0x00000100, 1240 0x3c214, 0xffffffff, 0x06000100, 1241 0x3c220, 0xffffffff, 0x00000100, 1242 0x3c218, 0xffffffff, 0x06000100, 1243 0x3c204, 0xffffffff, 0x00000100, 1244 0x3c2e0, 0xffffffff, 0x00000100, 1245 0x3c224, 0xffffffff, 0x00000100, 1246 0x3c200, 0xffffffff, 0x00000100, 1247 0x3c230, 0xffffffff, 0x00000100, 1248 0x3c234, 0xffffffff, 0x00000100, 1249 0x3c250, 0xffffffff, 0x00000100, 1250 0x3c254, 0xffffffff, 0x00000100, 1251 0x3c258, 0xffffffff, 0x00000100, 1252 0x3c25c, 0xffffffff, 0x00000100, 1253 0x3c260, 0xffffffff, 0x00000100, 1254 0x3c27c, 0xffffffff, 0x00000100, 1255 0x3c278, 0xffffffff, 0x00000100, 1256 0x3c210, 0xffffffff, 0x06000100, 1257 0x3c290, 0xffffffff, 0x00000100, 1258 0x3c274, 0xffffffff, 0x00000100, 1259 0x3c2b4, 0xffffffff, 0x00000100, 1260 0x3c2b0, 0xffffffff, 0x00000100, 1261 0x3c270, 0xffffffff, 0x00000100, 1262 0x30800, 0xffffffff, 0xe0000000, 1263 0x3c020, 0xffffffff, 0x00010000, 1264 0x3c024, 0xffffffff, 0x00030002, 1265 0x3c028, 0xffffffff, 0x00040007, 1266 0x3c02c, 0xffffffff, 0x00060005, 1267 0x3c030, 0xffffffff, 0x00090008, 1268 0x3c034, 0xffffffff, 0x00010000, 1269 0x3c038, 0xffffffff, 0x00030002, 1270 0x3c03c, 0xffffffff, 0x00040007, 1271 0x3c040, 0xffffffff, 0x00060005, 1272 0x3c044, 0xffffffff, 0x00090008, 1273 0x3c048, 0xffffffff, 0x00010000, 1274 0x3c04c, 0xffffffff, 0x00030002, 1275 0x3c050, 0xffffffff, 0x00040007, 1276 0x3c054, 0xffffffff, 0x00060005, 1277 0x3c058, 0xffffffff, 0x00090008, 1278 0x3c05c, 0xffffffff, 0x00010000, 1279 0x3c060, 0xffffffff, 0x00030002, 1280 0x3c064, 0xffffffff, 0x00040007, 1281 0x3c068, 0xffffffff, 0x00060005, 1282 0x3c06c, 0xffffffff, 0x00090008, 1283 0x3c070, 0xffffffff, 0x00010000, 1284 0x3c074, 0xffffffff, 0x00030002, 1285 0x3c078, 0xffffffff, 0x00040007, 1286 0x3c07c, 0xffffffff, 0x00060005, 1287 0x3c080, 0xffffffff, 0x00090008, 1288 0x3c084, 0xffffffff, 0x00010000, 1289 0x3c088, 0xffffffff, 0x00030002, 1290 0x3c08c, 0xffffffff, 0x00040007, 1291 0x3c090, 0xffffffff, 0x00060005, 1292 0x3c094, 0xffffffff, 0x00090008, 1293 0x3c098, 0xffffffff, 0x00010000, 1294 0x3c09c, 0xffffffff, 0x00030002, 1295 0x3c0a0, 0xffffffff, 0x00040007, 1296 0x3c0a4, 0xffffffff, 0x00060005, 1297 0x3c0a8, 0xffffffff, 0x00090008, 1298 0x3c0ac, 0xffffffff, 0x00010000, 1299 0x3c0b0, 0xffffffff, 0x00030002, 1300 0x3c0b4, 0xffffffff, 0x00040007, 1301 0x3c0b8, 0xffffffff, 0x00060005, 1302 0x3c0bc, 0xffffffff, 0x00090008, 1303 0x3c000, 0xffffffff, 0x96e00200, 1304 0x8708, 0xffffffff, 0x00900100, 1305 0xc424, 0xffffffff, 0x0020003f, 1306 0x38, 0xffffffff, 0x0140001c, 1307 0x3c, 0x000f0000, 0x000f0000, 1308 0x220, 0xffffffff, 0xC060000C, 1309 0x224, 0xc0000fff, 0x00000100, 1310 0xf90, 0xffffffff, 0x00000100, 1311 0xf98, 0x00000101, 0x00000000, 1312 0x20a8, 0xffffffff, 0x00000104, 1313 0x55e4, 0xff000fff, 0x00000100, 1314 0x30cc, 0xc0000fff, 0x00000104, 1315 0xc1e4, 0x00000001, 0x00000001, 1316 0xd00c, 0xff000ff0, 0x00000100, 1317 0xd80c, 0xff000ff0, 0x00000100 1318 }; 1319 1320 static const u32 kalindi_golden_spm_registers[] = 1321 { 1322 0x30800, 0xe0ffffff, 0xe0000000 1323 }; 1324 1325 static const u32 kalindi_golden_common_registers[] = 1326 { 1327 0xc770, 0xffffffff, 0x00000800, 1328 0xc774, 0xffffffff, 0x00000800, 1329 0xc798, 0xffffffff, 0x00007fbf, 1330 0xc79c, 0xffffffff, 0x00007faf 1331 }; 1332 1333 static const u32 kalindi_golden_registers[] = 1334 { 1335 0x3c000, 0xffffdfff, 0x6e944040, 1336 0x55e4, 0xff607fff, 0xfc000100, 1337 0x3c220, 0xff000fff, 0x00000100, 1338 0x3c224, 0xff000fff, 0x00000100, 1339 0x3c200, 0xfffc0fff, 0x00000100, 1340 0x6ed8, 0x00010101, 0x00010000, 1341 0x9830, 0xffffffff, 0x00000000, 1342 0x9834, 0xf00fffff, 0x00000400, 1343 0x5bb0, 0x000000f0, 0x00000070, 1344 0x5bc0, 0xf0311fff, 0x80300000, 1345 0x98f8, 0x73773777, 0x12010001, 1346 0x98fc, 0xffffffff, 0x00000010, 1347 0x9b7c, 0x00ff0000, 0x00fc0000, 1348 0x8030, 0x00001f0f, 0x0000100a, 1349 0x2f48, 0x73773777, 0x12010001, 1350 0x2408, 0x000fffff, 0x000c007f, 1351 0x8a14, 0xf000003f, 0x00000007, 1352 0x8b24, 0x3fff3fff, 0x00ffcfff, 1353 0x30a04, 0x0000ff0f, 0x00000000, 1354 0x28a4c, 0x07ffffff, 0x06000000, 1355 0x4d8, 0x00000fff, 0x00000100, 1356 0x3e78, 0x00000001, 0x00000002, 1357 0xc768, 0x00000008, 0x00000008, 1358 0x8c00, 0x000000ff, 0x00000003, 1359 0x214f8, 0x01ff01ff, 0x00000002, 1360 0x21498, 0x007ff800, 0x00200000, 1361 0x2015c, 0xffffffff, 0x00000f40, 1362 0x88c4, 0x001f3ae3, 0x00000082, 1363 0x88d4, 0x0000001f, 0x00000010, 1364 0x30934, 0xffffffff, 0x00000000 1365 }; 1366 1367 static const u32 kalindi_mgcg_cgcg_init[] = 1368 { 1369 0xc420, 0xffffffff, 0xfffffffc, 1370 0x30800, 0xffffffff, 0xe0000000, 1371 0x3c2a0, 0xffffffff, 0x00000100, 1372 0x3c208, 0xffffffff, 0x00000100, 1373 0x3c2c0, 0xffffffff, 0x00000100, 1374 0x3c2c8, 0xffffffff, 0x00000100, 1375 0x3c2c4, 0xffffffff, 0x00000100, 1376 0x55e4, 0xffffffff, 0x00600100, 1377 0x3c280, 0xffffffff, 0x00000100, 1378 0x3c214, 0xffffffff, 0x06000100, 1379 0x3c220, 0xffffffff, 0x00000100, 1380 0x3c218, 0xffffffff, 0x06000100, 1381 0x3c204, 0xffffffff, 0x00000100, 1382 0x3c2e0, 0xffffffff, 0x00000100, 1383 0x3c224, 0xffffffff, 0x00000100, 1384 0x3c200, 0xffffffff, 0x00000100, 1385 0x3c230, 0xffffffff, 0x00000100, 1386 0x3c234, 0xffffffff, 0x00000100, 1387 0x3c250, 0xffffffff, 0x00000100, 1388 0x3c254, 0xffffffff, 0x00000100, 1389 0x3c258, 0xffffffff, 0x00000100, 1390 0x3c25c, 0xffffffff, 0x00000100, 1391 0x3c260, 0xffffffff, 0x00000100, 1392 0x3c27c, 0xffffffff, 0x00000100, 1393 0x3c278, 0xffffffff, 0x00000100, 1394 0x3c210, 0xffffffff, 0x06000100, 1395 0x3c290, 0xffffffff, 0x00000100, 1396 0x3c274, 0xffffffff, 0x00000100, 1397 0x3c2b4, 0xffffffff, 0x00000100, 1398 0x3c2b0, 0xffffffff, 0x00000100, 1399 0x3c270, 0xffffffff, 0x00000100, 1400 0x30800, 0xffffffff, 0xe0000000, 1401 0x3c020, 0xffffffff, 0x00010000, 1402 0x3c024, 0xffffffff, 0x00030002, 1403 0x3c028, 0xffffffff, 0x00040007, 1404 0x3c02c, 0xffffffff, 0x00060005, 1405 0x3c030, 0xffffffff, 0x00090008, 1406 0x3c034, 0xffffffff, 0x00010000, 1407 0x3c038, 0xffffffff, 0x00030002, 1408 0x3c03c, 0xffffffff, 0x00040007, 1409 0x3c040, 0xffffffff, 0x00060005, 1410 0x3c044, 0xffffffff, 0x00090008, 1411 0x3c000, 0xffffffff, 0x96e00200, 1412 0x8708, 0xffffffff, 0x00900100, 1413 0xc424, 0xffffffff, 0x0020003f, 1414 0x38, 0xffffffff, 0x0140001c, 1415 0x3c, 0x000f0000, 0x000f0000, 1416 0x220, 0xffffffff, 0xC060000C, 1417 0x224, 0xc0000fff, 0x00000100, 1418 0x20a8, 0xffffffff, 0x00000104, 1419 0x55e4, 0xff000fff, 0x00000100, 1420 0x30cc, 0xc0000fff, 0x00000104, 1421 0xc1e4, 0x00000001, 0x00000001, 1422 0xd00c, 0xff000ff0, 0x00000100, 1423 0xd80c, 0xff000ff0, 0x00000100 1424 }; 1425 1426 static const u32 hawaii_golden_spm_registers[] = 1427 { 1428 0x30800, 0xe0ffffff, 0xe0000000 1429 }; 1430 1431 static const u32 hawaii_golden_common_registers[] = 1432 { 1433 0x30800, 0xffffffff, 0xe0000000, 1434 0x28350, 0xffffffff, 0x3a00161a, 1435 0x28354, 0xffffffff, 0x0000002e, 1436 0x9a10, 0xffffffff, 0x00018208, 1437 0x98f8, 0xffffffff, 0x12011003 1438 }; 1439 1440 static const u32 hawaii_golden_registers[] = 1441 { 1442 0x3354, 0x00000333, 0x00000333, 1443 0x9a10, 0x00010000, 0x00058208, 1444 0x9830, 0xffffffff, 0x00000000, 1445 0x9834, 0xf00fffff, 0x00000400, 1446 0x9838, 0x0002021c, 0x00020200, 1447 0xc78, 0x00000080, 0x00000000, 1448 0x5bb0, 0x000000f0, 0x00000070, 1449 0x5bc0, 0xf0311fff, 0x80300000, 1450 0x350c, 0x00810000, 0x408af000, 1451 0x7030, 0x31000111, 0x00000011, 1452 0x2f48, 0x73773777, 0x12010001, 1453 0x2120, 0x0000007f, 0x0000001b, 1454 0x21dc, 0x00007fb6, 0x00002191, 1455 0x3628, 0x0000003f, 0x0000000a, 1456 0x362c, 0x0000003f, 0x0000000a, 1457 0x2ae4, 0x00073ffe, 0x000022a2, 1458 0x240c, 0x000007ff, 0x00000000, 1459 0x8bf0, 0x00002001, 0x00000001, 1460 0x8b24, 0xffffffff, 0x00ffffff, 1461 0x30a04, 0x0000ff0f, 0x00000000, 1462 0x28a4c, 0x07ffffff, 0x06000000, 1463 0x3e78, 0x00000001, 0x00000002, 1464 0xc768, 0x00000008, 0x00000008, 1465 0xc770, 0x00000f00, 0x00000800, 1466 0xc774, 0x00000f00, 0x00000800, 1467 0xc798, 0x00ffffff, 0x00ff7fbf, 1468 0xc79c, 0x00ffffff, 0x00ff7faf, 1469 0x8c00, 0x000000ff, 0x00000800, 1470 0xe40, 0x00001fff, 0x00001fff, 1471 0x9060, 0x0000007f, 0x00000020, 1472 0x9508, 0x00010000, 0x00010000, 1473 0xae00, 0x00100000, 0x000ff07c, 1474 0xac14, 0x000003ff, 0x0000000f, 1475 0xac10, 0xffffffff, 0x7564fdec, 1476 0xac0c, 0xffffffff, 0x3120b9a8, 1477 0xac08, 0x20000000, 0x0f9c0000 1478 }; 1479 1480 static const u32 hawaii_mgcg_cgcg_init[] = 1481 { 1482 0xc420, 0xffffffff, 0xfffffffd, 1483 0x30800, 0xffffffff, 0xe0000000, 1484 0x3c2a0, 0xffffffff, 0x00000100, 1485 0x3c208, 0xffffffff, 0x00000100, 1486 0x3c2c0, 0xffffffff, 0x00000100, 1487 0x3c2c8, 0xffffffff, 0x00000100, 1488 0x3c2c4, 0xffffffff, 0x00000100, 1489 0x55e4, 0xffffffff, 0x00200100, 1490 0x3c280, 0xffffffff, 0x00000100, 1491 0x3c214, 0xffffffff, 0x06000100, 1492 0x3c220, 0xffffffff, 0x00000100, 1493 0x3c218, 0xffffffff, 0x06000100, 1494 0x3c204, 0xffffffff, 0x00000100, 1495 0x3c2e0, 0xffffffff, 0x00000100, 1496 0x3c224, 0xffffffff, 0x00000100, 1497 0x3c200, 0xffffffff, 0x00000100, 1498 0x3c230, 0xffffffff, 0x00000100, 1499 0x3c234, 0xffffffff, 0x00000100, 1500 0x3c250, 0xffffffff, 0x00000100, 1501 0x3c254, 0xffffffff, 0x00000100, 1502 0x3c258, 0xffffffff, 0x00000100, 1503 0x3c25c, 0xffffffff, 0x00000100, 1504 0x3c260, 0xffffffff, 0x00000100, 1505 0x3c27c, 0xffffffff, 0x00000100, 1506 0x3c278, 0xffffffff, 0x00000100, 1507 0x3c210, 0xffffffff, 0x06000100, 1508 0x3c290, 0xffffffff, 0x00000100, 1509 0x3c274, 0xffffffff, 0x00000100, 1510 0x3c2b4, 0xffffffff, 0x00000100, 1511 0x3c2b0, 0xffffffff, 0x00000100, 1512 0x3c270, 0xffffffff, 0x00000100, 1513 0x30800, 0xffffffff, 0xe0000000, 1514 0x3c020, 0xffffffff, 0x00010000, 1515 0x3c024, 0xffffffff, 0x00030002, 1516 0x3c028, 0xffffffff, 0x00040007, 1517 0x3c02c, 0xffffffff, 0x00060005, 1518 0x3c030, 0xffffffff, 0x00090008, 1519 0x3c034, 0xffffffff, 0x00010000, 1520 0x3c038, 0xffffffff, 0x00030002, 1521 0x3c03c, 0xffffffff, 0x00040007, 1522 0x3c040, 0xffffffff, 0x00060005, 1523 0x3c044, 0xffffffff, 0x00090008, 1524 0x3c048, 0xffffffff, 0x00010000, 1525 0x3c04c, 0xffffffff, 0x00030002, 1526 0x3c050, 0xffffffff, 0x00040007, 1527 0x3c054, 0xffffffff, 0x00060005, 1528 0x3c058, 0xffffffff, 0x00090008, 1529 0x3c05c, 0xffffffff, 0x00010000, 1530 0x3c060, 0xffffffff, 0x00030002, 1531 0x3c064, 0xffffffff, 0x00040007, 1532 0x3c068, 0xffffffff, 0x00060005, 1533 0x3c06c, 0xffffffff, 0x00090008, 1534 0x3c070, 0xffffffff, 0x00010000, 1535 0x3c074, 0xffffffff, 0x00030002, 1536 0x3c078, 0xffffffff, 0x00040007, 1537 0x3c07c, 0xffffffff, 0x00060005, 1538 0x3c080, 0xffffffff, 0x00090008, 1539 0x3c084, 0xffffffff, 0x00010000, 1540 0x3c088, 0xffffffff, 0x00030002, 1541 0x3c08c, 0xffffffff, 0x00040007, 1542 0x3c090, 0xffffffff, 0x00060005, 1543 0x3c094, 0xffffffff, 0x00090008, 1544 0x3c098, 0xffffffff, 0x00010000, 1545 0x3c09c, 0xffffffff, 0x00030002, 1546 0x3c0a0, 0xffffffff, 0x00040007, 1547 0x3c0a4, 0xffffffff, 0x00060005, 1548 0x3c0a8, 0xffffffff, 0x00090008, 1549 0x3c0ac, 0xffffffff, 0x00010000, 1550 0x3c0b0, 0xffffffff, 0x00030002, 1551 0x3c0b4, 0xffffffff, 0x00040007, 1552 0x3c0b8, 0xffffffff, 0x00060005, 1553 0x3c0bc, 0xffffffff, 0x00090008, 1554 0x3c0c0, 0xffffffff, 0x00010000, 1555 0x3c0c4, 0xffffffff, 0x00030002, 1556 0x3c0c8, 0xffffffff, 0x00040007, 1557 0x3c0cc, 0xffffffff, 0x00060005, 1558 0x3c0d0, 0xffffffff, 0x00090008, 1559 0x3c0d4, 0xffffffff, 0x00010000, 1560 0x3c0d8, 0xffffffff, 0x00030002, 1561 0x3c0dc, 0xffffffff, 0x00040007, 1562 0x3c0e0, 0xffffffff, 0x00060005, 1563 0x3c0e4, 0xffffffff, 0x00090008, 1564 0x3c0e8, 0xffffffff, 0x00010000, 1565 0x3c0ec, 0xffffffff, 0x00030002, 1566 0x3c0f0, 0xffffffff, 0x00040007, 1567 0x3c0f4, 0xffffffff, 0x00060005, 1568 0x3c0f8, 0xffffffff, 0x00090008, 1569 0xc318, 0xffffffff, 0x00020200, 1570 0x3350, 0xffffffff, 0x00000200, 1571 0x15c0, 0xffffffff, 0x00000400, 1572 0x55e8, 0xffffffff, 0x00000000, 1573 0x2f50, 0xffffffff, 0x00000902, 1574 0x3c000, 0xffffffff, 0x96940200, 1575 0x8708, 0xffffffff, 0x00900100, 1576 0xc424, 0xffffffff, 0x0020003f, 1577 0x38, 0xffffffff, 0x0140001c, 1578 0x3c, 0x000f0000, 0x000f0000, 1579 0x220, 0xffffffff, 0xc060000c, 1580 0x224, 0xc0000fff, 0x00000100, 1581 0xf90, 0xffffffff, 0x00000100, 1582 0xf98, 0x00000101, 0x00000000, 1583 0x20a8, 0xffffffff, 0x00000104, 1584 0x55e4, 0xff000fff, 0x00000100, 1585 0x30cc, 0xc0000fff, 0x00000104, 1586 0xc1e4, 0x00000001, 0x00000001, 1587 0xd00c, 0xff000ff0, 0x00000100, 1588 0xd80c, 0xff000ff0, 0x00000100 1589 }; 1590 1591 static const u32 godavari_golden_registers[] = 1592 { 1593 0x55e4, 0xff607fff, 0xfc000100, 1594 0x6ed8, 0x00010101, 0x00010000, 1595 0x9830, 0xffffffff, 0x00000000, 1596 0x98302, 0xf00fffff, 0x00000400, 1597 0x6130, 0xffffffff, 0x00010000, 1598 0x5bb0, 0x000000f0, 0x00000070, 1599 0x5bc0, 0xf0311fff, 0x80300000, 1600 0x98f8, 0x73773777, 0x12010001, 1601 0x98fc, 0xffffffff, 0x00000010, 1602 0x8030, 0x00001f0f, 0x0000100a, 1603 0x2f48, 0x73773777, 0x12010001, 1604 0x2408, 0x000fffff, 0x000c007f, 1605 0x8a14, 0xf000003f, 0x00000007, 1606 0x8b24, 0xffffffff, 0x00ff0fff, 1607 0x30a04, 0x0000ff0f, 0x00000000, 1608 0x28a4c, 0x07ffffff, 0x06000000, 1609 0x4d8, 0x00000fff, 0x00000100, 1610 0xd014, 0x00010000, 0x00810001, 1611 0xd814, 0x00010000, 0x00810001, 1612 0x3e78, 0x00000001, 0x00000002, 1613 0xc768, 0x00000008, 0x00000008, 1614 0xc770, 0x00000f00, 0x00000800, 1615 0xc774, 0x00000f00, 0x00000800, 1616 0xc798, 0x00ffffff, 0x00ff7fbf, 1617 0xc79c, 0x00ffffff, 0x00ff7faf, 1618 0x8c00, 0x000000ff, 0x00000001, 1619 0x214f8, 0x01ff01ff, 0x00000002, 1620 0x21498, 0x007ff800, 0x00200000, 1621 0x2015c, 0xffffffff, 0x00000f40, 1622 0x88c4, 0x001f3ae3, 0x00000082, 1623 0x88d4, 0x0000001f, 0x00000010, 1624 0x30934, 0xffffffff, 0x00000000 1625 }; 1626 1627 1628 static void cik_init_golden_registers(struct radeon_device *rdev) 1629 { 1630 switch (rdev->family) { 1631 case CHIP_BONAIRE: 1632 radeon_program_register_sequence(rdev, 1633 bonaire_mgcg_cgcg_init, 1634 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1635 radeon_program_register_sequence(rdev, 1636 bonaire_golden_registers, 1637 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1638 radeon_program_register_sequence(rdev, 1639 bonaire_golden_common_registers, 1640 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1641 radeon_program_register_sequence(rdev, 1642 bonaire_golden_spm_registers, 1643 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1644 break; 1645 case CHIP_KABINI: 1646 radeon_program_register_sequence(rdev, 1647 kalindi_mgcg_cgcg_init, 1648 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1649 radeon_program_register_sequence(rdev, 1650 kalindi_golden_registers, 1651 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1652 radeon_program_register_sequence(rdev, 1653 kalindi_golden_common_registers, 1654 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1655 radeon_program_register_sequence(rdev, 1656 kalindi_golden_spm_registers, 1657 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1658 break; 1659 case CHIP_MULLINS: 1660 radeon_program_register_sequence(rdev, 1661 kalindi_mgcg_cgcg_init, 1662 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1663 radeon_program_register_sequence(rdev, 1664 godavari_golden_registers, 1665 (const u32)ARRAY_SIZE(godavari_golden_registers)); 1666 radeon_program_register_sequence(rdev, 1667 kalindi_golden_common_registers, 1668 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1669 radeon_program_register_sequence(rdev, 1670 kalindi_golden_spm_registers, 1671 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1672 break; 1673 case CHIP_KAVERI: 1674 radeon_program_register_sequence(rdev, 1675 spectre_mgcg_cgcg_init, 1676 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1677 radeon_program_register_sequence(rdev, 1678 spectre_golden_registers, 1679 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1680 radeon_program_register_sequence(rdev, 1681 spectre_golden_common_registers, 1682 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1683 radeon_program_register_sequence(rdev, 1684 spectre_golden_spm_registers, 1685 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1686 break; 1687 case CHIP_HAWAII: 1688 radeon_program_register_sequence(rdev, 1689 hawaii_mgcg_cgcg_init, 1690 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1691 radeon_program_register_sequence(rdev, 1692 hawaii_golden_registers, 1693 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1694 radeon_program_register_sequence(rdev, 1695 hawaii_golden_common_registers, 1696 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1697 radeon_program_register_sequence(rdev, 1698 hawaii_golden_spm_registers, 1699 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1700 break; 1701 default: 1702 break; 1703 } 1704 } 1705 1706 /** 1707 * cik_get_xclk - get the xclk 1708 * 1709 * @rdev: radeon_device pointer 1710 * 1711 * Returns the reference clock used by the gfx engine 1712 * (CIK). 1713 */ 1714 u32 cik_get_xclk(struct radeon_device *rdev) 1715 { 1716 u32 reference_clock = rdev->clock.spll.reference_freq; 1717 1718 if (rdev->flags & RADEON_IS_IGP) { 1719 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1720 return reference_clock / 2; 1721 } else { 1722 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1723 return reference_clock / 4; 1724 } 1725 return reference_clock; 1726 } 1727 1728 /** 1729 * cik_mm_rdoorbell - read a doorbell dword 1730 * 1731 * @rdev: radeon_device pointer 1732 * @index: doorbell index 1733 * 1734 * Returns the value in the doorbell aperture at the 1735 * requested doorbell index (CIK). 1736 */ 1737 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1738 { 1739 if (index < rdev->doorbell.num_doorbells) { 1740 return readl(rdev->doorbell.ptr + index); 1741 } else { 1742 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1743 return 0; 1744 } 1745 } 1746 1747 /** 1748 * cik_mm_wdoorbell - write a doorbell dword 1749 * 1750 * @rdev: radeon_device pointer 1751 * @index: doorbell index 1752 * @v: value to write 1753 * 1754 * Writes @v to the doorbell aperture at the 1755 * requested doorbell index (CIK). 1756 */ 1757 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1758 { 1759 if (index < rdev->doorbell.num_doorbells) { 1760 writel(v, rdev->doorbell.ptr + index); 1761 } else { 1762 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1763 } 1764 } 1765 1766 #define BONAIRE_IO_MC_REGS_SIZE 36 1767 1768 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1769 { 1770 {0x00000070, 0x04400000}, 1771 {0x00000071, 0x80c01803}, 1772 {0x00000072, 0x00004004}, 1773 {0x00000073, 0x00000100}, 1774 {0x00000074, 0x00ff0000}, 1775 {0x00000075, 0x34000000}, 1776 {0x00000076, 0x08000014}, 1777 {0x00000077, 0x00cc08ec}, 1778 {0x00000078, 0x00000400}, 1779 {0x00000079, 0x00000000}, 1780 {0x0000007a, 0x04090000}, 1781 {0x0000007c, 0x00000000}, 1782 {0x0000007e, 0x4408a8e8}, 1783 {0x0000007f, 0x00000304}, 1784 {0x00000080, 0x00000000}, 1785 {0x00000082, 0x00000001}, 1786 {0x00000083, 0x00000002}, 1787 {0x00000084, 0xf3e4f400}, 1788 {0x00000085, 0x052024e3}, 1789 {0x00000087, 0x00000000}, 1790 {0x00000088, 0x01000000}, 1791 {0x0000008a, 0x1c0a0000}, 1792 {0x0000008b, 0xff010000}, 1793 {0x0000008d, 0xffffefff}, 1794 {0x0000008e, 0xfff3efff}, 1795 {0x0000008f, 0xfff3efbf}, 1796 {0x00000092, 0xf7ffffff}, 1797 {0x00000093, 0xffffff7f}, 1798 {0x00000095, 0x00101101}, 1799 {0x00000096, 0x00000fff}, 1800 {0x00000097, 0x00116fff}, 1801 {0x00000098, 0x60010000}, 1802 {0x00000099, 0x10010000}, 1803 {0x0000009a, 0x00006000}, 1804 {0x0000009b, 0x00001000}, 1805 {0x0000009f, 0x00b48000} 1806 }; 1807 1808 #define HAWAII_IO_MC_REGS_SIZE 22 1809 1810 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1811 { 1812 {0x0000007d, 0x40000000}, 1813 {0x0000007e, 0x40180304}, 1814 {0x0000007f, 0x0000ff00}, 1815 {0x00000081, 0x00000000}, 1816 {0x00000083, 0x00000800}, 1817 {0x00000086, 0x00000000}, 1818 {0x00000087, 0x00000100}, 1819 {0x00000088, 0x00020100}, 1820 {0x00000089, 0x00000000}, 1821 {0x0000008b, 0x00040000}, 1822 {0x0000008c, 0x00000100}, 1823 {0x0000008e, 0xff010000}, 1824 {0x00000090, 0xffffefff}, 1825 {0x00000091, 0xfff3efff}, 1826 {0x00000092, 0xfff3efbf}, 1827 {0x00000093, 0xf7ffffff}, 1828 {0x00000094, 0xffffff7f}, 1829 {0x00000095, 0x00000fff}, 1830 {0x00000096, 0x00116fff}, 1831 {0x00000097, 0x60010000}, 1832 {0x00000098, 0x10010000}, 1833 {0x0000009f, 0x00c79000} 1834 }; 1835 1836 1837 /** 1838 * cik_srbm_select - select specific register instances 1839 * 1840 * @rdev: radeon_device pointer 1841 * @me: selected ME (micro engine) 1842 * @pipe: pipe 1843 * @queue: queue 1844 * @vmid: VMID 1845 * 1846 * Switches the currently active registers instances. Some 1847 * registers are instanced per VMID, others are instanced per 1848 * me/pipe/queue combination. 1849 */ 1850 static void cik_srbm_select(struct radeon_device *rdev, 1851 u32 me, u32 pipe, u32 queue, u32 vmid) 1852 { 1853 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1854 MEID(me & 0x3) | 1855 VMID(vmid & 0xf) | 1856 QUEUEID(queue & 0x7)); 1857 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1858 } 1859 1860 /* ucode loading */ 1861 /** 1862 * ci_mc_load_microcode - load MC ucode into the hw 1863 * 1864 * @rdev: radeon_device pointer 1865 * 1866 * Load the GDDR MC ucode into the hw (CIK). 1867 * Returns 0 on success, error on failure. 1868 */ 1869 int ci_mc_load_microcode(struct radeon_device *rdev) 1870 { 1871 const __be32 *fw_data = NULL; 1872 const __le32 *new_fw_data = NULL; 1873 u32 running, tmp; 1874 u32 *io_mc_regs = NULL; 1875 const __le32 *new_io_mc_regs = NULL; 1876 int i, regs_size, ucode_size; 1877 1878 if (!rdev->mc_fw) 1879 return -EINVAL; 1880 1881 if (rdev->new_fw) { 1882 const struct mc_firmware_header_v1_0 *hdr = 1883 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1884 1885 radeon_ucode_print_mc_hdr(&hdr->header); 1886 1887 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1888 new_io_mc_regs = (const __le32 *) 1889 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1890 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1891 new_fw_data = (const __le32 *) 1892 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1893 } else { 1894 ucode_size = rdev->mc_fw->size / 4; 1895 1896 switch (rdev->family) { 1897 case CHIP_BONAIRE: 1898 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1899 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1900 break; 1901 case CHIP_HAWAII: 1902 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1903 regs_size = HAWAII_IO_MC_REGS_SIZE; 1904 break; 1905 default: 1906 return -EINVAL; 1907 } 1908 fw_data = (const __be32 *)rdev->mc_fw->data; 1909 } 1910 1911 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1912 1913 if (running == 0) { 1914 /* reset the engine and set to writable */ 1915 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1916 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1917 1918 /* load mc io regs */ 1919 for (i = 0; i < regs_size; i++) { 1920 if (rdev->new_fw) { 1921 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1922 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1923 } else { 1924 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1925 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1926 } 1927 } 1928 1929 tmp = RREG32(MC_SEQ_MISC0); 1930 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { 1931 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5); 1932 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023); 1933 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9); 1934 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0); 1935 } 1936 1937 /* load the MC ucode */ 1938 for (i = 0; i < ucode_size; i++) { 1939 if (rdev->new_fw) 1940 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1941 else 1942 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1943 } 1944 1945 /* put the engine back into the active state */ 1946 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1947 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1948 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1949 1950 /* wait for training to complete */ 1951 for (i = 0; i < rdev->usec_timeout; i++) { 1952 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1953 break; 1954 udelay(1); 1955 } 1956 for (i = 0; i < rdev->usec_timeout; i++) { 1957 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1958 break; 1959 udelay(1); 1960 } 1961 } 1962 1963 return 0; 1964 } 1965 1966 /** 1967 * cik_init_microcode - load ucode images from disk 1968 * 1969 * @rdev: radeon_device pointer 1970 * 1971 * Use the firmware interface to load the ucode images into 1972 * the driver (not loaded into hw). 1973 * Returns 0 on success, error on failure. 1974 */ 1975 static int cik_init_microcode(struct radeon_device *rdev) 1976 { 1977 const char *chip_name; 1978 const char *new_chip_name; 1979 size_t pfp_req_size, me_req_size, ce_req_size, 1980 mec_req_size, rlc_req_size, mc_req_size = 0, 1981 sdma_req_size, smc_req_size = 0, mc2_req_size = 0; 1982 char fw_name[30]; 1983 int new_fw = 0; 1984 int err; 1985 int num_fw; 1986 bool new_smc = false; 1987 1988 DRM_DEBUG("\n"); 1989 1990 switch (rdev->family) { 1991 case CHIP_BONAIRE: 1992 chip_name = "BONAIRE"; 1993 if ((rdev->pdev->revision == 0x80) || 1994 (rdev->pdev->revision == 0x81) || 1995 (rdev->pdev->device == 0x665f)) 1996 new_smc = true; 1997 new_chip_name = "bonaire"; 1998 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1999 me_req_size = CIK_ME_UCODE_SIZE * 4; 2000 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2001 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2002 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 2003 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; 2004 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; 2005 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2006 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 2007 num_fw = 8; 2008 break; 2009 case CHIP_HAWAII: 2010 chip_name = "HAWAII"; 2011 if (rdev->pdev->revision == 0x80) 2012 new_smc = true; 2013 new_chip_name = "hawaii"; 2014 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2015 me_req_size = CIK_ME_UCODE_SIZE * 4; 2016 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2017 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2018 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 2019 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 2020 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; 2021 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2022 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 2023 num_fw = 8; 2024 break; 2025 case CHIP_KAVERI: 2026 chip_name = "KAVERI"; 2027 new_chip_name = "kaveri"; 2028 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2029 me_req_size = CIK_ME_UCODE_SIZE * 4; 2030 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2031 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2032 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 2033 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2034 num_fw = 7; 2035 break; 2036 case CHIP_KABINI: 2037 chip_name = "KABINI"; 2038 new_chip_name = "kabini"; 2039 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2040 me_req_size = CIK_ME_UCODE_SIZE * 4; 2041 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2042 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2043 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 2044 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2045 num_fw = 6; 2046 break; 2047 case CHIP_MULLINS: 2048 chip_name = "MULLINS"; 2049 new_chip_name = "mullins"; 2050 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 2051 me_req_size = CIK_ME_UCODE_SIZE * 4; 2052 ce_req_size = CIK_CE_UCODE_SIZE * 4; 2053 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 2054 rlc_req_size = ML_RLC_UCODE_SIZE * 4; 2055 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 2056 num_fw = 6; 2057 break; 2058 default: BUG(); 2059 } 2060 2061 DRM_INFO("Loading %s Microcode\n", new_chip_name); 2062 2063 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); 2064 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2065 if (err) { 2066 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 2067 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2068 if (err) 2069 goto out; 2070 if (rdev->pfp_fw->size != pfp_req_size) { 2071 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n", 2072 rdev->pfp_fw->size, fw_name); 2073 err = -EINVAL; 2074 goto out; 2075 } 2076 } else { 2077 err = radeon_ucode_validate(rdev->pfp_fw); 2078 if (err) { 2079 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2080 fw_name); 2081 goto out; 2082 } else { 2083 new_fw++; 2084 } 2085 } 2086 2087 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); 2088 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2089 if (err) { 2090 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 2091 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2092 if (err) 2093 goto out; 2094 if (rdev->me_fw->size != me_req_size) { 2095 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n", 2096 rdev->me_fw->size, fw_name); 2097 err = -EINVAL; 2098 } 2099 } else { 2100 err = radeon_ucode_validate(rdev->me_fw); 2101 if (err) { 2102 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2103 fw_name); 2104 goto out; 2105 } else { 2106 new_fw++; 2107 } 2108 } 2109 2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); 2111 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2112 if (err) { 2113 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 2114 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2115 if (err) 2116 goto out; 2117 if (rdev->ce_fw->size != ce_req_size) { 2118 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n", 2119 rdev->ce_fw->size, fw_name); 2120 err = -EINVAL; 2121 } 2122 } else { 2123 err = radeon_ucode_validate(rdev->ce_fw); 2124 if (err) { 2125 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2126 fw_name); 2127 goto out; 2128 } else { 2129 new_fw++; 2130 } 2131 } 2132 2133 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name); 2134 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2135 if (err) { 2136 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name); 2137 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2138 if (err) 2139 goto out; 2140 if (rdev->mec_fw->size != mec_req_size) { 2141 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n", 2142 rdev->mec_fw->size, fw_name); 2143 err = -EINVAL; 2144 } 2145 } else { 2146 err = radeon_ucode_validate(rdev->mec_fw); 2147 if (err) { 2148 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2149 fw_name); 2150 goto out; 2151 } else { 2152 new_fw++; 2153 } 2154 } 2155 2156 if (rdev->family == CHIP_KAVERI) { 2157 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name); 2158 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); 2159 if (err) { 2160 goto out; 2161 } else { 2162 err = radeon_ucode_validate(rdev->mec2_fw); 2163 if (err) { 2164 goto out; 2165 } else { 2166 new_fw++; 2167 } 2168 } 2169 } 2170 2171 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); 2172 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2173 if (err) { 2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); 2175 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2176 if (err) 2177 goto out; 2178 if (rdev->rlc_fw->size != rlc_req_size) { 2179 pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n", 2180 rdev->rlc_fw->size, fw_name); 2181 err = -EINVAL; 2182 } 2183 } else { 2184 err = radeon_ucode_validate(rdev->rlc_fw); 2185 if (err) { 2186 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2187 fw_name); 2188 goto out; 2189 } else { 2190 new_fw++; 2191 } 2192 } 2193 2194 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name); 2195 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2196 if (err) { 2197 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name); 2198 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2199 if (err) 2200 goto out; 2201 if (rdev->sdma_fw->size != sdma_req_size) { 2202 pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n", 2203 rdev->sdma_fw->size, fw_name); 2204 err = -EINVAL; 2205 } 2206 } else { 2207 err = radeon_ucode_validate(rdev->sdma_fw); 2208 if (err) { 2209 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2210 fw_name); 2211 goto out; 2212 } else { 2213 new_fw++; 2214 } 2215 } 2216 2217 /* No SMC, MC ucode on APUs */ 2218 if (!(rdev->flags & RADEON_IS_IGP)) { 2219 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); 2220 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2221 if (err) { 2222 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); 2223 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2224 if (err) { 2225 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 2226 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2227 if (err) 2228 goto out; 2229 } 2230 if ((rdev->mc_fw->size != mc_req_size) && 2231 (rdev->mc_fw->size != mc2_req_size)){ 2232 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n", 2233 rdev->mc_fw->size, fw_name); 2234 err = -EINVAL; 2235 } 2236 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); 2237 } else { 2238 err = radeon_ucode_validate(rdev->mc_fw); 2239 if (err) { 2240 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2241 fw_name); 2242 goto out; 2243 } else { 2244 new_fw++; 2245 } 2246 } 2247 2248 if (new_smc) 2249 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name); 2250 else 2251 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); 2252 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2253 if (err) { 2254 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 2255 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2256 if (err) { 2257 pr_err("smc: error loading firmware \"%s\"\n", 2258 fw_name); 2259 release_firmware(rdev->smc_fw); 2260 rdev->smc_fw = NULL; 2261 err = 0; 2262 } else if (rdev->smc_fw->size != smc_req_size) { 2263 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n", 2264 rdev->smc_fw->size, fw_name); 2265 err = -EINVAL; 2266 } 2267 } else { 2268 err = radeon_ucode_validate(rdev->smc_fw); 2269 if (err) { 2270 pr_err("cik_fw: validation failed for firmware \"%s\"\n", 2271 fw_name); 2272 goto out; 2273 } else { 2274 new_fw++; 2275 } 2276 } 2277 } 2278 2279 if (new_fw == 0) { 2280 rdev->new_fw = false; 2281 } else if (new_fw < num_fw) { 2282 pr_err("ci_fw: mixing new and old firmware!\n"); 2283 err = -EINVAL; 2284 } else { 2285 rdev->new_fw = true; 2286 } 2287 2288 out: 2289 if (err) { 2290 if (err != -EINVAL) 2291 pr_err("cik_cp: Failed to load firmware \"%s\"\n", 2292 fw_name); 2293 release_firmware(rdev->pfp_fw); 2294 rdev->pfp_fw = NULL; 2295 release_firmware(rdev->me_fw); 2296 rdev->me_fw = NULL; 2297 release_firmware(rdev->ce_fw); 2298 rdev->ce_fw = NULL; 2299 release_firmware(rdev->mec_fw); 2300 rdev->mec_fw = NULL; 2301 release_firmware(rdev->mec2_fw); 2302 rdev->mec2_fw = NULL; 2303 release_firmware(rdev->rlc_fw); 2304 rdev->rlc_fw = NULL; 2305 release_firmware(rdev->sdma_fw); 2306 rdev->sdma_fw = NULL; 2307 release_firmware(rdev->mc_fw); 2308 rdev->mc_fw = NULL; 2309 release_firmware(rdev->smc_fw); 2310 rdev->smc_fw = NULL; 2311 } 2312 return err; 2313 } 2314 2315 /* 2316 * Core functions 2317 */ 2318 /** 2319 * cik_tiling_mode_table_init - init the hw tiling table 2320 * 2321 * @rdev: radeon_device pointer 2322 * 2323 * Starting with SI, the tiling setup is done globally in a 2324 * set of 32 tiling modes. Rather than selecting each set of 2325 * parameters per surface as on older asics, we just select 2326 * which index in the tiling table we want to use, and the 2327 * surface uses those parameters (CIK). 2328 */ 2329 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 2330 { 2331 u32 *tile = rdev->config.cik.tile_mode_array; 2332 u32 *macrotile = rdev->config.cik.macrotile_mode_array; 2333 const u32 num_tile_mode_states = 2334 ARRAY_SIZE(rdev->config.cik.tile_mode_array); 2335 const u32 num_secondary_tile_mode_states = 2336 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array); 2337 u32 reg_offset, split_equal_to_row_size; 2338 u32 num_pipe_configs; 2339 u32 num_rbs = rdev->config.cik.max_backends_per_se * 2340 rdev->config.cik.max_shader_engines; 2341 2342 switch (rdev->config.cik.mem_row_size_in_kb) { 2343 case 1: 2344 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2345 break; 2346 case 2: 2347 default: 2348 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2349 break; 2350 case 4: 2351 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2352 break; 2353 } 2354 2355 num_pipe_configs = rdev->config.cik.max_tile_pipes; 2356 if (num_pipe_configs > 8) 2357 num_pipe_configs = 16; 2358 2359 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2360 tile[reg_offset] = 0; 2361 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2362 macrotile[reg_offset] = 0; 2363 2364 switch(num_pipe_configs) { 2365 case 16: 2366 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2367 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2369 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2370 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2374 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2375 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2376 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2378 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2382 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2384 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2385 TILE_SPLIT(split_equal_to_row_size)); 2386 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2387 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2388 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2389 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2390 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2392 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2393 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2394 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2396 TILE_SPLIT(split_equal_to_row_size)); 2397 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2398 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2399 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2401 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2402 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2403 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2404 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2405 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2406 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2407 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2408 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2409 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2410 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2411 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2414 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2416 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2417 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2418 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2419 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2420 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2421 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2422 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2423 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2424 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2425 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2426 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2427 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2428 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2429 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2432 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2433 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2434 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2435 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2436 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2437 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2438 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2440 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2441 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2442 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2443 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2444 2445 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2448 NUM_BANKS(ADDR_SURF_16_BANK)); 2449 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2452 NUM_BANKS(ADDR_SURF_16_BANK)); 2453 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2456 NUM_BANKS(ADDR_SURF_16_BANK)); 2457 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2460 NUM_BANKS(ADDR_SURF_16_BANK)); 2461 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2464 NUM_BANKS(ADDR_SURF_8_BANK)); 2465 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2468 NUM_BANKS(ADDR_SURF_4_BANK)); 2469 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2472 NUM_BANKS(ADDR_SURF_2_BANK)); 2473 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2476 NUM_BANKS(ADDR_SURF_16_BANK)); 2477 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2480 NUM_BANKS(ADDR_SURF_16_BANK)); 2481 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2484 NUM_BANKS(ADDR_SURF_16_BANK)); 2485 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2488 NUM_BANKS(ADDR_SURF_8_BANK)); 2489 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2492 NUM_BANKS(ADDR_SURF_4_BANK)); 2493 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2496 NUM_BANKS(ADDR_SURF_2_BANK)); 2497 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2500 NUM_BANKS(ADDR_SURF_2_BANK)); 2501 2502 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2503 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 2504 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2505 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 2506 break; 2507 2508 case 8: 2509 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2513 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2514 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2515 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2517 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2518 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2519 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2520 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2521 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2522 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2523 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2524 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2525 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2528 TILE_SPLIT(split_equal_to_row_size)); 2529 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2531 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2532 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2533 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2536 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2537 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2539 TILE_SPLIT(split_equal_to_row_size)); 2540 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2542 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2545 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2548 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2549 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2553 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2554 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2555 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2556 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2557 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2559 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2560 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2561 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2563 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2564 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2565 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2566 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2567 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2568 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2569 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2571 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2572 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2574 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2575 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2576 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2579 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2580 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2582 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2583 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2584 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2585 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2586 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2587 2588 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2591 NUM_BANKS(ADDR_SURF_16_BANK)); 2592 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2593 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2594 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2595 NUM_BANKS(ADDR_SURF_16_BANK)); 2596 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2597 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2598 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2599 NUM_BANKS(ADDR_SURF_16_BANK)); 2600 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2603 NUM_BANKS(ADDR_SURF_16_BANK)); 2604 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2605 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2606 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2607 NUM_BANKS(ADDR_SURF_8_BANK)); 2608 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2611 NUM_BANKS(ADDR_SURF_4_BANK)); 2612 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2613 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2614 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2615 NUM_BANKS(ADDR_SURF_2_BANK)); 2616 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2617 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2618 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2619 NUM_BANKS(ADDR_SURF_16_BANK)); 2620 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2623 NUM_BANKS(ADDR_SURF_16_BANK)); 2624 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2627 NUM_BANKS(ADDR_SURF_16_BANK)); 2628 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2631 NUM_BANKS(ADDR_SURF_16_BANK)); 2632 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2633 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2634 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2635 NUM_BANKS(ADDR_SURF_8_BANK)); 2636 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2639 NUM_BANKS(ADDR_SURF_4_BANK)); 2640 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2641 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2642 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2643 NUM_BANKS(ADDR_SURF_2_BANK)); 2644 2645 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2646 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 2647 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2648 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 2649 break; 2650 2651 case 4: 2652 if (num_rbs == 4) { 2653 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2654 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2655 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2657 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2658 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2659 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2660 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2661 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2662 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2663 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2665 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2666 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2667 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2668 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2669 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2670 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2671 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2672 TILE_SPLIT(split_equal_to_row_size)); 2673 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2674 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2675 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2676 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2677 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2678 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2679 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2680 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2681 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2682 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2683 TILE_SPLIT(split_equal_to_row_size)); 2684 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2685 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2686 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2687 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2688 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2689 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2690 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2691 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2692 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2693 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2694 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2695 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2696 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2697 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2698 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2699 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2700 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2701 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2702 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2703 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2704 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2705 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2706 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2707 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2708 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2709 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2710 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2711 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2712 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2713 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2714 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2715 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2716 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2717 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2718 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2719 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2720 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2721 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2723 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2724 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2725 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2727 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2728 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2729 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2730 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2731 2732 } else if (num_rbs < 4) { 2733 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2734 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2735 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2736 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2737 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2738 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2739 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2740 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2741 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2742 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2744 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2745 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2746 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2747 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2748 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2749 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2751 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2752 TILE_SPLIT(split_equal_to_row_size)); 2753 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2754 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2755 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2756 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2757 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2759 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2760 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2761 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2762 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2763 TILE_SPLIT(split_equal_to_row_size)); 2764 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2765 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2766 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2769 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2771 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2772 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2773 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2775 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2776 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2777 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2778 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2779 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2780 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2781 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2782 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2783 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2784 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2785 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2786 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2787 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2788 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2789 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2790 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2791 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2792 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2793 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2795 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2796 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2798 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2799 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2800 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2801 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2802 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2803 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2804 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2805 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2806 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2807 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2808 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2809 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2810 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2811 } 2812 2813 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2814 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2815 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2816 NUM_BANKS(ADDR_SURF_16_BANK)); 2817 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2818 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2819 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2820 NUM_BANKS(ADDR_SURF_16_BANK)); 2821 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2822 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2823 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2824 NUM_BANKS(ADDR_SURF_16_BANK)); 2825 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2826 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2827 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2828 NUM_BANKS(ADDR_SURF_16_BANK)); 2829 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2830 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2831 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2832 NUM_BANKS(ADDR_SURF_16_BANK)); 2833 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2836 NUM_BANKS(ADDR_SURF_8_BANK)); 2837 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2838 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2839 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2840 NUM_BANKS(ADDR_SURF_4_BANK)); 2841 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2842 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2843 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2844 NUM_BANKS(ADDR_SURF_16_BANK)); 2845 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2846 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2847 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2848 NUM_BANKS(ADDR_SURF_16_BANK)); 2849 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2850 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2851 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2852 NUM_BANKS(ADDR_SURF_16_BANK)); 2853 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2854 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2855 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2856 NUM_BANKS(ADDR_SURF_16_BANK)); 2857 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2858 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2859 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2860 NUM_BANKS(ADDR_SURF_16_BANK)); 2861 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2862 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2863 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2864 NUM_BANKS(ADDR_SURF_8_BANK)); 2865 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2866 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2867 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2868 NUM_BANKS(ADDR_SURF_4_BANK)); 2869 2870 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2871 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 2872 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2873 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 2874 break; 2875 2876 case 2: 2877 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2878 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2879 PIPE_CONFIG(ADDR_SURF_P2) | 2880 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2881 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2882 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2883 PIPE_CONFIG(ADDR_SURF_P2) | 2884 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2885 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2887 PIPE_CONFIG(ADDR_SURF_P2) | 2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2889 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2890 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2891 PIPE_CONFIG(ADDR_SURF_P2) | 2892 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2893 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2895 PIPE_CONFIG(ADDR_SURF_P2) | 2896 TILE_SPLIT(split_equal_to_row_size)); 2897 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2898 PIPE_CONFIG(ADDR_SURF_P2) | 2899 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2900 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2901 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2902 PIPE_CONFIG(ADDR_SURF_P2) | 2903 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2904 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2905 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2906 PIPE_CONFIG(ADDR_SURF_P2) | 2907 TILE_SPLIT(split_equal_to_row_size)); 2908 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2909 PIPE_CONFIG(ADDR_SURF_P2); 2910 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2912 PIPE_CONFIG(ADDR_SURF_P2)); 2913 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2915 PIPE_CONFIG(ADDR_SURF_P2) | 2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2917 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2919 PIPE_CONFIG(ADDR_SURF_P2) | 2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2921 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2922 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2923 PIPE_CONFIG(ADDR_SURF_P2) | 2924 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2925 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2926 PIPE_CONFIG(ADDR_SURF_P2) | 2927 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2928 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2929 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2930 PIPE_CONFIG(ADDR_SURF_P2) | 2931 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2932 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2933 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2934 PIPE_CONFIG(ADDR_SURF_P2) | 2935 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2936 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2937 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2938 PIPE_CONFIG(ADDR_SURF_P2) | 2939 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2940 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2941 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2942 PIPE_CONFIG(ADDR_SURF_P2)); 2943 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2944 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2945 PIPE_CONFIG(ADDR_SURF_P2) | 2946 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2947 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2948 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2949 PIPE_CONFIG(ADDR_SURF_P2) | 2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2951 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2952 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2953 PIPE_CONFIG(ADDR_SURF_P2) | 2954 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2955 2956 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2959 NUM_BANKS(ADDR_SURF_16_BANK)); 2960 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2963 NUM_BANKS(ADDR_SURF_16_BANK)); 2964 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2967 NUM_BANKS(ADDR_SURF_16_BANK)); 2968 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2971 NUM_BANKS(ADDR_SURF_16_BANK)); 2972 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2975 NUM_BANKS(ADDR_SURF_16_BANK)); 2976 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2977 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2978 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2979 NUM_BANKS(ADDR_SURF_16_BANK)); 2980 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2983 NUM_BANKS(ADDR_SURF_8_BANK)); 2984 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2987 NUM_BANKS(ADDR_SURF_16_BANK)); 2988 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2991 NUM_BANKS(ADDR_SURF_16_BANK)); 2992 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2993 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2994 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2995 NUM_BANKS(ADDR_SURF_16_BANK)); 2996 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2999 NUM_BANKS(ADDR_SURF_16_BANK)); 3000 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3003 NUM_BANKS(ADDR_SURF_16_BANK)); 3004 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3007 NUM_BANKS(ADDR_SURF_16_BANK)); 3008 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3011 NUM_BANKS(ADDR_SURF_8_BANK)); 3012 3013 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3014 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]); 3015 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3016 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]); 3017 break; 3018 3019 default: 3020 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3021 } 3022 } 3023 3024 /** 3025 * cik_select_se_sh - select which SE, SH to address 3026 * 3027 * @rdev: radeon_device pointer 3028 * @se_num: shader engine to address 3029 * @sh_num: sh block to address 3030 * 3031 * Select which SE, SH combinations to address. Certain 3032 * registers are instanced per SE or SH. 0xffffffff means 3033 * broadcast to all SEs or SHs (CIK). 3034 */ 3035 static void cik_select_se_sh(struct radeon_device *rdev, 3036 u32 se_num, u32 sh_num) 3037 { 3038 u32 data = INSTANCE_BROADCAST_WRITES; 3039 3040 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3041 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3042 else if (se_num == 0xffffffff) 3043 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3044 else if (sh_num == 0xffffffff) 3045 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3046 else 3047 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3048 WREG32(GRBM_GFX_INDEX, data); 3049 } 3050 3051 /** 3052 * cik_create_bitmask - create a bitmask 3053 * 3054 * @bit_width: length of the mask 3055 * 3056 * create a variable length bit mask (CIK). 3057 * Returns the bitmask. 3058 */ 3059 static u32 cik_create_bitmask(u32 bit_width) 3060 { 3061 u32 i, mask = 0; 3062 3063 for (i = 0; i < bit_width; i++) { 3064 mask <<= 1; 3065 mask |= 1; 3066 } 3067 return mask; 3068 } 3069 3070 /** 3071 * cik_get_rb_disabled - computes the mask of disabled RBs 3072 * 3073 * @rdev: radeon_device pointer 3074 * @max_rb_num: max RBs (render backends) for the asic 3075 * @se_num: number of SEs (shader engines) for the asic 3076 * @sh_per_se: number of SH blocks per SE for the asic 3077 * 3078 * Calculates the bitmask of disabled RBs (CIK). 3079 * Returns the disabled RB bitmask. 3080 */ 3081 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3082 u32 max_rb_num_per_se, 3083 u32 sh_per_se) 3084 { 3085 u32 data, mask; 3086 3087 data = RREG32(CC_RB_BACKEND_DISABLE); 3088 if (data & 1) 3089 data &= BACKEND_DISABLE_MASK; 3090 else 3091 data = 0; 3092 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3093 3094 data >>= BACKEND_DISABLE_SHIFT; 3095 3096 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3097 3098 return data & mask; 3099 } 3100 3101 /** 3102 * cik_setup_rb - setup the RBs on the asic 3103 * 3104 * @rdev: radeon_device pointer 3105 * @se_num: number of SEs (shader engines) for the asic 3106 * @sh_per_se: number of SH blocks per SE for the asic 3107 * @max_rb_num: max RBs (render backends) for the asic 3108 * 3109 * Configures per-SE/SH RB registers (CIK). 3110 */ 3111 static void cik_setup_rb(struct radeon_device *rdev, 3112 u32 se_num, u32 sh_per_se, 3113 u32 max_rb_num_per_se) 3114 { 3115 int i, j; 3116 u32 data, mask; 3117 u32 disabled_rbs = 0; 3118 u32 enabled_rbs = 0; 3119 3120 for (i = 0; i < se_num; i++) { 3121 for (j = 0; j < sh_per_se; j++) { 3122 cik_select_se_sh(rdev, i, j); 3123 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3124 if (rdev->family == CHIP_HAWAII) 3125 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3126 else 3127 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3128 } 3129 } 3130 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3131 3132 mask = 1; 3133 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3134 if (!(disabled_rbs & mask)) 3135 enabled_rbs |= mask; 3136 mask <<= 1; 3137 } 3138 3139 rdev->config.cik.backend_enable_mask = enabled_rbs; 3140 3141 for (i = 0; i < se_num; i++) { 3142 cik_select_se_sh(rdev, i, 0xffffffff); 3143 data = 0; 3144 for (j = 0; j < sh_per_se; j++) { 3145 switch (enabled_rbs & 3) { 3146 case 0: 3147 if (j == 0) 3148 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3149 else 3150 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3151 break; 3152 case 1: 3153 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3154 break; 3155 case 2: 3156 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3157 break; 3158 case 3: 3159 default: 3160 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3161 break; 3162 } 3163 enabled_rbs >>= 2; 3164 } 3165 WREG32(PA_SC_RASTER_CONFIG, data); 3166 } 3167 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3168 } 3169 3170 /** 3171 * cik_gpu_init - setup the 3D engine 3172 * 3173 * @rdev: radeon_device pointer 3174 * 3175 * Configures the 3D engine and tiling configuration 3176 * registers so that the 3D engine is usable. 3177 */ 3178 static void cik_gpu_init(struct radeon_device *rdev) 3179 { 3180 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3181 u32 mc_shared_chmap, mc_arb_ramcfg; 3182 u32 hdp_host_path_cntl; 3183 u32 tmp; 3184 int i, j; 3185 3186 switch (rdev->family) { 3187 case CHIP_BONAIRE: 3188 rdev->config.cik.max_shader_engines = 2; 3189 rdev->config.cik.max_tile_pipes = 4; 3190 rdev->config.cik.max_cu_per_sh = 7; 3191 rdev->config.cik.max_sh_per_se = 1; 3192 rdev->config.cik.max_backends_per_se = 2; 3193 rdev->config.cik.max_texture_channel_caches = 4; 3194 rdev->config.cik.max_gprs = 256; 3195 rdev->config.cik.max_gs_threads = 32; 3196 rdev->config.cik.max_hw_contexts = 8; 3197 3198 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3199 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3200 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3201 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3202 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3203 break; 3204 case CHIP_HAWAII: 3205 rdev->config.cik.max_shader_engines = 4; 3206 rdev->config.cik.max_tile_pipes = 16; 3207 rdev->config.cik.max_cu_per_sh = 11; 3208 rdev->config.cik.max_sh_per_se = 1; 3209 rdev->config.cik.max_backends_per_se = 4; 3210 rdev->config.cik.max_texture_channel_caches = 16; 3211 rdev->config.cik.max_gprs = 256; 3212 rdev->config.cik.max_gs_threads = 32; 3213 rdev->config.cik.max_hw_contexts = 8; 3214 3215 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3216 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3217 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3218 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3219 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3220 break; 3221 case CHIP_KAVERI: 3222 rdev->config.cik.max_shader_engines = 1; 3223 rdev->config.cik.max_tile_pipes = 4; 3224 if ((rdev->pdev->device == 0x1304) || 3225 (rdev->pdev->device == 0x1305) || 3226 (rdev->pdev->device == 0x130C) || 3227 (rdev->pdev->device == 0x130F) || 3228 (rdev->pdev->device == 0x1310) || 3229 (rdev->pdev->device == 0x1311) || 3230 (rdev->pdev->device == 0x131C)) { 3231 rdev->config.cik.max_cu_per_sh = 8; 3232 rdev->config.cik.max_backends_per_se = 2; 3233 } else if ((rdev->pdev->device == 0x1309) || 3234 (rdev->pdev->device == 0x130A) || 3235 (rdev->pdev->device == 0x130D) || 3236 (rdev->pdev->device == 0x1313) || 3237 (rdev->pdev->device == 0x131D)) { 3238 rdev->config.cik.max_cu_per_sh = 6; 3239 rdev->config.cik.max_backends_per_se = 2; 3240 } else if ((rdev->pdev->device == 0x1306) || 3241 (rdev->pdev->device == 0x1307) || 3242 (rdev->pdev->device == 0x130B) || 3243 (rdev->pdev->device == 0x130E) || 3244 (rdev->pdev->device == 0x1315) || 3245 (rdev->pdev->device == 0x1318) || 3246 (rdev->pdev->device == 0x131B)) { 3247 rdev->config.cik.max_cu_per_sh = 4; 3248 rdev->config.cik.max_backends_per_se = 1; 3249 } else { 3250 rdev->config.cik.max_cu_per_sh = 3; 3251 rdev->config.cik.max_backends_per_se = 1; 3252 } 3253 rdev->config.cik.max_sh_per_se = 1; 3254 rdev->config.cik.max_texture_channel_caches = 4; 3255 rdev->config.cik.max_gprs = 256; 3256 rdev->config.cik.max_gs_threads = 16; 3257 rdev->config.cik.max_hw_contexts = 8; 3258 3259 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3260 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3261 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3262 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3263 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3264 break; 3265 case CHIP_KABINI: 3266 case CHIP_MULLINS: 3267 default: 3268 rdev->config.cik.max_shader_engines = 1; 3269 rdev->config.cik.max_tile_pipes = 2; 3270 rdev->config.cik.max_cu_per_sh = 2; 3271 rdev->config.cik.max_sh_per_se = 1; 3272 rdev->config.cik.max_backends_per_se = 1; 3273 rdev->config.cik.max_texture_channel_caches = 2; 3274 rdev->config.cik.max_gprs = 256; 3275 rdev->config.cik.max_gs_threads = 16; 3276 rdev->config.cik.max_hw_contexts = 8; 3277 3278 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3279 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3280 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3281 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3282 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3283 break; 3284 } 3285 3286 /* Initialize HDP */ 3287 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3288 WREG32((0x2c14 + j), 0x00000000); 3289 WREG32((0x2c18 + j), 0x00000000); 3290 WREG32((0x2c1c + j), 0x00000000); 3291 WREG32((0x2c20 + j), 0x00000000); 3292 WREG32((0x2c24 + j), 0x00000000); 3293 } 3294 3295 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3296 WREG32(SRBM_INT_CNTL, 0x1); 3297 WREG32(SRBM_INT_ACK, 0x1); 3298 3299 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3300 3301 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3302 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3303 3304 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3305 rdev->config.cik.mem_max_burst_length_bytes = 256; 3306 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3307 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3308 if (rdev->config.cik.mem_row_size_in_kb > 4) 3309 rdev->config.cik.mem_row_size_in_kb = 4; 3310 /* XXX use MC settings? */ 3311 rdev->config.cik.shader_engine_tile_size = 32; 3312 rdev->config.cik.num_gpus = 1; 3313 rdev->config.cik.multi_gpu_tile_size = 64; 3314 3315 /* fix up row size */ 3316 gb_addr_config &= ~ROW_SIZE_MASK; 3317 switch (rdev->config.cik.mem_row_size_in_kb) { 3318 case 1: 3319 default: 3320 gb_addr_config |= ROW_SIZE(0); 3321 break; 3322 case 2: 3323 gb_addr_config |= ROW_SIZE(1); 3324 break; 3325 case 4: 3326 gb_addr_config |= ROW_SIZE(2); 3327 break; 3328 } 3329 3330 /* setup tiling info dword. gb_addr_config is not adequate since it does 3331 * not have bank info, so create a custom tiling dword. 3332 * bits 3:0 num_pipes 3333 * bits 7:4 num_banks 3334 * bits 11:8 group_size 3335 * bits 15:12 row_size 3336 */ 3337 rdev->config.cik.tile_config = 0; 3338 switch (rdev->config.cik.num_tile_pipes) { 3339 case 1: 3340 rdev->config.cik.tile_config |= (0 << 0); 3341 break; 3342 case 2: 3343 rdev->config.cik.tile_config |= (1 << 0); 3344 break; 3345 case 4: 3346 rdev->config.cik.tile_config |= (2 << 0); 3347 break; 3348 case 8: 3349 default: 3350 /* XXX what about 12? */ 3351 rdev->config.cik.tile_config |= (3 << 0); 3352 break; 3353 } 3354 rdev->config.cik.tile_config |= 3355 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3356 rdev->config.cik.tile_config |= 3357 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3358 rdev->config.cik.tile_config |= 3359 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3360 3361 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3362 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3363 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3364 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3365 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3366 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3367 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3368 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3369 3370 cik_tiling_mode_table_init(rdev); 3371 3372 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3373 rdev->config.cik.max_sh_per_se, 3374 rdev->config.cik.max_backends_per_se); 3375 3376 rdev->config.cik.active_cus = 0; 3377 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 3378 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 3379 rdev->config.cik.active_cus += 3380 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); 3381 } 3382 } 3383 3384 /* set HW defaults for 3D engine */ 3385 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3386 3387 WREG32(SX_DEBUG_1, 0x20); 3388 3389 WREG32(TA_CNTL_AUX, 0x00010000); 3390 3391 tmp = RREG32(SPI_CONFIG_CNTL); 3392 tmp |= 0x03000000; 3393 WREG32(SPI_CONFIG_CNTL, tmp); 3394 3395 WREG32(SQ_CONFIG, 1); 3396 3397 WREG32(DB_DEBUG, 0); 3398 3399 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3400 tmp |= 0x00000400; 3401 WREG32(DB_DEBUG2, tmp); 3402 3403 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3404 tmp |= 0x00020200; 3405 WREG32(DB_DEBUG3, tmp); 3406 3407 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3408 tmp |= 0x00018208; 3409 WREG32(CB_HW_CONTROL, tmp); 3410 3411 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3412 3413 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3414 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3415 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3416 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3417 3418 WREG32(VGT_NUM_INSTANCES, 1); 3419 3420 WREG32(CP_PERFMON_CNTL, 0); 3421 3422 WREG32(SQ_CONFIG, 0); 3423 3424 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3425 FORCE_EOV_MAX_REZ_CNT(255))); 3426 3427 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3428 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3429 3430 WREG32(VGT_GS_VERTEX_REUSE, 16); 3431 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3432 3433 tmp = RREG32(HDP_MISC_CNTL); 3434 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3435 WREG32(HDP_MISC_CNTL, tmp); 3436 3437 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3438 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3439 3440 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3441 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3442 3443 udelay(50); 3444 } 3445 3446 /* 3447 * GPU scratch registers helpers function. 3448 */ 3449 /** 3450 * cik_scratch_init - setup driver info for CP scratch regs 3451 * 3452 * @rdev: radeon_device pointer 3453 * 3454 * Set up the number and offset of the CP scratch registers. 3455 * NOTE: use of CP scratch registers is a legacy inferface and 3456 * is not used by default on newer asics (r6xx+). On newer asics, 3457 * memory buffers are used for fences rather than scratch regs. 3458 */ 3459 static void cik_scratch_init(struct radeon_device *rdev) 3460 { 3461 int i; 3462 3463 rdev->scratch.num_reg = 7; 3464 rdev->scratch.reg_base = SCRATCH_REG0; 3465 for (i = 0; i < rdev->scratch.num_reg; i++) { 3466 rdev->scratch.free[i] = true; 3467 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3468 } 3469 } 3470 3471 /** 3472 * cik_ring_test - basic gfx ring test 3473 * 3474 * @rdev: radeon_device pointer 3475 * @ring: radeon_ring structure holding ring information 3476 * 3477 * Allocate a scratch register and write to it using the gfx ring (CIK). 3478 * Provides a basic gfx ring test to verify that the ring is working. 3479 * Used by cik_cp_gfx_resume(); 3480 * Returns 0 on success, error on failure. 3481 */ 3482 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3483 { 3484 uint32_t scratch; 3485 uint32_t tmp = 0; 3486 unsigned i; 3487 int r; 3488 3489 r = radeon_scratch_get(rdev, &scratch); 3490 if (r) { 3491 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3492 return r; 3493 } 3494 WREG32(scratch, 0xCAFEDEAD); 3495 r = radeon_ring_lock(rdev, ring, 3); 3496 if (r) { 3497 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3498 radeon_scratch_free(rdev, scratch); 3499 return r; 3500 } 3501 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3502 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3503 radeon_ring_write(ring, 0xDEADBEEF); 3504 radeon_ring_unlock_commit(rdev, ring, false); 3505 3506 for (i = 0; i < rdev->usec_timeout; i++) { 3507 tmp = RREG32(scratch); 3508 if (tmp == 0xDEADBEEF) 3509 break; 3510 DRM_UDELAY(1); 3511 } 3512 if (i < rdev->usec_timeout) { 3513 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3514 } else { 3515 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3516 ring->idx, scratch, tmp); 3517 r = -EINVAL; 3518 } 3519 radeon_scratch_free(rdev, scratch); 3520 return r; 3521 } 3522 3523 /** 3524 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3525 * 3526 * @rdev: radeon_device pointer 3527 * @ridx: radeon ring index 3528 * 3529 * Emits an hdp flush on the cp. 3530 */ 3531 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3532 int ridx) 3533 { 3534 struct radeon_ring *ring = &rdev->ring[ridx]; 3535 u32 ref_and_mask; 3536 3537 switch (ring->idx) { 3538 case CAYMAN_RING_TYPE_CP1_INDEX: 3539 case CAYMAN_RING_TYPE_CP2_INDEX: 3540 default: 3541 switch (ring->me) { 3542 case 0: 3543 ref_and_mask = CP2 << ring->pipe; 3544 break; 3545 case 1: 3546 ref_and_mask = CP6 << ring->pipe; 3547 break; 3548 default: 3549 return; 3550 } 3551 break; 3552 case RADEON_RING_TYPE_GFX_INDEX: 3553 ref_and_mask = CP0; 3554 break; 3555 } 3556 3557 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3558 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3559 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3560 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3561 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3562 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3563 radeon_ring_write(ring, ref_and_mask); 3564 radeon_ring_write(ring, ref_and_mask); 3565 radeon_ring_write(ring, 0x20); /* poll interval */ 3566 } 3567 3568 /** 3569 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3570 * 3571 * @rdev: radeon_device pointer 3572 * @fence: radeon fence object 3573 * 3574 * Emits a fence sequnce number on the gfx ring and flushes 3575 * GPU caches. 3576 */ 3577 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3578 struct radeon_fence *fence) 3579 { 3580 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3581 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3582 3583 /* Workaround for cache flush problems. First send a dummy EOP 3584 * event down the pipe with seq one below. 3585 */ 3586 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3587 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3588 EOP_TC_ACTION_EN | 3589 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3590 EVENT_INDEX(5))); 3591 radeon_ring_write(ring, addr & 0xfffffffc); 3592 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 3593 DATA_SEL(1) | INT_SEL(0)); 3594 radeon_ring_write(ring, fence->seq - 1); 3595 radeon_ring_write(ring, 0); 3596 3597 /* Then send the real EOP event down the pipe. */ 3598 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3599 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3600 EOP_TC_ACTION_EN | 3601 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3602 EVENT_INDEX(5))); 3603 radeon_ring_write(ring, addr & 0xfffffffc); 3604 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3605 radeon_ring_write(ring, fence->seq); 3606 radeon_ring_write(ring, 0); 3607 } 3608 3609 /** 3610 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3611 * 3612 * @rdev: radeon_device pointer 3613 * @fence: radeon fence object 3614 * 3615 * Emits a fence sequnce number on the compute ring and flushes 3616 * GPU caches. 3617 */ 3618 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3619 struct radeon_fence *fence) 3620 { 3621 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3622 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3623 3624 /* RELEASE_MEM - flush caches, send int */ 3625 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3626 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3627 EOP_TC_ACTION_EN | 3628 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3629 EVENT_INDEX(5))); 3630 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3631 radeon_ring_write(ring, addr & 0xfffffffc); 3632 radeon_ring_write(ring, upper_32_bits(addr)); 3633 radeon_ring_write(ring, fence->seq); 3634 radeon_ring_write(ring, 0); 3635 } 3636 3637 /** 3638 * cik_semaphore_ring_emit - emit a semaphore on the CP ring 3639 * 3640 * @rdev: radeon_device pointer 3641 * @ring: radeon ring buffer object 3642 * @semaphore: radeon semaphore object 3643 * @emit_wait: Is this a sempahore wait? 3644 * 3645 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP 3646 * from running ahead of semaphore waits. 3647 */ 3648 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3649 struct radeon_ring *ring, 3650 struct radeon_semaphore *semaphore, 3651 bool emit_wait) 3652 { 3653 uint64_t addr = semaphore->gpu_addr; 3654 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3655 3656 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3657 radeon_ring_write(ring, lower_32_bits(addr)); 3658 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3659 3660 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { 3661 /* Prevent the PFP from running ahead of the semaphore wait */ 3662 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3663 radeon_ring_write(ring, 0x0); 3664 } 3665 3666 return true; 3667 } 3668 3669 /** 3670 * cik_copy_cpdma - copy pages using the CP DMA engine 3671 * 3672 * @rdev: radeon_device pointer 3673 * @src_offset: src GPU address 3674 * @dst_offset: dst GPU address 3675 * @num_gpu_pages: number of GPU pages to xfer 3676 * @resv: reservation object to sync to 3677 * 3678 * Copy GPU paging using the CP DMA engine (CIK+). 3679 * Used by the radeon ttm implementation to move pages if 3680 * registered as the asic copy callback. 3681 */ 3682 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, 3683 uint64_t src_offset, uint64_t dst_offset, 3684 unsigned num_gpu_pages, 3685 struct reservation_object *resv) 3686 { 3687 struct radeon_fence *fence; 3688 struct radeon_sync sync; 3689 int ring_index = rdev->asic->copy.blit_ring_index; 3690 struct radeon_ring *ring = &rdev->ring[ring_index]; 3691 u32 size_in_bytes, cur_size_in_bytes, control; 3692 int i, num_loops; 3693 int r = 0; 3694 3695 radeon_sync_create(&sync); 3696 3697 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 3698 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 3699 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 3700 if (r) { 3701 DRM_ERROR("radeon: moving bo (%d).\n", r); 3702 radeon_sync_free(rdev, &sync, NULL); 3703 return ERR_PTR(r); 3704 } 3705 3706 radeon_sync_resv(rdev, &sync, resv, false); 3707 radeon_sync_rings(rdev, &sync, ring->idx); 3708 3709 for (i = 0; i < num_loops; i++) { 3710 cur_size_in_bytes = size_in_bytes; 3711 if (cur_size_in_bytes > 0x1fffff) 3712 cur_size_in_bytes = 0x1fffff; 3713 size_in_bytes -= cur_size_in_bytes; 3714 control = 0; 3715 if (size_in_bytes == 0) 3716 control |= PACKET3_DMA_DATA_CP_SYNC; 3717 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 3718 radeon_ring_write(ring, control); 3719 radeon_ring_write(ring, lower_32_bits(src_offset)); 3720 radeon_ring_write(ring, upper_32_bits(src_offset)); 3721 radeon_ring_write(ring, lower_32_bits(dst_offset)); 3722 radeon_ring_write(ring, upper_32_bits(dst_offset)); 3723 radeon_ring_write(ring, cur_size_in_bytes); 3724 src_offset += cur_size_in_bytes; 3725 dst_offset += cur_size_in_bytes; 3726 } 3727 3728 r = radeon_fence_emit(rdev, &fence, ring->idx); 3729 if (r) { 3730 radeon_ring_unlock_undo(rdev, ring); 3731 radeon_sync_free(rdev, &sync, NULL); 3732 return ERR_PTR(r); 3733 } 3734 3735 radeon_ring_unlock_commit(rdev, ring, false); 3736 radeon_sync_free(rdev, &sync, fence); 3737 3738 return fence; 3739 } 3740 3741 /* 3742 * IB stuff 3743 */ 3744 /** 3745 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 3746 * 3747 * @rdev: radeon_device pointer 3748 * @ib: radeon indirect buffer object 3749 * 3750 * Emits a DE (drawing engine) or CE (constant engine) IB 3751 * on the gfx ring. IBs are usually generated by userspace 3752 * acceleration drivers and submitted to the kernel for 3753 * scheduling on the ring. This function schedules the IB 3754 * on the gfx ring for execution by the GPU. 3755 */ 3756 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3757 { 3758 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3759 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; 3760 u32 header, control = INDIRECT_BUFFER_VALID; 3761 3762 if (ib->is_const_ib) { 3763 /* set switch buffer packet before const IB */ 3764 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3765 radeon_ring_write(ring, 0); 3766 3767 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3768 } else { 3769 u32 next_rptr; 3770 if (ring->rptr_save_reg) { 3771 next_rptr = ring->wptr + 3 + 4; 3772 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3773 radeon_ring_write(ring, ((ring->rptr_save_reg - 3774 PACKET3_SET_UCONFIG_REG_START) >> 2)); 3775 radeon_ring_write(ring, next_rptr); 3776 } else if (rdev->wb.enabled) { 3777 next_rptr = ring->wptr + 5 + 4; 3778 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3779 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 3780 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3781 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 3782 radeon_ring_write(ring, next_rptr); 3783 } 3784 3785 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3786 } 3787 3788 control |= ib->length_dw | (vm_id << 24); 3789 3790 radeon_ring_write(ring, header); 3791 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC)); 3792 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3793 radeon_ring_write(ring, control); 3794 } 3795 3796 /** 3797 * cik_ib_test - basic gfx ring IB test 3798 * 3799 * @rdev: radeon_device pointer 3800 * @ring: radeon_ring structure holding ring information 3801 * 3802 * Allocate an IB and execute it on the gfx ring (CIK). 3803 * Provides a basic gfx ring test to verify that IBs are working. 3804 * Returns 0 on success, error on failure. 3805 */ 3806 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 3807 { 3808 struct radeon_ib ib; 3809 uint32_t scratch; 3810 uint32_t tmp = 0; 3811 unsigned i; 3812 int r; 3813 3814 r = radeon_scratch_get(rdev, &scratch); 3815 if (r) { 3816 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 3817 return r; 3818 } 3819 WREG32(scratch, 0xCAFEDEAD); 3820 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 3821 if (r) { 3822 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 3823 radeon_scratch_free(rdev, scratch); 3824 return r; 3825 } 3826 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 3827 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 3828 ib.ptr[2] = 0xDEADBEEF; 3829 ib.length_dw = 3; 3830 r = radeon_ib_schedule(rdev, &ib, NULL, false); 3831 if (r) { 3832 radeon_scratch_free(rdev, scratch); 3833 radeon_ib_free(rdev, &ib); 3834 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 3835 return r; 3836 } 3837 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies( 3838 RADEON_USEC_IB_TEST_TIMEOUT)); 3839 if (r < 0) { 3840 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 3841 radeon_scratch_free(rdev, scratch); 3842 radeon_ib_free(rdev, &ib); 3843 return r; 3844 } else if (r == 0) { 3845 DRM_ERROR("radeon: fence wait timed out.\n"); 3846 radeon_scratch_free(rdev, scratch); 3847 radeon_ib_free(rdev, &ib); 3848 return -ETIMEDOUT; 3849 } 3850 r = 0; 3851 for (i = 0; i < rdev->usec_timeout; i++) { 3852 tmp = RREG32(scratch); 3853 if (tmp == 0xDEADBEEF) 3854 break; 3855 DRM_UDELAY(1); 3856 } 3857 if (i < rdev->usec_timeout) { 3858 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 3859 } else { 3860 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 3861 scratch, tmp); 3862 r = -EINVAL; 3863 } 3864 radeon_scratch_free(rdev, scratch); 3865 radeon_ib_free(rdev, &ib); 3866 return r; 3867 } 3868 3869 /* 3870 * CP. 3871 * On CIK, gfx and compute now have independant command processors. 3872 * 3873 * GFX 3874 * Gfx consists of a single ring and can process both gfx jobs and 3875 * compute jobs. The gfx CP consists of three microengines (ME): 3876 * PFP - Pre-Fetch Parser 3877 * ME - Micro Engine 3878 * CE - Constant Engine 3879 * The PFP and ME make up what is considered the Drawing Engine (DE). 3880 * The CE is an asynchronous engine used for updating buffer desciptors 3881 * used by the DE so that they can be loaded into cache in parallel 3882 * while the DE is processing state update packets. 3883 * 3884 * Compute 3885 * The compute CP consists of two microengines (ME): 3886 * MEC1 - Compute MicroEngine 1 3887 * MEC2 - Compute MicroEngine 2 3888 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 3889 * The queues are exposed to userspace and are programmed directly 3890 * by the compute runtime. 3891 */ 3892 /** 3893 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 3894 * 3895 * @rdev: radeon_device pointer 3896 * @enable: enable or disable the MEs 3897 * 3898 * Halts or unhalts the gfx MEs. 3899 */ 3900 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 3901 { 3902 if (enable) 3903 WREG32(CP_ME_CNTL, 0); 3904 else { 3905 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3906 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3907 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3908 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3909 } 3910 udelay(50); 3911 } 3912 3913 /** 3914 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 3915 * 3916 * @rdev: radeon_device pointer 3917 * 3918 * Loads the gfx PFP, ME, and CE ucode. 3919 * Returns 0 for success, -EINVAL if the ucode is not available. 3920 */ 3921 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 3922 { 3923 int i; 3924 3925 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 3926 return -EINVAL; 3927 3928 cik_cp_gfx_enable(rdev, false); 3929 3930 if (rdev->new_fw) { 3931 const struct gfx_firmware_header_v1_0 *pfp_hdr = 3932 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 3933 const struct gfx_firmware_header_v1_0 *ce_hdr = 3934 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 3935 const struct gfx_firmware_header_v1_0 *me_hdr = 3936 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 3937 const __le32 *fw_data; 3938 u32 fw_size; 3939 3940 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 3941 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 3942 radeon_ucode_print_gfx_hdr(&me_hdr->header); 3943 3944 /* PFP */ 3945 fw_data = (const __le32 *) 3946 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3947 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3948 WREG32(CP_PFP_UCODE_ADDR, 0); 3949 for (i = 0; i < fw_size; i++) 3950 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3951 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); 3952 3953 /* CE */ 3954 fw_data = (const __le32 *) 3955 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3956 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3957 WREG32(CP_CE_UCODE_ADDR, 0); 3958 for (i = 0; i < fw_size; i++) 3959 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3960 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); 3961 3962 /* ME */ 3963 fw_data = (const __be32 *) 3964 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3965 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3966 WREG32(CP_ME_RAM_WADDR, 0); 3967 for (i = 0; i < fw_size; i++) 3968 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3969 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); 3970 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); 3971 } else { 3972 const __be32 *fw_data; 3973 3974 /* PFP */ 3975 fw_data = (const __be32 *)rdev->pfp_fw->data; 3976 WREG32(CP_PFP_UCODE_ADDR, 0); 3977 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 3978 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3979 WREG32(CP_PFP_UCODE_ADDR, 0); 3980 3981 /* CE */ 3982 fw_data = (const __be32 *)rdev->ce_fw->data; 3983 WREG32(CP_CE_UCODE_ADDR, 0); 3984 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 3985 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 3986 WREG32(CP_CE_UCODE_ADDR, 0); 3987 3988 /* ME */ 3989 fw_data = (const __be32 *)rdev->me_fw->data; 3990 WREG32(CP_ME_RAM_WADDR, 0); 3991 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 3992 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3993 WREG32(CP_ME_RAM_WADDR, 0); 3994 } 3995 3996 return 0; 3997 } 3998 3999 /** 4000 * cik_cp_gfx_start - start the gfx ring 4001 * 4002 * @rdev: radeon_device pointer 4003 * 4004 * Enables the ring and loads the clear state context and other 4005 * packets required to init the ring. 4006 * Returns 0 for success, error for failure. 4007 */ 4008 static int cik_cp_gfx_start(struct radeon_device *rdev) 4009 { 4010 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4011 int r, i; 4012 4013 /* init the CP */ 4014 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 4015 WREG32(CP_ENDIAN_SWAP, 0); 4016 WREG32(CP_DEVICE_ID, 1); 4017 4018 cik_cp_gfx_enable(rdev, true); 4019 4020 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 4021 if (r) { 4022 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 4023 return r; 4024 } 4025 4026 /* init the CE partitions. CE only used for gfx on CIK */ 4027 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4028 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4029 radeon_ring_write(ring, 0x8000); 4030 radeon_ring_write(ring, 0x8000); 4031 4032 /* setup clear context state */ 4033 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4034 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4035 4036 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4037 radeon_ring_write(ring, 0x80000000); 4038 radeon_ring_write(ring, 0x80000000); 4039 4040 for (i = 0; i < cik_default_size; i++) 4041 radeon_ring_write(ring, cik_default_state[i]); 4042 4043 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4044 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4045 4046 /* set clear context state */ 4047 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4048 radeon_ring_write(ring, 0); 4049 4050 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4051 radeon_ring_write(ring, 0x00000316); 4052 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 4053 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 4054 4055 radeon_ring_unlock_commit(rdev, ring, false); 4056 4057 return 0; 4058 } 4059 4060 /** 4061 * cik_cp_gfx_fini - stop the gfx ring 4062 * 4063 * @rdev: radeon_device pointer 4064 * 4065 * Stop the gfx ring and tear down the driver ring 4066 * info. 4067 */ 4068 static void cik_cp_gfx_fini(struct radeon_device *rdev) 4069 { 4070 cik_cp_gfx_enable(rdev, false); 4071 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4072 } 4073 4074 /** 4075 * cik_cp_gfx_resume - setup the gfx ring buffer registers 4076 * 4077 * @rdev: radeon_device pointer 4078 * 4079 * Program the location and size of the gfx ring buffer 4080 * and test it to make sure it's working. 4081 * Returns 0 for success, error for failure. 4082 */ 4083 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4084 { 4085 struct radeon_ring *ring; 4086 u32 tmp; 4087 u32 rb_bufsz; 4088 u64 rb_addr; 4089 int r; 4090 4091 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4092 if (rdev->family != CHIP_HAWAII) 4093 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4094 4095 /* Set the write pointer delay */ 4096 WREG32(CP_RB_WPTR_DELAY, 0); 4097 4098 /* set the RB to use vmid 0 */ 4099 WREG32(CP_RB_VMID, 0); 4100 4101 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4102 4103 /* ring 0 - compute and gfx */ 4104 /* Set ring buffer size */ 4105 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4106 rb_bufsz = order_base_2(ring->ring_size / 8); 4107 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4108 #ifdef __BIG_ENDIAN 4109 tmp |= BUF_SWAP_32BIT; 4110 #endif 4111 WREG32(CP_RB0_CNTL, tmp); 4112 4113 /* Initialize the ring buffer's read and write pointers */ 4114 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4115 ring->wptr = 0; 4116 WREG32(CP_RB0_WPTR, ring->wptr); 4117 4118 /* set the wb address wether it's enabled or not */ 4119 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4120 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4121 4122 /* scratch register shadowing is no longer supported */ 4123 WREG32(SCRATCH_UMSK, 0); 4124 4125 if (!rdev->wb.enabled) 4126 tmp |= RB_NO_UPDATE; 4127 4128 mdelay(1); 4129 WREG32(CP_RB0_CNTL, tmp); 4130 4131 rb_addr = ring->gpu_addr >> 8; 4132 WREG32(CP_RB0_BASE, rb_addr); 4133 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4134 4135 /* start the ring */ 4136 cik_cp_gfx_start(rdev); 4137 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4138 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4139 if (r) { 4140 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4141 return r; 4142 } 4143 4144 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4145 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4146 4147 return 0; 4148 } 4149 4150 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4151 struct radeon_ring *ring) 4152 { 4153 u32 rptr; 4154 4155 if (rdev->wb.enabled) 4156 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4157 else 4158 rptr = RREG32(CP_RB0_RPTR); 4159 4160 return rptr; 4161 } 4162 4163 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4164 struct radeon_ring *ring) 4165 { 4166 return RREG32(CP_RB0_WPTR); 4167 } 4168 4169 void cik_gfx_set_wptr(struct radeon_device *rdev, 4170 struct radeon_ring *ring) 4171 { 4172 WREG32(CP_RB0_WPTR, ring->wptr); 4173 (void)RREG32(CP_RB0_WPTR); 4174 } 4175 4176 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4177 struct radeon_ring *ring) 4178 { 4179 u32 rptr; 4180 4181 if (rdev->wb.enabled) { 4182 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4183 } else { 4184 mutex_lock(&rdev->srbm_mutex); 4185 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4186 rptr = RREG32(CP_HQD_PQ_RPTR); 4187 cik_srbm_select(rdev, 0, 0, 0, 0); 4188 mutex_unlock(&rdev->srbm_mutex); 4189 } 4190 4191 return rptr; 4192 } 4193 4194 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4195 struct radeon_ring *ring) 4196 { 4197 u32 wptr; 4198 4199 if (rdev->wb.enabled) { 4200 /* XXX check if swapping is necessary on BE */ 4201 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4202 } else { 4203 mutex_lock(&rdev->srbm_mutex); 4204 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4205 wptr = RREG32(CP_HQD_PQ_WPTR); 4206 cik_srbm_select(rdev, 0, 0, 0, 0); 4207 mutex_unlock(&rdev->srbm_mutex); 4208 } 4209 4210 return wptr; 4211 } 4212 4213 void cik_compute_set_wptr(struct radeon_device *rdev, 4214 struct radeon_ring *ring) 4215 { 4216 /* XXX check if swapping is necessary on BE */ 4217 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4218 WDOORBELL32(ring->doorbell_index, ring->wptr); 4219 } 4220 4221 static void cik_compute_stop(struct radeon_device *rdev, 4222 struct radeon_ring *ring) 4223 { 4224 u32 j, tmp; 4225 4226 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4227 /* Disable wptr polling. */ 4228 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4229 tmp &= ~WPTR_POLL_EN; 4230 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4231 /* Disable HQD. */ 4232 if (RREG32(CP_HQD_ACTIVE) & 1) { 4233 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4234 for (j = 0; j < rdev->usec_timeout; j++) { 4235 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4236 break; 4237 udelay(1); 4238 } 4239 WREG32(CP_HQD_DEQUEUE_REQUEST, 0); 4240 WREG32(CP_HQD_PQ_RPTR, 0); 4241 WREG32(CP_HQD_PQ_WPTR, 0); 4242 } 4243 cik_srbm_select(rdev, 0, 0, 0, 0); 4244 } 4245 4246 /** 4247 * cik_cp_compute_enable - enable/disable the compute CP MEs 4248 * 4249 * @rdev: radeon_device pointer 4250 * @enable: enable or disable the MEs 4251 * 4252 * Halts or unhalts the compute MEs. 4253 */ 4254 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4255 { 4256 if (enable) 4257 WREG32(CP_MEC_CNTL, 0); 4258 else { 4259 /* 4260 * To make hibernation reliable we need to clear compute ring 4261 * configuration before halting the compute ring. 4262 */ 4263 mutex_lock(&rdev->srbm_mutex); 4264 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 4265 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 4266 mutex_unlock(&rdev->srbm_mutex); 4267 4268 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4269 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4270 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4271 } 4272 udelay(50); 4273 } 4274 4275 /** 4276 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4277 * 4278 * @rdev: radeon_device pointer 4279 * 4280 * Loads the compute MEC1&2 ucode. 4281 * Returns 0 for success, -EINVAL if the ucode is not available. 4282 */ 4283 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4284 { 4285 int i; 4286 4287 if (!rdev->mec_fw) 4288 return -EINVAL; 4289 4290 cik_cp_compute_enable(rdev, false); 4291 4292 if (rdev->new_fw) { 4293 const struct gfx_firmware_header_v1_0 *mec_hdr = 4294 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 4295 const __le32 *fw_data; 4296 u32 fw_size; 4297 4298 radeon_ucode_print_gfx_hdr(&mec_hdr->header); 4299 4300 /* MEC1 */ 4301 fw_data = (const __le32 *) 4302 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 4303 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 4304 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4305 for (i = 0; i < fw_size; i++) 4306 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); 4307 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); 4308 4309 /* MEC2 */ 4310 if (rdev->family == CHIP_KAVERI) { 4311 const struct gfx_firmware_header_v1_0 *mec2_hdr = 4312 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 4313 4314 fw_data = (const __le32 *) 4315 (rdev->mec2_fw->data + 4316 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); 4317 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; 4318 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4319 for (i = 0; i < fw_size; i++) 4320 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); 4321 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); 4322 } 4323 } else { 4324 const __be32 *fw_data; 4325 4326 /* MEC1 */ 4327 fw_data = (const __be32 *)rdev->mec_fw->data; 4328 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4329 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4330 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4331 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4332 4333 if (rdev->family == CHIP_KAVERI) { 4334 /* MEC2 */ 4335 fw_data = (const __be32 *)rdev->mec_fw->data; 4336 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4337 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4338 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4339 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4340 } 4341 } 4342 4343 return 0; 4344 } 4345 4346 /** 4347 * cik_cp_compute_start - start the compute queues 4348 * 4349 * @rdev: radeon_device pointer 4350 * 4351 * Enable the compute queues. 4352 * Returns 0 for success, error for failure. 4353 */ 4354 static int cik_cp_compute_start(struct radeon_device *rdev) 4355 { 4356 cik_cp_compute_enable(rdev, true); 4357 4358 return 0; 4359 } 4360 4361 /** 4362 * cik_cp_compute_fini - stop the compute queues 4363 * 4364 * @rdev: radeon_device pointer 4365 * 4366 * Stop the compute queues and tear down the driver queue 4367 * info. 4368 */ 4369 static void cik_cp_compute_fini(struct radeon_device *rdev) 4370 { 4371 int i, idx, r; 4372 4373 cik_cp_compute_enable(rdev, false); 4374 4375 for (i = 0; i < 2; i++) { 4376 if (i == 0) 4377 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4378 else 4379 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4380 4381 if (rdev->ring[idx].mqd_obj) { 4382 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4383 if (unlikely(r != 0)) 4384 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4385 4386 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4387 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4388 4389 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4390 rdev->ring[idx].mqd_obj = NULL; 4391 } 4392 } 4393 } 4394 4395 static void cik_mec_fini(struct radeon_device *rdev) 4396 { 4397 int r; 4398 4399 if (rdev->mec.hpd_eop_obj) { 4400 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4401 if (unlikely(r != 0)) 4402 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4403 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4404 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4405 4406 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4407 rdev->mec.hpd_eop_obj = NULL; 4408 } 4409 } 4410 4411 #define MEC_HPD_SIZE 2048 4412 4413 static int cik_mec_init(struct radeon_device *rdev) 4414 { 4415 int r; 4416 u32 *hpd; 4417 4418 /* 4419 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4420 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4421 */ 4422 if (rdev->family == CHIP_KAVERI) 4423 rdev->mec.num_mec = 2; 4424 else 4425 rdev->mec.num_mec = 1; 4426 rdev->mec.num_pipe = 4; 4427 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4428 4429 if (rdev->mec.hpd_eop_obj == NULL) { 4430 r = radeon_bo_create(rdev, 4431 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4432 PAGE_SIZE, true, 4433 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, 4434 &rdev->mec.hpd_eop_obj); 4435 if (r) { 4436 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4437 return r; 4438 } 4439 } 4440 4441 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4442 if (unlikely(r != 0)) { 4443 cik_mec_fini(rdev); 4444 return r; 4445 } 4446 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4447 &rdev->mec.hpd_eop_gpu_addr); 4448 if (r) { 4449 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4450 cik_mec_fini(rdev); 4451 return r; 4452 } 4453 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4454 if (r) { 4455 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4456 cik_mec_fini(rdev); 4457 return r; 4458 } 4459 4460 /* clear memory. Not sure if this is required or not */ 4461 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4462 4463 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4464 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4465 4466 return 0; 4467 } 4468 4469 struct hqd_registers 4470 { 4471 u32 cp_mqd_base_addr; 4472 u32 cp_mqd_base_addr_hi; 4473 u32 cp_hqd_active; 4474 u32 cp_hqd_vmid; 4475 u32 cp_hqd_persistent_state; 4476 u32 cp_hqd_pipe_priority; 4477 u32 cp_hqd_queue_priority; 4478 u32 cp_hqd_quantum; 4479 u32 cp_hqd_pq_base; 4480 u32 cp_hqd_pq_base_hi; 4481 u32 cp_hqd_pq_rptr; 4482 u32 cp_hqd_pq_rptr_report_addr; 4483 u32 cp_hqd_pq_rptr_report_addr_hi; 4484 u32 cp_hqd_pq_wptr_poll_addr; 4485 u32 cp_hqd_pq_wptr_poll_addr_hi; 4486 u32 cp_hqd_pq_doorbell_control; 4487 u32 cp_hqd_pq_wptr; 4488 u32 cp_hqd_pq_control; 4489 u32 cp_hqd_ib_base_addr; 4490 u32 cp_hqd_ib_base_addr_hi; 4491 u32 cp_hqd_ib_rptr; 4492 u32 cp_hqd_ib_control; 4493 u32 cp_hqd_iq_timer; 4494 u32 cp_hqd_iq_rptr; 4495 u32 cp_hqd_dequeue_request; 4496 u32 cp_hqd_dma_offload; 4497 u32 cp_hqd_sema_cmd; 4498 u32 cp_hqd_msg_type; 4499 u32 cp_hqd_atomic0_preop_lo; 4500 u32 cp_hqd_atomic0_preop_hi; 4501 u32 cp_hqd_atomic1_preop_lo; 4502 u32 cp_hqd_atomic1_preop_hi; 4503 u32 cp_hqd_hq_scheduler0; 4504 u32 cp_hqd_hq_scheduler1; 4505 u32 cp_mqd_control; 4506 }; 4507 4508 struct bonaire_mqd 4509 { 4510 u32 header; 4511 u32 dispatch_initiator; 4512 u32 dimensions[3]; 4513 u32 start_idx[3]; 4514 u32 num_threads[3]; 4515 u32 pipeline_stat_enable; 4516 u32 perf_counter_enable; 4517 u32 pgm[2]; 4518 u32 tba[2]; 4519 u32 tma[2]; 4520 u32 pgm_rsrc[2]; 4521 u32 vmid; 4522 u32 resource_limits; 4523 u32 static_thread_mgmt01[2]; 4524 u32 tmp_ring_size; 4525 u32 static_thread_mgmt23[2]; 4526 u32 restart[3]; 4527 u32 thread_trace_enable; 4528 u32 reserved1; 4529 u32 user_data[16]; 4530 u32 vgtcs_invoke_count[2]; 4531 struct hqd_registers queue_state; 4532 u32 dequeue_cntr; 4533 u32 interrupt_queue[64]; 4534 }; 4535 4536 /** 4537 * cik_cp_compute_resume - setup the compute queue registers 4538 * 4539 * @rdev: radeon_device pointer 4540 * 4541 * Program the compute queues and test them to make sure they 4542 * are working. 4543 * Returns 0 for success, error for failure. 4544 */ 4545 static int cik_cp_compute_resume(struct radeon_device *rdev) 4546 { 4547 int r, i, j, idx; 4548 u32 tmp; 4549 bool use_doorbell = true; 4550 u64 hqd_gpu_addr; 4551 u64 mqd_gpu_addr; 4552 u64 eop_gpu_addr; 4553 u64 wb_gpu_addr; 4554 u32 *buf; 4555 struct bonaire_mqd *mqd; 4556 4557 r = cik_cp_compute_start(rdev); 4558 if (r) 4559 return r; 4560 4561 /* fix up chicken bits */ 4562 tmp = RREG32(CP_CPF_DEBUG); 4563 tmp |= (1 << 23); 4564 WREG32(CP_CPF_DEBUG, tmp); 4565 4566 /* init the pipes */ 4567 mutex_lock(&rdev->srbm_mutex); 4568 4569 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) { 4570 int me = (i < 4) ? 1 : 2; 4571 int pipe = (i < 4) ? i : (i - 4); 4572 4573 cik_srbm_select(rdev, me, pipe, 0, 0); 4574 4575 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ; 4576 /* write the EOP addr */ 4577 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4578 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4579 4580 /* set the VMID assigned */ 4581 WREG32(CP_HPD_EOP_VMID, 0); 4582 4583 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4584 tmp = RREG32(CP_HPD_EOP_CONTROL); 4585 tmp &= ~EOP_SIZE_MASK; 4586 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4587 WREG32(CP_HPD_EOP_CONTROL, tmp); 4588 4589 } 4590 cik_srbm_select(rdev, 0, 0, 0, 0); 4591 mutex_unlock(&rdev->srbm_mutex); 4592 4593 /* init the queues. Just two for now. */ 4594 for (i = 0; i < 2; i++) { 4595 if (i == 0) 4596 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4597 else 4598 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4599 4600 if (rdev->ring[idx].mqd_obj == NULL) { 4601 r = radeon_bo_create(rdev, 4602 sizeof(struct bonaire_mqd), 4603 PAGE_SIZE, true, 4604 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4605 NULL, &rdev->ring[idx].mqd_obj); 4606 if (r) { 4607 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4608 return r; 4609 } 4610 } 4611 4612 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4613 if (unlikely(r != 0)) { 4614 cik_cp_compute_fini(rdev); 4615 return r; 4616 } 4617 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4618 &mqd_gpu_addr); 4619 if (r) { 4620 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4621 cik_cp_compute_fini(rdev); 4622 return r; 4623 } 4624 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4625 if (r) { 4626 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4627 cik_cp_compute_fini(rdev); 4628 return r; 4629 } 4630 4631 /* init the mqd struct */ 4632 memset(buf, 0, sizeof(struct bonaire_mqd)); 4633 4634 mqd = (struct bonaire_mqd *)buf; 4635 mqd->header = 0xC0310800; 4636 mqd->static_thread_mgmt01[0] = 0xffffffff; 4637 mqd->static_thread_mgmt01[1] = 0xffffffff; 4638 mqd->static_thread_mgmt23[0] = 0xffffffff; 4639 mqd->static_thread_mgmt23[1] = 0xffffffff; 4640 4641 mutex_lock(&rdev->srbm_mutex); 4642 cik_srbm_select(rdev, rdev->ring[idx].me, 4643 rdev->ring[idx].pipe, 4644 rdev->ring[idx].queue, 0); 4645 4646 /* disable wptr polling */ 4647 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4648 tmp &= ~WPTR_POLL_EN; 4649 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4650 4651 /* enable doorbell? */ 4652 mqd->queue_state.cp_hqd_pq_doorbell_control = 4653 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4654 if (use_doorbell) 4655 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4656 else 4657 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4658 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4659 mqd->queue_state.cp_hqd_pq_doorbell_control); 4660 4661 /* disable the queue if it's active */ 4662 mqd->queue_state.cp_hqd_dequeue_request = 0; 4663 mqd->queue_state.cp_hqd_pq_rptr = 0; 4664 mqd->queue_state.cp_hqd_pq_wptr= 0; 4665 if (RREG32(CP_HQD_ACTIVE) & 1) { 4666 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4667 for (j = 0; j < rdev->usec_timeout; j++) { 4668 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4669 break; 4670 udelay(1); 4671 } 4672 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4673 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4674 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4675 } 4676 4677 /* set the pointer to the MQD */ 4678 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4679 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4680 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4681 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4682 /* set MQD vmid to 0 */ 4683 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4684 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4685 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4686 4687 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4688 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4689 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4690 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4691 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4692 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4693 4694 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4695 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4696 mqd->queue_state.cp_hqd_pq_control &= 4697 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4698 4699 mqd->queue_state.cp_hqd_pq_control |= 4700 order_base_2(rdev->ring[idx].ring_size / 8); 4701 mqd->queue_state.cp_hqd_pq_control |= 4702 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4703 #ifdef __BIG_ENDIAN 4704 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4705 #endif 4706 mqd->queue_state.cp_hqd_pq_control &= 4707 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4708 mqd->queue_state.cp_hqd_pq_control |= 4709 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4710 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4711 4712 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 4713 if (i == 0) 4714 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 4715 else 4716 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 4717 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 4718 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4719 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 4720 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 4721 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 4722 4723 /* set the wb address wether it's enabled or not */ 4724 if (i == 0) 4725 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 4726 else 4727 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 4728 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 4729 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 4730 upper_32_bits(wb_gpu_addr) & 0xffff; 4731 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 4732 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 4733 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4734 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 4735 4736 /* enable the doorbell if requested */ 4737 if (use_doorbell) { 4738 mqd->queue_state.cp_hqd_pq_doorbell_control = 4739 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4740 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 4741 mqd->queue_state.cp_hqd_pq_doorbell_control |= 4742 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 4743 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4744 mqd->queue_state.cp_hqd_pq_doorbell_control &= 4745 ~(DOORBELL_SOURCE | DOORBELL_HIT); 4746 4747 } else { 4748 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 4749 } 4750 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4751 mqd->queue_state.cp_hqd_pq_doorbell_control); 4752 4753 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4754 rdev->ring[idx].wptr = 0; 4755 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 4756 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4757 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 4758 4759 /* set the vmid for the queue */ 4760 mqd->queue_state.cp_hqd_vmid = 0; 4761 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 4762 4763 /* activate the queue */ 4764 mqd->queue_state.cp_hqd_active = 1; 4765 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 4766 4767 cik_srbm_select(rdev, 0, 0, 0, 0); 4768 mutex_unlock(&rdev->srbm_mutex); 4769 4770 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 4771 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4772 4773 rdev->ring[idx].ready = true; 4774 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 4775 if (r) 4776 rdev->ring[idx].ready = false; 4777 } 4778 4779 return 0; 4780 } 4781 4782 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 4783 { 4784 cik_cp_gfx_enable(rdev, enable); 4785 cik_cp_compute_enable(rdev, enable); 4786 } 4787 4788 static int cik_cp_load_microcode(struct radeon_device *rdev) 4789 { 4790 int r; 4791 4792 r = cik_cp_gfx_load_microcode(rdev); 4793 if (r) 4794 return r; 4795 r = cik_cp_compute_load_microcode(rdev); 4796 if (r) 4797 return r; 4798 4799 return 0; 4800 } 4801 4802 static void cik_cp_fini(struct radeon_device *rdev) 4803 { 4804 cik_cp_gfx_fini(rdev); 4805 cik_cp_compute_fini(rdev); 4806 } 4807 4808 static int cik_cp_resume(struct radeon_device *rdev) 4809 { 4810 int r; 4811 4812 cik_enable_gui_idle_interrupt(rdev, false); 4813 4814 r = cik_cp_load_microcode(rdev); 4815 if (r) 4816 return r; 4817 4818 r = cik_cp_gfx_resume(rdev); 4819 if (r) 4820 return r; 4821 r = cik_cp_compute_resume(rdev); 4822 if (r) 4823 return r; 4824 4825 cik_enable_gui_idle_interrupt(rdev, true); 4826 4827 return 0; 4828 } 4829 4830 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 4831 { 4832 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 4833 RREG32(GRBM_STATUS)); 4834 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 4835 RREG32(GRBM_STATUS2)); 4836 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 4837 RREG32(GRBM_STATUS_SE0)); 4838 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 4839 RREG32(GRBM_STATUS_SE1)); 4840 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 4841 RREG32(GRBM_STATUS_SE2)); 4842 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 4843 RREG32(GRBM_STATUS_SE3)); 4844 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 4845 RREG32(SRBM_STATUS)); 4846 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 4847 RREG32(SRBM_STATUS2)); 4848 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 4849 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 4850 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 4851 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 4852 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 4853 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 4854 RREG32(CP_STALLED_STAT1)); 4855 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 4856 RREG32(CP_STALLED_STAT2)); 4857 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 4858 RREG32(CP_STALLED_STAT3)); 4859 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 4860 RREG32(CP_CPF_BUSY_STAT)); 4861 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 4862 RREG32(CP_CPF_STALLED_STAT1)); 4863 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 4864 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 4865 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 4866 RREG32(CP_CPC_STALLED_STAT1)); 4867 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 4868 } 4869 4870 /** 4871 * cik_gpu_check_soft_reset - check which blocks are busy 4872 * 4873 * @rdev: radeon_device pointer 4874 * 4875 * Check which blocks are busy and return the relevant reset 4876 * mask to be used by cik_gpu_soft_reset(). 4877 * Returns a mask of the blocks to be reset. 4878 */ 4879 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 4880 { 4881 u32 reset_mask = 0; 4882 u32 tmp; 4883 4884 /* GRBM_STATUS */ 4885 tmp = RREG32(GRBM_STATUS); 4886 if (tmp & (PA_BUSY | SC_BUSY | 4887 BCI_BUSY | SX_BUSY | 4888 TA_BUSY | VGT_BUSY | 4889 DB_BUSY | CB_BUSY | 4890 GDS_BUSY | SPI_BUSY | 4891 IA_BUSY | IA_BUSY_NO_DMA)) 4892 reset_mask |= RADEON_RESET_GFX; 4893 4894 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 4895 reset_mask |= RADEON_RESET_CP; 4896 4897 /* GRBM_STATUS2 */ 4898 tmp = RREG32(GRBM_STATUS2); 4899 if (tmp & RLC_BUSY) 4900 reset_mask |= RADEON_RESET_RLC; 4901 4902 /* SDMA0_STATUS_REG */ 4903 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 4904 if (!(tmp & SDMA_IDLE)) 4905 reset_mask |= RADEON_RESET_DMA; 4906 4907 /* SDMA1_STATUS_REG */ 4908 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 4909 if (!(tmp & SDMA_IDLE)) 4910 reset_mask |= RADEON_RESET_DMA1; 4911 4912 /* SRBM_STATUS2 */ 4913 tmp = RREG32(SRBM_STATUS2); 4914 if (tmp & SDMA_BUSY) 4915 reset_mask |= RADEON_RESET_DMA; 4916 4917 if (tmp & SDMA1_BUSY) 4918 reset_mask |= RADEON_RESET_DMA1; 4919 4920 /* SRBM_STATUS */ 4921 tmp = RREG32(SRBM_STATUS); 4922 4923 if (tmp & IH_BUSY) 4924 reset_mask |= RADEON_RESET_IH; 4925 4926 if (tmp & SEM_BUSY) 4927 reset_mask |= RADEON_RESET_SEM; 4928 4929 if (tmp & GRBM_RQ_PENDING) 4930 reset_mask |= RADEON_RESET_GRBM; 4931 4932 if (tmp & VMC_BUSY) 4933 reset_mask |= RADEON_RESET_VMC; 4934 4935 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 4936 MCC_BUSY | MCD_BUSY)) 4937 reset_mask |= RADEON_RESET_MC; 4938 4939 if (evergreen_is_display_hung(rdev)) 4940 reset_mask |= RADEON_RESET_DISPLAY; 4941 4942 /* Skip MC reset as it's mostly likely not hung, just busy */ 4943 if (reset_mask & RADEON_RESET_MC) { 4944 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 4945 reset_mask &= ~RADEON_RESET_MC; 4946 } 4947 4948 return reset_mask; 4949 } 4950 4951 /** 4952 * cik_gpu_soft_reset - soft reset GPU 4953 * 4954 * @rdev: radeon_device pointer 4955 * @reset_mask: mask of which blocks to reset 4956 * 4957 * Soft reset the blocks specified in @reset_mask. 4958 */ 4959 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 4960 { 4961 struct evergreen_mc_save save; 4962 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 4963 u32 tmp; 4964 4965 if (reset_mask == 0) 4966 return; 4967 4968 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 4969 4970 cik_print_gpu_status_regs(rdev); 4971 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 4972 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 4973 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 4974 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 4975 4976 /* disable CG/PG */ 4977 cik_fini_pg(rdev); 4978 cik_fini_cg(rdev); 4979 4980 /* stop the rlc */ 4981 cik_rlc_stop(rdev); 4982 4983 /* Disable GFX parsing/prefetching */ 4984 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 4985 4986 /* Disable MEC parsing/prefetching */ 4987 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 4988 4989 if (reset_mask & RADEON_RESET_DMA) { 4990 /* sdma0 */ 4991 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 4992 tmp |= SDMA_HALT; 4993 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 4994 } 4995 if (reset_mask & RADEON_RESET_DMA1) { 4996 /* sdma1 */ 4997 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 4998 tmp |= SDMA_HALT; 4999 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5000 } 5001 5002 evergreen_mc_stop(rdev, &save); 5003 if (evergreen_mc_wait_for_idle(rdev)) { 5004 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5005 } 5006 5007 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 5008 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 5009 5010 if (reset_mask & RADEON_RESET_CP) { 5011 grbm_soft_reset |= SOFT_RESET_CP; 5012 5013 srbm_soft_reset |= SOFT_RESET_GRBM; 5014 } 5015 5016 if (reset_mask & RADEON_RESET_DMA) 5017 srbm_soft_reset |= SOFT_RESET_SDMA; 5018 5019 if (reset_mask & RADEON_RESET_DMA1) 5020 srbm_soft_reset |= SOFT_RESET_SDMA1; 5021 5022 if (reset_mask & RADEON_RESET_DISPLAY) 5023 srbm_soft_reset |= SOFT_RESET_DC; 5024 5025 if (reset_mask & RADEON_RESET_RLC) 5026 grbm_soft_reset |= SOFT_RESET_RLC; 5027 5028 if (reset_mask & RADEON_RESET_SEM) 5029 srbm_soft_reset |= SOFT_RESET_SEM; 5030 5031 if (reset_mask & RADEON_RESET_IH) 5032 srbm_soft_reset |= SOFT_RESET_IH; 5033 5034 if (reset_mask & RADEON_RESET_GRBM) 5035 srbm_soft_reset |= SOFT_RESET_GRBM; 5036 5037 if (reset_mask & RADEON_RESET_VMC) 5038 srbm_soft_reset |= SOFT_RESET_VMC; 5039 5040 if (!(rdev->flags & RADEON_IS_IGP)) { 5041 if (reset_mask & RADEON_RESET_MC) 5042 srbm_soft_reset |= SOFT_RESET_MC; 5043 } 5044 5045 if (grbm_soft_reset) { 5046 tmp = RREG32(GRBM_SOFT_RESET); 5047 tmp |= grbm_soft_reset; 5048 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5049 WREG32(GRBM_SOFT_RESET, tmp); 5050 tmp = RREG32(GRBM_SOFT_RESET); 5051 5052 udelay(50); 5053 5054 tmp &= ~grbm_soft_reset; 5055 WREG32(GRBM_SOFT_RESET, tmp); 5056 tmp = RREG32(GRBM_SOFT_RESET); 5057 } 5058 5059 if (srbm_soft_reset) { 5060 tmp = RREG32(SRBM_SOFT_RESET); 5061 tmp |= srbm_soft_reset; 5062 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5063 WREG32(SRBM_SOFT_RESET, tmp); 5064 tmp = RREG32(SRBM_SOFT_RESET); 5065 5066 udelay(50); 5067 5068 tmp &= ~srbm_soft_reset; 5069 WREG32(SRBM_SOFT_RESET, tmp); 5070 tmp = RREG32(SRBM_SOFT_RESET); 5071 } 5072 5073 /* Wait a little for things to settle down */ 5074 udelay(50); 5075 5076 evergreen_mc_resume(rdev, &save); 5077 udelay(50); 5078 5079 cik_print_gpu_status_regs(rdev); 5080 } 5081 5082 struct kv_reset_save_regs { 5083 u32 gmcon_reng_execute; 5084 u32 gmcon_misc; 5085 u32 gmcon_misc3; 5086 }; 5087 5088 static void kv_save_regs_for_reset(struct radeon_device *rdev, 5089 struct kv_reset_save_regs *save) 5090 { 5091 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 5092 save->gmcon_misc = RREG32(GMCON_MISC); 5093 save->gmcon_misc3 = RREG32(GMCON_MISC3); 5094 5095 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 5096 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 5097 STCTRL_STUTTER_EN)); 5098 } 5099 5100 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 5101 struct kv_reset_save_regs *save) 5102 { 5103 int i; 5104 5105 WREG32(GMCON_PGFSM_WRITE, 0); 5106 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 5107 5108 for (i = 0; i < 5; i++) 5109 WREG32(GMCON_PGFSM_WRITE, 0); 5110 5111 WREG32(GMCON_PGFSM_WRITE, 0); 5112 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 5113 5114 for (i = 0; i < 5; i++) 5115 WREG32(GMCON_PGFSM_WRITE, 0); 5116 5117 WREG32(GMCON_PGFSM_WRITE, 0x210000); 5118 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 5119 5120 for (i = 0; i < 5; i++) 5121 WREG32(GMCON_PGFSM_WRITE, 0); 5122 5123 WREG32(GMCON_PGFSM_WRITE, 0x21003); 5124 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 5125 5126 for (i = 0; i < 5; i++) 5127 WREG32(GMCON_PGFSM_WRITE, 0); 5128 5129 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 5130 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 5131 5132 for (i = 0; i < 5; i++) 5133 WREG32(GMCON_PGFSM_WRITE, 0); 5134 5135 WREG32(GMCON_PGFSM_WRITE, 0); 5136 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 5137 5138 for (i = 0; i < 5; i++) 5139 WREG32(GMCON_PGFSM_WRITE, 0); 5140 5141 WREG32(GMCON_PGFSM_WRITE, 0x420000); 5142 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 5143 5144 for (i = 0; i < 5; i++) 5145 WREG32(GMCON_PGFSM_WRITE, 0); 5146 5147 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5148 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5149 5150 for (i = 0; i < 5; i++) 5151 WREG32(GMCON_PGFSM_WRITE, 0); 5152 5153 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5154 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5155 5156 for (i = 0; i < 5; i++) 5157 WREG32(GMCON_PGFSM_WRITE, 0); 5158 5159 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5160 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5161 5162 for (i = 0; i < 5; i++) 5163 WREG32(GMCON_PGFSM_WRITE, 0); 5164 5165 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5166 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5167 5168 WREG32(GMCON_MISC3, save->gmcon_misc3); 5169 WREG32(GMCON_MISC, save->gmcon_misc); 5170 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5171 } 5172 5173 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5174 { 5175 struct evergreen_mc_save save; 5176 struct kv_reset_save_regs kv_save = { 0 }; 5177 u32 tmp, i; 5178 5179 dev_info(rdev->dev, "GPU pci config reset\n"); 5180 5181 /* disable dpm? */ 5182 5183 /* disable cg/pg */ 5184 cik_fini_pg(rdev); 5185 cik_fini_cg(rdev); 5186 5187 /* Disable GFX parsing/prefetching */ 5188 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5189 5190 /* Disable MEC parsing/prefetching */ 5191 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5192 5193 /* sdma0 */ 5194 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5195 tmp |= SDMA_HALT; 5196 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5197 /* sdma1 */ 5198 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5199 tmp |= SDMA_HALT; 5200 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5201 /* XXX other engines? */ 5202 5203 /* halt the rlc, disable cp internal ints */ 5204 cik_rlc_stop(rdev); 5205 5206 udelay(50); 5207 5208 /* disable mem access */ 5209 evergreen_mc_stop(rdev, &save); 5210 if (evergreen_mc_wait_for_idle(rdev)) { 5211 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5212 } 5213 5214 if (rdev->flags & RADEON_IS_IGP) 5215 kv_save_regs_for_reset(rdev, &kv_save); 5216 5217 /* disable BM */ 5218 pci_clear_master(rdev->pdev); 5219 /* reset */ 5220 radeon_pci_config_reset(rdev); 5221 5222 udelay(100); 5223 5224 /* wait for asic to come out of reset */ 5225 for (i = 0; i < rdev->usec_timeout; i++) { 5226 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5227 break; 5228 udelay(1); 5229 } 5230 5231 /* does asic init need to be run first??? */ 5232 if (rdev->flags & RADEON_IS_IGP) 5233 kv_restore_regs_for_reset(rdev, &kv_save); 5234 } 5235 5236 /** 5237 * cik_asic_reset - soft reset GPU 5238 * 5239 * @rdev: radeon_device pointer 5240 * @hard: force hard reset 5241 * 5242 * Look up which blocks are hung and attempt 5243 * to reset them. 5244 * Returns 0 for success. 5245 */ 5246 int cik_asic_reset(struct radeon_device *rdev, bool hard) 5247 { 5248 u32 reset_mask; 5249 5250 if (hard) { 5251 cik_gpu_pci_config_reset(rdev); 5252 return 0; 5253 } 5254 5255 reset_mask = cik_gpu_check_soft_reset(rdev); 5256 5257 if (reset_mask) 5258 r600_set_bios_scratch_engine_hung(rdev, true); 5259 5260 /* try soft reset */ 5261 cik_gpu_soft_reset(rdev, reset_mask); 5262 5263 reset_mask = cik_gpu_check_soft_reset(rdev); 5264 5265 /* try pci config reset */ 5266 if (reset_mask && radeon_hard_reset) 5267 cik_gpu_pci_config_reset(rdev); 5268 5269 reset_mask = cik_gpu_check_soft_reset(rdev); 5270 5271 if (!reset_mask) 5272 r600_set_bios_scratch_engine_hung(rdev, false); 5273 5274 return 0; 5275 } 5276 5277 /** 5278 * cik_gfx_is_lockup - check if the 3D engine is locked up 5279 * 5280 * @rdev: radeon_device pointer 5281 * @ring: radeon_ring structure holding ring information 5282 * 5283 * Check if the 3D engine is locked up (CIK). 5284 * Returns true if the engine is locked, false if not. 5285 */ 5286 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5287 { 5288 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5289 5290 if (!(reset_mask & (RADEON_RESET_GFX | 5291 RADEON_RESET_COMPUTE | 5292 RADEON_RESET_CP))) { 5293 radeon_ring_lockup_update(rdev, ring); 5294 return false; 5295 } 5296 return radeon_ring_test_lockup(rdev, ring); 5297 } 5298 5299 /* MC */ 5300 /** 5301 * cik_mc_program - program the GPU memory controller 5302 * 5303 * @rdev: radeon_device pointer 5304 * 5305 * Set the location of vram, gart, and AGP in the GPU's 5306 * physical address space (CIK). 5307 */ 5308 static void cik_mc_program(struct radeon_device *rdev) 5309 { 5310 struct evergreen_mc_save save; 5311 u32 tmp; 5312 int i, j; 5313 5314 /* Initialize HDP */ 5315 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5316 WREG32((0x2c14 + j), 0x00000000); 5317 WREG32((0x2c18 + j), 0x00000000); 5318 WREG32((0x2c1c + j), 0x00000000); 5319 WREG32((0x2c20 + j), 0x00000000); 5320 WREG32((0x2c24 + j), 0x00000000); 5321 } 5322 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5323 5324 evergreen_mc_stop(rdev, &save); 5325 if (radeon_mc_wait_for_idle(rdev)) { 5326 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5327 } 5328 /* Lockout access through VGA aperture*/ 5329 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5330 /* Update configuration */ 5331 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5332 rdev->mc.vram_start >> 12); 5333 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5334 rdev->mc.vram_end >> 12); 5335 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5336 rdev->vram_scratch.gpu_addr >> 12); 5337 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5338 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5339 WREG32(MC_VM_FB_LOCATION, tmp); 5340 /* XXX double check these! */ 5341 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5342 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5343 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5344 WREG32(MC_VM_AGP_BASE, 0); 5345 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5346 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5347 if (radeon_mc_wait_for_idle(rdev)) { 5348 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5349 } 5350 evergreen_mc_resume(rdev, &save); 5351 /* we need to own VRAM, so turn off the VGA renderer here 5352 * to stop it overwriting our objects */ 5353 rv515_vga_render_disable(rdev); 5354 } 5355 5356 /** 5357 * cik_mc_init - initialize the memory controller driver params 5358 * 5359 * @rdev: radeon_device pointer 5360 * 5361 * Look up the amount of vram, vram width, and decide how to place 5362 * vram and gart within the GPU's physical address space (CIK). 5363 * Returns 0 for success. 5364 */ 5365 static int cik_mc_init(struct radeon_device *rdev) 5366 { 5367 u32 tmp; 5368 int chansize, numchan; 5369 5370 /* Get VRAM informations */ 5371 rdev->mc.vram_is_ddr = true; 5372 tmp = RREG32(MC_ARB_RAMCFG); 5373 if (tmp & CHANSIZE_MASK) { 5374 chansize = 64; 5375 } else { 5376 chansize = 32; 5377 } 5378 tmp = RREG32(MC_SHARED_CHMAP); 5379 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5380 case 0: 5381 default: 5382 numchan = 1; 5383 break; 5384 case 1: 5385 numchan = 2; 5386 break; 5387 case 2: 5388 numchan = 4; 5389 break; 5390 case 3: 5391 numchan = 8; 5392 break; 5393 case 4: 5394 numchan = 3; 5395 break; 5396 case 5: 5397 numchan = 6; 5398 break; 5399 case 6: 5400 numchan = 10; 5401 break; 5402 case 7: 5403 numchan = 12; 5404 break; 5405 case 8: 5406 numchan = 16; 5407 break; 5408 } 5409 rdev->mc.vram_width = numchan * chansize; 5410 /* Could aper size report 0 ? */ 5411 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5412 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5413 /* size in MB on si */ 5414 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5415 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5416 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5417 si_vram_gtt_location(rdev, &rdev->mc); 5418 radeon_update_bandwidth_info(rdev); 5419 5420 return 0; 5421 } 5422 5423 /* 5424 * GART 5425 * VMID 0 is the physical GPU addresses as used by the kernel. 5426 * VMIDs 1-15 are used for userspace clients and are handled 5427 * by the radeon vm/hsa code. 5428 */ 5429 /** 5430 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5431 * 5432 * @rdev: radeon_device pointer 5433 * 5434 * Flush the TLB for the VMID 0 page table (CIK). 5435 */ 5436 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5437 { 5438 /* flush hdp cache */ 5439 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5440 5441 /* bits 0-15 are the VM contexts0-15 */ 5442 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5443 } 5444 5445 /** 5446 * cik_pcie_gart_enable - gart enable 5447 * 5448 * @rdev: radeon_device pointer 5449 * 5450 * This sets up the TLBs, programs the page tables for VMID0, 5451 * sets up the hw for VMIDs 1-15 which are allocated on 5452 * demand, and sets up the global locations for the LDS, GDS, 5453 * and GPUVM for FSA64 clients (CIK). 5454 * Returns 0 for success, errors for failure. 5455 */ 5456 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5457 { 5458 int r, i; 5459 5460 if (rdev->gart.robj == NULL) { 5461 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5462 return -EINVAL; 5463 } 5464 r = radeon_gart_table_vram_pin(rdev); 5465 if (r) 5466 return r; 5467 /* Setup TLB control */ 5468 WREG32(MC_VM_MX_L1_TLB_CNTL, 5469 (0xA << 7) | 5470 ENABLE_L1_TLB | 5471 ENABLE_L1_FRAGMENT_PROCESSING | 5472 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5473 ENABLE_ADVANCED_DRIVER_MODEL | 5474 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5475 /* Setup L2 cache */ 5476 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5477 ENABLE_L2_FRAGMENT_PROCESSING | 5478 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5479 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5480 EFFECTIVE_L2_QUEUE_SIZE(7) | 5481 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5482 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5483 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5484 BANK_SELECT(4) | 5485 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 5486 /* setup context0 */ 5487 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5488 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5489 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5490 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5491 (u32)(rdev->dummy_page.addr >> 12)); 5492 WREG32(VM_CONTEXT0_CNTL2, 0); 5493 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5494 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5495 5496 WREG32(0x15D4, 0); 5497 WREG32(0x15D8, 0); 5498 WREG32(0x15DC, 0); 5499 5500 /* restore context1-15 */ 5501 /* set vm size, must be a multiple of 4 */ 5502 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5503 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); 5504 for (i = 1; i < 16; i++) { 5505 if (i < 8) 5506 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5507 rdev->vm_manager.saved_table_addr[i]); 5508 else 5509 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5510 rdev->vm_manager.saved_table_addr[i]); 5511 } 5512 5513 /* enable context1-15 */ 5514 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5515 (u32)(rdev->dummy_page.addr >> 12)); 5516 WREG32(VM_CONTEXT1_CNTL2, 4); 5517 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5518 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 5519 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5520 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5521 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5522 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5523 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5524 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5525 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5526 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5527 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5528 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5529 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5530 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5531 5532 if (rdev->family == CHIP_KAVERI) { 5533 u32 tmp = RREG32(CHUB_CONTROL); 5534 tmp &= ~BYPASS_VM; 5535 WREG32(CHUB_CONTROL, tmp); 5536 } 5537 5538 /* XXX SH_MEM regs */ 5539 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5540 mutex_lock(&rdev->srbm_mutex); 5541 for (i = 0; i < 16; i++) { 5542 cik_srbm_select(rdev, 0, 0, 0, i); 5543 /* CP and shaders */ 5544 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT); 5545 WREG32(SH_MEM_APE1_BASE, 1); 5546 WREG32(SH_MEM_APE1_LIMIT, 0); 5547 WREG32(SH_MEM_BASES, 0); 5548 /* SDMA GFX */ 5549 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5550 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5551 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5552 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5553 /* XXX SDMA RLC - todo */ 5554 } 5555 cik_srbm_select(rdev, 0, 0, 0, 0); 5556 mutex_unlock(&rdev->srbm_mutex); 5557 5558 cik_pcie_gart_tlb_flush(rdev); 5559 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5560 (unsigned)(rdev->mc.gtt_size >> 20), 5561 (unsigned long long)rdev->gart.table_addr); 5562 rdev->gart.ready = true; 5563 return 0; 5564 } 5565 5566 /** 5567 * cik_pcie_gart_disable - gart disable 5568 * 5569 * @rdev: radeon_device pointer 5570 * 5571 * This disables all VM page table (CIK). 5572 */ 5573 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5574 { 5575 unsigned i; 5576 5577 for (i = 1; i < 16; ++i) { 5578 uint32_t reg; 5579 if (i < 8) 5580 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); 5581 else 5582 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); 5583 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); 5584 } 5585 5586 /* Disable all tables */ 5587 WREG32(VM_CONTEXT0_CNTL, 0); 5588 WREG32(VM_CONTEXT1_CNTL, 0); 5589 /* Setup TLB control */ 5590 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5591 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5592 /* Setup L2 cache */ 5593 WREG32(VM_L2_CNTL, 5594 ENABLE_L2_FRAGMENT_PROCESSING | 5595 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5596 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5597 EFFECTIVE_L2_QUEUE_SIZE(7) | 5598 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5599 WREG32(VM_L2_CNTL2, 0); 5600 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5601 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5602 radeon_gart_table_vram_unpin(rdev); 5603 } 5604 5605 /** 5606 * cik_pcie_gart_fini - vm fini callback 5607 * 5608 * @rdev: radeon_device pointer 5609 * 5610 * Tears down the driver GART/VM setup (CIK). 5611 */ 5612 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5613 { 5614 cik_pcie_gart_disable(rdev); 5615 radeon_gart_table_vram_free(rdev); 5616 radeon_gart_fini(rdev); 5617 } 5618 5619 /* vm parser */ 5620 /** 5621 * cik_ib_parse - vm ib_parse callback 5622 * 5623 * @rdev: radeon_device pointer 5624 * @ib: indirect buffer pointer 5625 * 5626 * CIK uses hw IB checking so this is a nop (CIK). 5627 */ 5628 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5629 { 5630 return 0; 5631 } 5632 5633 /* 5634 * vm 5635 * VMID 0 is the physical GPU addresses as used by the kernel. 5636 * VMIDs 1-15 are used for userspace clients and are handled 5637 * by the radeon vm/hsa code. 5638 */ 5639 /** 5640 * cik_vm_init - cik vm init callback 5641 * 5642 * @rdev: radeon_device pointer 5643 * 5644 * Inits cik specific vm parameters (number of VMs, base of vram for 5645 * VMIDs 1-15) (CIK). 5646 * Returns 0 for success. 5647 */ 5648 int cik_vm_init(struct radeon_device *rdev) 5649 { 5650 /* 5651 * number of VMs 5652 * VMID 0 is reserved for System 5653 * radeon graphics/compute will use VMIDs 1-15 5654 */ 5655 rdev->vm_manager.nvm = 16; 5656 /* base offset of vram pages */ 5657 if (rdev->flags & RADEON_IS_IGP) { 5658 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5659 tmp <<= 22; 5660 rdev->vm_manager.vram_base_offset = tmp; 5661 } else 5662 rdev->vm_manager.vram_base_offset = 0; 5663 5664 return 0; 5665 } 5666 5667 /** 5668 * cik_vm_fini - cik vm fini callback 5669 * 5670 * @rdev: radeon_device pointer 5671 * 5672 * Tear down any asic specific VM setup (CIK). 5673 */ 5674 void cik_vm_fini(struct radeon_device *rdev) 5675 { 5676 } 5677 5678 /** 5679 * cik_vm_decode_fault - print human readable fault info 5680 * 5681 * @rdev: radeon_device pointer 5682 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5683 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5684 * 5685 * Print human readable fault information (CIK). 5686 */ 5687 static void cik_vm_decode_fault(struct radeon_device *rdev, 5688 u32 status, u32 addr, u32 mc_client) 5689 { 5690 u32 mc_id; 5691 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5692 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 5693 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 5694 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 5695 5696 if (rdev->family == CHIP_HAWAII) 5697 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5698 else 5699 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5700 5701 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 5702 protections, vmid, addr, 5703 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 5704 block, mc_client, mc_id); 5705 } 5706 5707 /** 5708 * cik_vm_flush - cik vm flush using the CP 5709 * 5710 * @rdev: radeon_device pointer 5711 * 5712 * Update the page table base and flush the VM TLB 5713 * using the CP (CIK). 5714 */ 5715 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, 5716 unsigned vm_id, uint64_t pd_addr) 5717 { 5718 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); 5719 5720 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5721 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5722 WRITE_DATA_DST_SEL(0))); 5723 if (vm_id < 8) { 5724 radeon_ring_write(ring, 5725 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); 5726 } else { 5727 radeon_ring_write(ring, 5728 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); 5729 } 5730 radeon_ring_write(ring, 0); 5731 radeon_ring_write(ring, pd_addr >> 12); 5732 5733 /* update SH_MEM_* regs */ 5734 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5735 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5736 WRITE_DATA_DST_SEL(0))); 5737 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5738 radeon_ring_write(ring, 0); 5739 radeon_ring_write(ring, VMID(vm_id)); 5740 5741 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 5742 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5743 WRITE_DATA_DST_SEL(0))); 5744 radeon_ring_write(ring, SH_MEM_BASES >> 2); 5745 radeon_ring_write(ring, 0); 5746 5747 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 5748 radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */ 5749 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 5750 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 5751 5752 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5753 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5754 WRITE_DATA_DST_SEL(0))); 5755 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5756 radeon_ring_write(ring, 0); 5757 radeon_ring_write(ring, VMID(0)); 5758 5759 /* HDP flush */ 5760 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); 5761 5762 /* bits 0-15 are the VM contexts0-15 */ 5763 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5764 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 5765 WRITE_DATA_DST_SEL(0))); 5766 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5767 radeon_ring_write(ring, 0); 5768 radeon_ring_write(ring, 1 << vm_id); 5769 5770 /* wait for the invalidate to complete */ 5771 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 5772 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 5773 WAIT_REG_MEM_FUNCTION(0) | /* always */ 5774 WAIT_REG_MEM_ENGINE(0))); /* me */ 5775 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5776 radeon_ring_write(ring, 0); 5777 radeon_ring_write(ring, 0); /* ref */ 5778 radeon_ring_write(ring, 0); /* mask */ 5779 radeon_ring_write(ring, 0x20); /* poll interval */ 5780 5781 /* compute doesn't have PFP */ 5782 if (usepfp) { 5783 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5784 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5785 radeon_ring_write(ring, 0x0); 5786 } 5787 } 5788 5789 /* 5790 * RLC 5791 * The RLC is a multi-purpose microengine that handles a 5792 * variety of functions, the most important of which is 5793 * the interrupt controller. 5794 */ 5795 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 5796 bool enable) 5797 { 5798 u32 tmp = RREG32(CP_INT_CNTL_RING0); 5799 5800 if (enable) 5801 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5802 else 5803 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5804 WREG32(CP_INT_CNTL_RING0, tmp); 5805 } 5806 5807 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 5808 { 5809 u32 tmp; 5810 5811 tmp = RREG32(RLC_LB_CNTL); 5812 if (enable) 5813 tmp |= LOAD_BALANCE_ENABLE; 5814 else 5815 tmp &= ~LOAD_BALANCE_ENABLE; 5816 WREG32(RLC_LB_CNTL, tmp); 5817 } 5818 5819 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 5820 { 5821 u32 i, j, k; 5822 u32 mask; 5823 5824 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 5825 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 5826 cik_select_se_sh(rdev, i, j); 5827 for (k = 0; k < rdev->usec_timeout; k++) { 5828 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 5829 break; 5830 udelay(1); 5831 } 5832 } 5833 } 5834 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5835 5836 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 5837 for (k = 0; k < rdev->usec_timeout; k++) { 5838 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 5839 break; 5840 udelay(1); 5841 } 5842 } 5843 5844 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 5845 { 5846 u32 tmp; 5847 5848 tmp = RREG32(RLC_CNTL); 5849 if (tmp != rlc) 5850 WREG32(RLC_CNTL, rlc); 5851 } 5852 5853 static u32 cik_halt_rlc(struct radeon_device *rdev) 5854 { 5855 u32 data, orig; 5856 5857 orig = data = RREG32(RLC_CNTL); 5858 5859 if (data & RLC_ENABLE) { 5860 u32 i; 5861 5862 data &= ~RLC_ENABLE; 5863 WREG32(RLC_CNTL, data); 5864 5865 for (i = 0; i < rdev->usec_timeout; i++) { 5866 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 5867 break; 5868 udelay(1); 5869 } 5870 5871 cik_wait_for_rlc_serdes(rdev); 5872 } 5873 5874 return orig; 5875 } 5876 5877 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 5878 { 5879 u32 tmp, i, mask; 5880 5881 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 5882 WREG32(RLC_GPR_REG2, tmp); 5883 5884 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 5885 for (i = 0; i < rdev->usec_timeout; i++) { 5886 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 5887 break; 5888 udelay(1); 5889 } 5890 5891 for (i = 0; i < rdev->usec_timeout; i++) { 5892 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 5893 break; 5894 udelay(1); 5895 } 5896 } 5897 5898 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 5899 { 5900 u32 tmp; 5901 5902 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 5903 WREG32(RLC_GPR_REG2, tmp); 5904 } 5905 5906 /** 5907 * cik_rlc_stop - stop the RLC ME 5908 * 5909 * @rdev: radeon_device pointer 5910 * 5911 * Halt the RLC ME (MicroEngine) (CIK). 5912 */ 5913 static void cik_rlc_stop(struct radeon_device *rdev) 5914 { 5915 WREG32(RLC_CNTL, 0); 5916 5917 cik_enable_gui_idle_interrupt(rdev, false); 5918 5919 cik_wait_for_rlc_serdes(rdev); 5920 } 5921 5922 /** 5923 * cik_rlc_start - start the RLC ME 5924 * 5925 * @rdev: radeon_device pointer 5926 * 5927 * Unhalt the RLC ME (MicroEngine) (CIK). 5928 */ 5929 static void cik_rlc_start(struct radeon_device *rdev) 5930 { 5931 WREG32(RLC_CNTL, RLC_ENABLE); 5932 5933 cik_enable_gui_idle_interrupt(rdev, true); 5934 5935 udelay(50); 5936 } 5937 5938 /** 5939 * cik_rlc_resume - setup the RLC hw 5940 * 5941 * @rdev: radeon_device pointer 5942 * 5943 * Initialize the RLC registers, load the ucode, 5944 * and start the RLC (CIK). 5945 * Returns 0 for success, -EINVAL if the ucode is not available. 5946 */ 5947 static int cik_rlc_resume(struct radeon_device *rdev) 5948 { 5949 u32 i, size, tmp; 5950 5951 if (!rdev->rlc_fw) 5952 return -EINVAL; 5953 5954 cik_rlc_stop(rdev); 5955 5956 /* disable CG */ 5957 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 5958 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 5959 5960 si_rlc_reset(rdev); 5961 5962 cik_init_pg(rdev); 5963 5964 cik_init_cg(rdev); 5965 5966 WREG32(RLC_LB_CNTR_INIT, 0); 5967 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 5968 5969 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5970 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 5971 WREG32(RLC_LB_PARAMS, 0x00600408); 5972 WREG32(RLC_LB_CNTL, 0x80000004); 5973 5974 WREG32(RLC_MC_CNTL, 0); 5975 WREG32(RLC_UCODE_CNTL, 0); 5976 5977 if (rdev->new_fw) { 5978 const struct rlc_firmware_header_v1_0 *hdr = 5979 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 5980 const __le32 *fw_data = (const __le32 *) 5981 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 5982 5983 radeon_ucode_print_rlc_hdr(&hdr->header); 5984 5985 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 5986 WREG32(RLC_GPM_UCODE_ADDR, 0); 5987 for (i = 0; i < size; i++) 5988 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 5989 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); 5990 } else { 5991 const __be32 *fw_data; 5992 5993 switch (rdev->family) { 5994 case CHIP_BONAIRE: 5995 case CHIP_HAWAII: 5996 default: 5997 size = BONAIRE_RLC_UCODE_SIZE; 5998 break; 5999 case CHIP_KAVERI: 6000 size = KV_RLC_UCODE_SIZE; 6001 break; 6002 case CHIP_KABINI: 6003 size = KB_RLC_UCODE_SIZE; 6004 break; 6005 case CHIP_MULLINS: 6006 size = ML_RLC_UCODE_SIZE; 6007 break; 6008 } 6009 6010 fw_data = (const __be32 *)rdev->rlc_fw->data; 6011 WREG32(RLC_GPM_UCODE_ADDR, 0); 6012 for (i = 0; i < size; i++) 6013 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 6014 WREG32(RLC_GPM_UCODE_ADDR, 0); 6015 } 6016 6017 /* XXX - find out what chips support lbpw */ 6018 cik_enable_lbpw(rdev, false); 6019 6020 if (rdev->family == CHIP_BONAIRE) 6021 WREG32(RLC_DRIVER_DMA_STATUS, 0); 6022 6023 cik_rlc_start(rdev); 6024 6025 return 0; 6026 } 6027 6028 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 6029 { 6030 u32 data, orig, tmp, tmp2; 6031 6032 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 6033 6034 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 6035 cik_enable_gui_idle_interrupt(rdev, true); 6036 6037 tmp = cik_halt_rlc(rdev); 6038 6039 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6040 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6041 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6042 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 6043 WREG32(RLC_SERDES_WR_CTRL, tmp2); 6044 6045 cik_update_rlc(rdev, tmp); 6046 6047 data |= CGCG_EN | CGLS_EN; 6048 } else { 6049 cik_enable_gui_idle_interrupt(rdev, false); 6050 6051 RREG32(CB_CGTT_SCLK_CTRL); 6052 RREG32(CB_CGTT_SCLK_CTRL); 6053 RREG32(CB_CGTT_SCLK_CTRL); 6054 RREG32(CB_CGTT_SCLK_CTRL); 6055 6056 data &= ~(CGCG_EN | CGLS_EN); 6057 } 6058 6059 if (orig != data) 6060 WREG32(RLC_CGCG_CGLS_CTRL, data); 6061 6062 } 6063 6064 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 6065 { 6066 u32 data, orig, tmp = 0; 6067 6068 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 6069 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 6070 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 6071 orig = data = RREG32(CP_MEM_SLP_CNTL); 6072 data |= CP_MEM_LS_EN; 6073 if (orig != data) 6074 WREG32(CP_MEM_SLP_CNTL, data); 6075 } 6076 } 6077 6078 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6079 data |= 0x00000001; 6080 data &= 0xfffffffd; 6081 if (orig != data) 6082 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6083 6084 tmp = cik_halt_rlc(rdev); 6085 6086 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6087 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6088 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6089 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 6090 WREG32(RLC_SERDES_WR_CTRL, data); 6091 6092 cik_update_rlc(rdev, tmp); 6093 6094 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 6095 orig = data = RREG32(CGTS_SM_CTRL_REG); 6096 data &= ~SM_MODE_MASK; 6097 data |= SM_MODE(0x2); 6098 data |= SM_MODE_ENABLE; 6099 data &= ~CGTS_OVERRIDE; 6100 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 6101 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 6102 data &= ~CGTS_LS_OVERRIDE; 6103 data &= ~ON_MONITOR_ADD_MASK; 6104 data |= ON_MONITOR_ADD_EN; 6105 data |= ON_MONITOR_ADD(0x96); 6106 if (orig != data) 6107 WREG32(CGTS_SM_CTRL_REG, data); 6108 } 6109 } else { 6110 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6111 data |= 0x00000003; 6112 if (orig != data) 6113 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6114 6115 data = RREG32(RLC_MEM_SLP_CNTL); 6116 if (data & RLC_MEM_LS_EN) { 6117 data &= ~RLC_MEM_LS_EN; 6118 WREG32(RLC_MEM_SLP_CNTL, data); 6119 } 6120 6121 data = RREG32(CP_MEM_SLP_CNTL); 6122 if (data & CP_MEM_LS_EN) { 6123 data &= ~CP_MEM_LS_EN; 6124 WREG32(CP_MEM_SLP_CNTL, data); 6125 } 6126 6127 orig = data = RREG32(CGTS_SM_CTRL_REG); 6128 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 6129 if (orig != data) 6130 WREG32(CGTS_SM_CTRL_REG, data); 6131 6132 tmp = cik_halt_rlc(rdev); 6133 6134 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6135 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6136 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6137 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 6138 WREG32(RLC_SERDES_WR_CTRL, data); 6139 6140 cik_update_rlc(rdev, tmp); 6141 } 6142 } 6143 6144 static const u32 mc_cg_registers[] = 6145 { 6146 MC_HUB_MISC_HUB_CG, 6147 MC_HUB_MISC_SIP_CG, 6148 MC_HUB_MISC_VM_CG, 6149 MC_XPB_CLK_GAT, 6150 ATC_MISC_CG, 6151 MC_CITF_MISC_WR_CG, 6152 MC_CITF_MISC_RD_CG, 6153 MC_CITF_MISC_VM_CG, 6154 VM_L2_CG, 6155 }; 6156 6157 static void cik_enable_mc_ls(struct radeon_device *rdev, 6158 bool enable) 6159 { 6160 int i; 6161 u32 orig, data; 6162 6163 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6164 orig = data = RREG32(mc_cg_registers[i]); 6165 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 6166 data |= MC_LS_ENABLE; 6167 else 6168 data &= ~MC_LS_ENABLE; 6169 if (data != orig) 6170 WREG32(mc_cg_registers[i], data); 6171 } 6172 } 6173 6174 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 6175 bool enable) 6176 { 6177 int i; 6178 u32 orig, data; 6179 6180 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6181 orig = data = RREG32(mc_cg_registers[i]); 6182 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 6183 data |= MC_CG_ENABLE; 6184 else 6185 data &= ~MC_CG_ENABLE; 6186 if (data != orig) 6187 WREG32(mc_cg_registers[i], data); 6188 } 6189 } 6190 6191 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 6192 bool enable) 6193 { 6194 u32 orig, data; 6195 6196 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6197 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6198 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6199 } else { 6200 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6201 data |= 0xff000000; 6202 if (data != orig) 6203 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6204 6205 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6206 data |= 0xff000000; 6207 if (data != orig) 6208 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6209 } 6210 } 6211 6212 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6213 bool enable) 6214 { 6215 u32 orig, data; 6216 6217 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6218 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6219 data |= 0x100; 6220 if (orig != data) 6221 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6222 6223 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6224 data |= 0x100; 6225 if (orig != data) 6226 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6227 } else { 6228 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6229 data &= ~0x100; 6230 if (orig != data) 6231 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6232 6233 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6234 data &= ~0x100; 6235 if (orig != data) 6236 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6237 } 6238 } 6239 6240 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6241 bool enable) 6242 { 6243 u32 orig, data; 6244 6245 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6246 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6247 data = 0xfff; 6248 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6249 6250 orig = data = RREG32(UVD_CGC_CTRL); 6251 data |= DCM; 6252 if (orig != data) 6253 WREG32(UVD_CGC_CTRL, data); 6254 } else { 6255 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6256 data &= ~0xfff; 6257 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6258 6259 orig = data = RREG32(UVD_CGC_CTRL); 6260 data &= ~DCM; 6261 if (orig != data) 6262 WREG32(UVD_CGC_CTRL, data); 6263 } 6264 } 6265 6266 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6267 bool enable) 6268 { 6269 u32 orig, data; 6270 6271 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6272 6273 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6274 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6275 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6276 else 6277 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6278 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6279 6280 if (orig != data) 6281 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6282 } 6283 6284 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6285 bool enable) 6286 { 6287 u32 orig, data; 6288 6289 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6290 6291 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6292 data &= ~CLOCK_GATING_DIS; 6293 else 6294 data |= CLOCK_GATING_DIS; 6295 6296 if (orig != data) 6297 WREG32(HDP_HOST_PATH_CNTL, data); 6298 } 6299 6300 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6301 bool enable) 6302 { 6303 u32 orig, data; 6304 6305 orig = data = RREG32(HDP_MEM_POWER_LS); 6306 6307 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6308 data |= HDP_LS_ENABLE; 6309 else 6310 data &= ~HDP_LS_ENABLE; 6311 6312 if (orig != data) 6313 WREG32(HDP_MEM_POWER_LS, data); 6314 } 6315 6316 void cik_update_cg(struct radeon_device *rdev, 6317 u32 block, bool enable) 6318 { 6319 6320 if (block & RADEON_CG_BLOCK_GFX) { 6321 cik_enable_gui_idle_interrupt(rdev, false); 6322 /* order matters! */ 6323 if (enable) { 6324 cik_enable_mgcg(rdev, true); 6325 cik_enable_cgcg(rdev, true); 6326 } else { 6327 cik_enable_cgcg(rdev, false); 6328 cik_enable_mgcg(rdev, false); 6329 } 6330 cik_enable_gui_idle_interrupt(rdev, true); 6331 } 6332 6333 if (block & RADEON_CG_BLOCK_MC) { 6334 if (!(rdev->flags & RADEON_IS_IGP)) { 6335 cik_enable_mc_mgcg(rdev, enable); 6336 cik_enable_mc_ls(rdev, enable); 6337 } 6338 } 6339 6340 if (block & RADEON_CG_BLOCK_SDMA) { 6341 cik_enable_sdma_mgcg(rdev, enable); 6342 cik_enable_sdma_mgls(rdev, enable); 6343 } 6344 6345 if (block & RADEON_CG_BLOCK_BIF) { 6346 cik_enable_bif_mgls(rdev, enable); 6347 } 6348 6349 if (block & RADEON_CG_BLOCK_UVD) { 6350 if (rdev->has_uvd) 6351 cik_enable_uvd_mgcg(rdev, enable); 6352 } 6353 6354 if (block & RADEON_CG_BLOCK_HDP) { 6355 cik_enable_hdp_mgcg(rdev, enable); 6356 cik_enable_hdp_ls(rdev, enable); 6357 } 6358 6359 if (block & RADEON_CG_BLOCK_VCE) { 6360 vce_v2_0_enable_mgcg(rdev, enable); 6361 } 6362 } 6363 6364 static void cik_init_cg(struct radeon_device *rdev) 6365 { 6366 6367 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6368 6369 if (rdev->has_uvd) 6370 si_init_uvd_internal_cg(rdev); 6371 6372 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6373 RADEON_CG_BLOCK_SDMA | 6374 RADEON_CG_BLOCK_BIF | 6375 RADEON_CG_BLOCK_UVD | 6376 RADEON_CG_BLOCK_HDP), true); 6377 } 6378 6379 static void cik_fini_cg(struct radeon_device *rdev) 6380 { 6381 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6382 RADEON_CG_BLOCK_SDMA | 6383 RADEON_CG_BLOCK_BIF | 6384 RADEON_CG_BLOCK_UVD | 6385 RADEON_CG_BLOCK_HDP), false); 6386 6387 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6388 } 6389 6390 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6391 bool enable) 6392 { 6393 u32 data, orig; 6394 6395 orig = data = RREG32(RLC_PG_CNTL); 6396 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6397 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6398 else 6399 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6400 if (orig != data) 6401 WREG32(RLC_PG_CNTL, data); 6402 } 6403 6404 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6405 bool enable) 6406 { 6407 u32 data, orig; 6408 6409 orig = data = RREG32(RLC_PG_CNTL); 6410 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6411 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6412 else 6413 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6414 if (orig != data) 6415 WREG32(RLC_PG_CNTL, data); 6416 } 6417 6418 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6419 { 6420 u32 data, orig; 6421 6422 orig = data = RREG32(RLC_PG_CNTL); 6423 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6424 data &= ~DISABLE_CP_PG; 6425 else 6426 data |= DISABLE_CP_PG; 6427 if (orig != data) 6428 WREG32(RLC_PG_CNTL, data); 6429 } 6430 6431 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6432 { 6433 u32 data, orig; 6434 6435 orig = data = RREG32(RLC_PG_CNTL); 6436 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6437 data &= ~DISABLE_GDS_PG; 6438 else 6439 data |= DISABLE_GDS_PG; 6440 if (orig != data) 6441 WREG32(RLC_PG_CNTL, data); 6442 } 6443 6444 #define CP_ME_TABLE_SIZE 96 6445 #define CP_ME_TABLE_OFFSET 2048 6446 #define CP_MEC_TABLE_OFFSET 4096 6447 6448 void cik_init_cp_pg_table(struct radeon_device *rdev) 6449 { 6450 volatile u32 *dst_ptr; 6451 int me, i, max_me = 4; 6452 u32 bo_offset = 0; 6453 u32 table_offset, table_size; 6454 6455 if (rdev->family == CHIP_KAVERI) 6456 max_me = 5; 6457 6458 if (rdev->rlc.cp_table_ptr == NULL) 6459 return; 6460 6461 /* write the cp table buffer */ 6462 dst_ptr = rdev->rlc.cp_table_ptr; 6463 for (me = 0; me < max_me; me++) { 6464 if (rdev->new_fw) { 6465 const __le32 *fw_data; 6466 const struct gfx_firmware_header_v1_0 *hdr; 6467 6468 if (me == 0) { 6469 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 6470 fw_data = (const __le32 *) 6471 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6472 table_offset = le32_to_cpu(hdr->jt_offset); 6473 table_size = le32_to_cpu(hdr->jt_size); 6474 } else if (me == 1) { 6475 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 6476 fw_data = (const __le32 *) 6477 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6478 table_offset = le32_to_cpu(hdr->jt_offset); 6479 table_size = le32_to_cpu(hdr->jt_size); 6480 } else if (me == 2) { 6481 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 6482 fw_data = (const __le32 *) 6483 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6484 table_offset = le32_to_cpu(hdr->jt_offset); 6485 table_size = le32_to_cpu(hdr->jt_size); 6486 } else if (me == 3) { 6487 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 6488 fw_data = (const __le32 *) 6489 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6490 table_offset = le32_to_cpu(hdr->jt_offset); 6491 table_size = le32_to_cpu(hdr->jt_size); 6492 } else { 6493 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 6494 fw_data = (const __le32 *) 6495 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6496 table_offset = le32_to_cpu(hdr->jt_offset); 6497 table_size = le32_to_cpu(hdr->jt_size); 6498 } 6499 6500 for (i = 0; i < table_size; i ++) { 6501 dst_ptr[bo_offset + i] = 6502 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); 6503 } 6504 bo_offset += table_size; 6505 } else { 6506 const __be32 *fw_data; 6507 table_size = CP_ME_TABLE_SIZE; 6508 6509 if (me == 0) { 6510 fw_data = (const __be32 *)rdev->ce_fw->data; 6511 table_offset = CP_ME_TABLE_OFFSET; 6512 } else if (me == 1) { 6513 fw_data = (const __be32 *)rdev->pfp_fw->data; 6514 table_offset = CP_ME_TABLE_OFFSET; 6515 } else if (me == 2) { 6516 fw_data = (const __be32 *)rdev->me_fw->data; 6517 table_offset = CP_ME_TABLE_OFFSET; 6518 } else { 6519 fw_data = (const __be32 *)rdev->mec_fw->data; 6520 table_offset = CP_MEC_TABLE_OFFSET; 6521 } 6522 6523 for (i = 0; i < table_size; i ++) { 6524 dst_ptr[bo_offset + i] = 6525 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6526 } 6527 bo_offset += table_size; 6528 } 6529 } 6530 } 6531 6532 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6533 bool enable) 6534 { 6535 u32 data, orig; 6536 6537 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6538 orig = data = RREG32(RLC_PG_CNTL); 6539 data |= GFX_PG_ENABLE; 6540 if (orig != data) 6541 WREG32(RLC_PG_CNTL, data); 6542 6543 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6544 data |= AUTO_PG_EN; 6545 if (orig != data) 6546 WREG32(RLC_AUTO_PG_CTRL, data); 6547 } else { 6548 orig = data = RREG32(RLC_PG_CNTL); 6549 data &= ~GFX_PG_ENABLE; 6550 if (orig != data) 6551 WREG32(RLC_PG_CNTL, data); 6552 6553 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6554 data &= ~AUTO_PG_EN; 6555 if (orig != data) 6556 WREG32(RLC_AUTO_PG_CTRL, data); 6557 6558 data = RREG32(DB_RENDER_CONTROL); 6559 } 6560 } 6561 6562 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6563 { 6564 u32 mask = 0, tmp, tmp1; 6565 int i; 6566 6567 cik_select_se_sh(rdev, se, sh); 6568 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6569 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6570 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6571 6572 tmp &= 0xffff0000; 6573 6574 tmp |= tmp1; 6575 tmp >>= 16; 6576 6577 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6578 mask <<= 1; 6579 mask |= 1; 6580 } 6581 6582 return (~tmp) & mask; 6583 } 6584 6585 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6586 { 6587 u32 i, j, k, active_cu_number = 0; 6588 u32 mask, counter, cu_bitmap; 6589 u32 tmp = 0; 6590 6591 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6592 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6593 mask = 1; 6594 cu_bitmap = 0; 6595 counter = 0; 6596 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6597 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6598 if (counter < 2) 6599 cu_bitmap |= mask; 6600 counter ++; 6601 } 6602 mask <<= 1; 6603 } 6604 6605 active_cu_number += counter; 6606 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6607 } 6608 } 6609 6610 WREG32(RLC_PG_AO_CU_MASK, tmp); 6611 6612 tmp = RREG32(RLC_MAX_PG_CU); 6613 tmp &= ~MAX_PU_CU_MASK; 6614 tmp |= MAX_PU_CU(active_cu_number); 6615 WREG32(RLC_MAX_PG_CU, tmp); 6616 } 6617 6618 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6619 bool enable) 6620 { 6621 u32 data, orig; 6622 6623 orig = data = RREG32(RLC_PG_CNTL); 6624 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6625 data |= STATIC_PER_CU_PG_ENABLE; 6626 else 6627 data &= ~STATIC_PER_CU_PG_ENABLE; 6628 if (orig != data) 6629 WREG32(RLC_PG_CNTL, data); 6630 } 6631 6632 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6633 bool enable) 6634 { 6635 u32 data, orig; 6636 6637 orig = data = RREG32(RLC_PG_CNTL); 6638 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6639 data |= DYN_PER_CU_PG_ENABLE; 6640 else 6641 data &= ~DYN_PER_CU_PG_ENABLE; 6642 if (orig != data) 6643 WREG32(RLC_PG_CNTL, data); 6644 } 6645 6646 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6647 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6648 6649 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6650 { 6651 u32 data, orig; 6652 u32 i; 6653 6654 if (rdev->rlc.cs_data) { 6655 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6656 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6657 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6658 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6659 } else { 6660 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6661 for (i = 0; i < 3; i++) 6662 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6663 } 6664 if (rdev->rlc.reg_list) { 6665 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6666 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6667 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6668 } 6669 6670 orig = data = RREG32(RLC_PG_CNTL); 6671 data |= GFX_PG_SRC; 6672 if (orig != data) 6673 WREG32(RLC_PG_CNTL, data); 6674 6675 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6676 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6677 6678 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6679 data &= ~IDLE_POLL_COUNT_MASK; 6680 data |= IDLE_POLL_COUNT(0x60); 6681 WREG32(CP_RB_WPTR_POLL_CNTL, data); 6682 6683 data = 0x10101010; 6684 WREG32(RLC_PG_DELAY, data); 6685 6686 data = RREG32(RLC_PG_DELAY_2); 6687 data &= ~0xff; 6688 data |= 0x3; 6689 WREG32(RLC_PG_DELAY_2, data); 6690 6691 data = RREG32(RLC_AUTO_PG_CTRL); 6692 data &= ~GRBM_REG_SGIT_MASK; 6693 data |= GRBM_REG_SGIT(0x700); 6694 WREG32(RLC_AUTO_PG_CTRL, data); 6695 6696 } 6697 6698 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 6699 { 6700 cik_enable_gfx_cgpg(rdev, enable); 6701 cik_enable_gfx_static_mgpg(rdev, enable); 6702 cik_enable_gfx_dynamic_mgpg(rdev, enable); 6703 } 6704 6705 u32 cik_get_csb_size(struct radeon_device *rdev) 6706 { 6707 u32 count = 0; 6708 const struct cs_section_def *sect = NULL; 6709 const struct cs_extent_def *ext = NULL; 6710 6711 if (rdev->rlc.cs_data == NULL) 6712 return 0; 6713 6714 /* begin clear state */ 6715 count += 2; 6716 /* context control state */ 6717 count += 3; 6718 6719 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6720 for (ext = sect->section; ext->extent != NULL; ++ext) { 6721 if (sect->id == SECT_CONTEXT) 6722 count += 2 + ext->reg_count; 6723 else 6724 return 0; 6725 } 6726 } 6727 /* pa_sc_raster_config/pa_sc_raster_config1 */ 6728 count += 4; 6729 /* end clear state */ 6730 count += 2; 6731 /* clear state */ 6732 count += 2; 6733 6734 return count; 6735 } 6736 6737 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 6738 { 6739 u32 count = 0, i; 6740 const struct cs_section_def *sect = NULL; 6741 const struct cs_extent_def *ext = NULL; 6742 6743 if (rdev->rlc.cs_data == NULL) 6744 return; 6745 if (buffer == NULL) 6746 return; 6747 6748 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6749 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 6750 6751 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6752 buffer[count++] = cpu_to_le32(0x80000000); 6753 buffer[count++] = cpu_to_le32(0x80000000); 6754 6755 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6756 for (ext = sect->section; ext->extent != NULL; ++ext) { 6757 if (sect->id == SECT_CONTEXT) { 6758 buffer[count++] = 6759 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 6760 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 6761 for (i = 0; i < ext->reg_count; i++) 6762 buffer[count++] = cpu_to_le32(ext->extent[i]); 6763 } else { 6764 return; 6765 } 6766 } 6767 } 6768 6769 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 6770 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 6771 switch (rdev->family) { 6772 case CHIP_BONAIRE: 6773 buffer[count++] = cpu_to_le32(0x16000012); 6774 buffer[count++] = cpu_to_le32(0x00000000); 6775 break; 6776 case CHIP_KAVERI: 6777 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6778 buffer[count++] = cpu_to_le32(0x00000000); 6779 break; 6780 case CHIP_KABINI: 6781 case CHIP_MULLINS: 6782 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6783 buffer[count++] = cpu_to_le32(0x00000000); 6784 break; 6785 case CHIP_HAWAII: 6786 buffer[count++] = cpu_to_le32(0x3a00161a); 6787 buffer[count++] = cpu_to_le32(0x0000002e); 6788 break; 6789 default: 6790 buffer[count++] = cpu_to_le32(0x00000000); 6791 buffer[count++] = cpu_to_le32(0x00000000); 6792 break; 6793 } 6794 6795 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6796 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 6797 6798 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 6799 buffer[count++] = cpu_to_le32(0); 6800 } 6801 6802 static void cik_init_pg(struct radeon_device *rdev) 6803 { 6804 if (rdev->pg_flags) { 6805 cik_enable_sck_slowdown_on_pu(rdev, true); 6806 cik_enable_sck_slowdown_on_pd(rdev, true); 6807 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 6808 cik_init_gfx_cgpg(rdev); 6809 cik_enable_cp_pg(rdev, true); 6810 cik_enable_gds_pg(rdev, true); 6811 } 6812 cik_init_ao_cu_mask(rdev); 6813 cik_update_gfx_pg(rdev, true); 6814 } 6815 } 6816 6817 static void cik_fini_pg(struct radeon_device *rdev) 6818 { 6819 if (rdev->pg_flags) { 6820 cik_update_gfx_pg(rdev, false); 6821 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 6822 cik_enable_cp_pg(rdev, false); 6823 cik_enable_gds_pg(rdev, false); 6824 } 6825 } 6826 } 6827 6828 /* 6829 * Interrupts 6830 * Starting with r6xx, interrupts are handled via a ring buffer. 6831 * Ring buffers are areas of GPU accessible memory that the GPU 6832 * writes interrupt vectors into and the host reads vectors out of. 6833 * There is a rptr (read pointer) that determines where the 6834 * host is currently reading, and a wptr (write pointer) 6835 * which determines where the GPU has written. When the 6836 * pointers are equal, the ring is idle. When the GPU 6837 * writes vectors to the ring buffer, it increments the 6838 * wptr. When there is an interrupt, the host then starts 6839 * fetching commands and processing them until the pointers are 6840 * equal again at which point it updates the rptr. 6841 */ 6842 6843 /** 6844 * cik_enable_interrupts - Enable the interrupt ring buffer 6845 * 6846 * @rdev: radeon_device pointer 6847 * 6848 * Enable the interrupt ring buffer (CIK). 6849 */ 6850 static void cik_enable_interrupts(struct radeon_device *rdev) 6851 { 6852 u32 ih_cntl = RREG32(IH_CNTL); 6853 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 6854 6855 ih_cntl |= ENABLE_INTR; 6856 ih_rb_cntl |= IH_RB_ENABLE; 6857 WREG32(IH_CNTL, ih_cntl); 6858 WREG32(IH_RB_CNTL, ih_rb_cntl); 6859 rdev->ih.enabled = true; 6860 } 6861 6862 /** 6863 * cik_disable_interrupts - Disable the interrupt ring buffer 6864 * 6865 * @rdev: radeon_device pointer 6866 * 6867 * Disable the interrupt ring buffer (CIK). 6868 */ 6869 static void cik_disable_interrupts(struct radeon_device *rdev) 6870 { 6871 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 6872 u32 ih_cntl = RREG32(IH_CNTL); 6873 6874 ih_rb_cntl &= ~IH_RB_ENABLE; 6875 ih_cntl &= ~ENABLE_INTR; 6876 WREG32(IH_RB_CNTL, ih_rb_cntl); 6877 WREG32(IH_CNTL, ih_cntl); 6878 /* set rptr, wptr to 0 */ 6879 WREG32(IH_RB_RPTR, 0); 6880 WREG32(IH_RB_WPTR, 0); 6881 rdev->ih.enabled = false; 6882 rdev->ih.rptr = 0; 6883 } 6884 6885 /** 6886 * cik_disable_interrupt_state - Disable all interrupt sources 6887 * 6888 * @rdev: radeon_device pointer 6889 * 6890 * Clear all interrupt enable bits used by the driver (CIK). 6891 */ 6892 static void cik_disable_interrupt_state(struct radeon_device *rdev) 6893 { 6894 u32 tmp; 6895 6896 /* gfx ring */ 6897 tmp = RREG32(CP_INT_CNTL_RING0) & 6898 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6899 WREG32(CP_INT_CNTL_RING0, tmp); 6900 /* sdma */ 6901 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 6902 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 6903 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 6904 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 6905 /* compute queues */ 6906 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 6907 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 6908 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 6909 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 6910 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 6911 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 6912 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 6913 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 6914 /* grbm */ 6915 WREG32(GRBM_INT_CNTL, 0); 6916 /* SRBM */ 6917 WREG32(SRBM_INT_CNTL, 0); 6918 /* vline/vblank, etc. */ 6919 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 6920 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 6921 if (rdev->num_crtc >= 4) { 6922 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 6923 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 6924 } 6925 if (rdev->num_crtc >= 6) { 6926 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 6927 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 6928 } 6929 /* pflip */ 6930 if (rdev->num_crtc >= 2) { 6931 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 6932 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 6933 } 6934 if (rdev->num_crtc >= 4) { 6935 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 6936 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 6937 } 6938 if (rdev->num_crtc >= 6) { 6939 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 6940 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 6941 } 6942 6943 /* dac hotplug */ 6944 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 6945 6946 /* digital hotplug */ 6947 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6948 WREG32(DC_HPD1_INT_CONTROL, tmp); 6949 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6950 WREG32(DC_HPD2_INT_CONTROL, tmp); 6951 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6952 WREG32(DC_HPD3_INT_CONTROL, tmp); 6953 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6954 WREG32(DC_HPD4_INT_CONTROL, tmp); 6955 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6956 WREG32(DC_HPD5_INT_CONTROL, tmp); 6957 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6958 WREG32(DC_HPD6_INT_CONTROL, tmp); 6959 6960 } 6961 6962 /** 6963 * cik_irq_init - init and enable the interrupt ring 6964 * 6965 * @rdev: radeon_device pointer 6966 * 6967 * Allocate a ring buffer for the interrupt controller, 6968 * enable the RLC, disable interrupts, enable the IH 6969 * ring buffer and enable it (CIK). 6970 * Called at device load and reume. 6971 * Returns 0 for success, errors for failure. 6972 */ 6973 static int cik_irq_init(struct radeon_device *rdev) 6974 { 6975 int ret = 0; 6976 int rb_bufsz; 6977 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 6978 6979 /* allocate ring */ 6980 ret = r600_ih_ring_alloc(rdev); 6981 if (ret) 6982 return ret; 6983 6984 /* disable irqs */ 6985 cik_disable_interrupts(rdev); 6986 6987 /* init rlc */ 6988 ret = cik_rlc_resume(rdev); 6989 if (ret) { 6990 r600_ih_ring_fini(rdev); 6991 return ret; 6992 } 6993 6994 /* setup interrupt control */ 6995 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 6996 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 6997 interrupt_cntl = RREG32(INTERRUPT_CNTL); 6998 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 6999 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 7000 */ 7001 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 7002 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 7003 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 7004 WREG32(INTERRUPT_CNTL, interrupt_cntl); 7005 7006 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 7007 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 7008 7009 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 7010 IH_WPTR_OVERFLOW_CLEAR | 7011 (rb_bufsz << 1)); 7012 7013 if (rdev->wb.enabled) 7014 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 7015 7016 /* set the writeback address whether it's enabled or not */ 7017 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 7018 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 7019 7020 WREG32(IH_RB_CNTL, ih_rb_cntl); 7021 7022 /* set rptr, wptr to 0 */ 7023 WREG32(IH_RB_RPTR, 0); 7024 WREG32(IH_RB_WPTR, 0); 7025 7026 /* Default settings for IH_CNTL (disabled at first) */ 7027 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 7028 /* RPTR_REARM only works if msi's are enabled */ 7029 if (rdev->msi_enabled) 7030 ih_cntl |= RPTR_REARM; 7031 WREG32(IH_CNTL, ih_cntl); 7032 7033 /* force the active interrupt state to all disabled */ 7034 cik_disable_interrupt_state(rdev); 7035 7036 pci_set_master(rdev->pdev); 7037 7038 /* enable irqs */ 7039 cik_enable_interrupts(rdev); 7040 7041 return ret; 7042 } 7043 7044 /** 7045 * cik_irq_set - enable/disable interrupt sources 7046 * 7047 * @rdev: radeon_device pointer 7048 * 7049 * Enable interrupt sources on the GPU (vblanks, hpd, 7050 * etc.) (CIK). 7051 * Returns 0 for success, errors for failure. 7052 */ 7053 int cik_irq_set(struct radeon_device *rdev) 7054 { 7055 u32 cp_int_cntl; 7056 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; 7057 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; 7058 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 7059 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 7060 u32 grbm_int_cntl = 0; 7061 u32 dma_cntl, dma_cntl1; 7062 7063 if (!rdev->irq.installed) { 7064 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 7065 return -EINVAL; 7066 } 7067 /* don't enable anything if the ih is disabled */ 7068 if (!rdev->ih.enabled) { 7069 cik_disable_interrupts(rdev); 7070 /* force the active interrupt state to all disabled */ 7071 cik_disable_interrupt_state(rdev); 7072 return 0; 7073 } 7074 7075 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 7076 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7077 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 7078 7079 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7080 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7081 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7082 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7083 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7084 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 7085 7086 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7087 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7088 7089 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7090 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7091 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7092 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7093 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7094 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7095 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7096 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7097 7098 /* enable CP interrupts on all rings */ 7099 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 7100 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 7101 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 7102 } 7103 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 7104 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7105 DRM_DEBUG("si_irq_set: sw int cp1\n"); 7106 if (ring->me == 1) { 7107 switch (ring->pipe) { 7108 case 0: 7109 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7110 break; 7111 case 1: 7112 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 7113 break; 7114 case 2: 7115 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7116 break; 7117 case 3: 7118 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7119 break; 7120 default: 7121 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7122 break; 7123 } 7124 } else if (ring->me == 2) { 7125 switch (ring->pipe) { 7126 case 0: 7127 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 7128 break; 7129 case 1: 7130 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 7131 break; 7132 case 2: 7133 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7134 break; 7135 case 3: 7136 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7137 break; 7138 default: 7139 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7140 break; 7141 } 7142 } else { 7143 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 7144 } 7145 } 7146 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 7147 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7148 DRM_DEBUG("si_irq_set: sw int cp2\n"); 7149 if (ring->me == 1) { 7150 switch (ring->pipe) { 7151 case 0: 7152 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7153 break; 7154 case 1: 7155 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 7156 break; 7157 case 2: 7158 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7159 break; 7160 case 3: 7161 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 7162 break; 7163 default: 7164 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7165 break; 7166 } 7167 } else if (ring->me == 2) { 7168 switch (ring->pipe) { 7169 case 0: 7170 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 7171 break; 7172 case 1: 7173 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 7174 break; 7175 case 2: 7176 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7177 break; 7178 case 3: 7179 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 7180 break; 7181 default: 7182 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7183 break; 7184 } 7185 } else { 7186 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 7187 } 7188 } 7189 7190 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 7191 DRM_DEBUG("cik_irq_set: sw int dma\n"); 7192 dma_cntl |= TRAP_ENABLE; 7193 } 7194 7195 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 7196 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 7197 dma_cntl1 |= TRAP_ENABLE; 7198 } 7199 7200 if (rdev->irq.crtc_vblank_int[0] || 7201 atomic_read(&rdev->irq.pflip[0])) { 7202 DRM_DEBUG("cik_irq_set: vblank 0\n"); 7203 crtc1 |= VBLANK_INTERRUPT_MASK; 7204 } 7205 if (rdev->irq.crtc_vblank_int[1] || 7206 atomic_read(&rdev->irq.pflip[1])) { 7207 DRM_DEBUG("cik_irq_set: vblank 1\n"); 7208 crtc2 |= VBLANK_INTERRUPT_MASK; 7209 } 7210 if (rdev->irq.crtc_vblank_int[2] || 7211 atomic_read(&rdev->irq.pflip[2])) { 7212 DRM_DEBUG("cik_irq_set: vblank 2\n"); 7213 crtc3 |= VBLANK_INTERRUPT_MASK; 7214 } 7215 if (rdev->irq.crtc_vblank_int[3] || 7216 atomic_read(&rdev->irq.pflip[3])) { 7217 DRM_DEBUG("cik_irq_set: vblank 3\n"); 7218 crtc4 |= VBLANK_INTERRUPT_MASK; 7219 } 7220 if (rdev->irq.crtc_vblank_int[4] || 7221 atomic_read(&rdev->irq.pflip[4])) { 7222 DRM_DEBUG("cik_irq_set: vblank 4\n"); 7223 crtc5 |= VBLANK_INTERRUPT_MASK; 7224 } 7225 if (rdev->irq.crtc_vblank_int[5] || 7226 atomic_read(&rdev->irq.pflip[5])) { 7227 DRM_DEBUG("cik_irq_set: vblank 5\n"); 7228 crtc6 |= VBLANK_INTERRUPT_MASK; 7229 } 7230 if (rdev->irq.hpd[0]) { 7231 DRM_DEBUG("cik_irq_set: hpd 1\n"); 7232 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7233 } 7234 if (rdev->irq.hpd[1]) { 7235 DRM_DEBUG("cik_irq_set: hpd 2\n"); 7236 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7237 } 7238 if (rdev->irq.hpd[2]) { 7239 DRM_DEBUG("cik_irq_set: hpd 3\n"); 7240 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7241 } 7242 if (rdev->irq.hpd[3]) { 7243 DRM_DEBUG("cik_irq_set: hpd 4\n"); 7244 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7245 } 7246 if (rdev->irq.hpd[4]) { 7247 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7248 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7249 } 7250 if (rdev->irq.hpd[5]) { 7251 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7252 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 7253 } 7254 7255 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7256 7257 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7258 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7259 7260 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7261 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); 7262 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); 7263 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); 7264 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); 7265 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); 7266 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); 7267 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3); 7268 7269 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7270 7271 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7272 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7273 if (rdev->num_crtc >= 4) { 7274 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7275 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7276 } 7277 if (rdev->num_crtc >= 6) { 7278 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7279 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7280 } 7281 7282 if (rdev->num_crtc >= 2) { 7283 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 7284 GRPH_PFLIP_INT_MASK); 7285 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 7286 GRPH_PFLIP_INT_MASK); 7287 } 7288 if (rdev->num_crtc >= 4) { 7289 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 7290 GRPH_PFLIP_INT_MASK); 7291 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 7292 GRPH_PFLIP_INT_MASK); 7293 } 7294 if (rdev->num_crtc >= 6) { 7295 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 7296 GRPH_PFLIP_INT_MASK); 7297 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 7298 GRPH_PFLIP_INT_MASK); 7299 } 7300 7301 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7302 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7303 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7304 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7305 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7306 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7307 7308 /* posting read */ 7309 RREG32(SRBM_STATUS); 7310 7311 return 0; 7312 } 7313 7314 /** 7315 * cik_irq_ack - ack interrupt sources 7316 * 7317 * @rdev: radeon_device pointer 7318 * 7319 * Ack interrupt sources on the GPU (vblanks, hpd, 7320 * etc.) (CIK). Certain interrupts sources are sw 7321 * generated and do not require an explicit ack. 7322 */ 7323 static inline void cik_irq_ack(struct radeon_device *rdev) 7324 { 7325 u32 tmp; 7326 7327 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7328 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7329 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7330 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7331 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7332 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7333 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7334 7335 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + 7336 EVERGREEN_CRTC0_REGISTER_OFFSET); 7337 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + 7338 EVERGREEN_CRTC1_REGISTER_OFFSET); 7339 if (rdev->num_crtc >= 4) { 7340 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + 7341 EVERGREEN_CRTC2_REGISTER_OFFSET); 7342 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + 7343 EVERGREEN_CRTC3_REGISTER_OFFSET); 7344 } 7345 if (rdev->num_crtc >= 6) { 7346 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + 7347 EVERGREEN_CRTC4_REGISTER_OFFSET); 7348 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + 7349 EVERGREEN_CRTC5_REGISTER_OFFSET); 7350 } 7351 7352 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 7353 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 7354 GRPH_PFLIP_INT_CLEAR); 7355 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 7356 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 7357 GRPH_PFLIP_INT_CLEAR); 7358 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7359 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7360 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7361 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7362 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7364 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7366 7367 if (rdev->num_crtc >= 4) { 7368 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 7369 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 7370 GRPH_PFLIP_INT_CLEAR); 7371 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 7372 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 7373 GRPH_PFLIP_INT_CLEAR); 7374 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7375 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7376 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7377 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7378 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7379 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7380 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7381 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7382 } 7383 7384 if (rdev->num_crtc >= 6) { 7385 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 7386 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 7387 GRPH_PFLIP_INT_CLEAR); 7388 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 7389 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, 7390 GRPH_PFLIP_INT_CLEAR); 7391 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7392 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7393 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7394 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7395 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7396 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7397 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7398 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7399 } 7400 7401 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7402 tmp = RREG32(DC_HPD1_INT_CONTROL); 7403 tmp |= DC_HPDx_INT_ACK; 7404 WREG32(DC_HPD1_INT_CONTROL, tmp); 7405 } 7406 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7407 tmp = RREG32(DC_HPD2_INT_CONTROL); 7408 tmp |= DC_HPDx_INT_ACK; 7409 WREG32(DC_HPD2_INT_CONTROL, tmp); 7410 } 7411 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7412 tmp = RREG32(DC_HPD3_INT_CONTROL); 7413 tmp |= DC_HPDx_INT_ACK; 7414 WREG32(DC_HPD3_INT_CONTROL, tmp); 7415 } 7416 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7417 tmp = RREG32(DC_HPD4_INT_CONTROL); 7418 tmp |= DC_HPDx_INT_ACK; 7419 WREG32(DC_HPD4_INT_CONTROL, tmp); 7420 } 7421 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7422 tmp = RREG32(DC_HPD5_INT_CONTROL); 7423 tmp |= DC_HPDx_INT_ACK; 7424 WREG32(DC_HPD5_INT_CONTROL, tmp); 7425 } 7426 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7427 tmp = RREG32(DC_HPD6_INT_CONTROL); 7428 tmp |= DC_HPDx_INT_ACK; 7429 WREG32(DC_HPD6_INT_CONTROL, tmp); 7430 } 7431 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) { 7432 tmp = RREG32(DC_HPD1_INT_CONTROL); 7433 tmp |= DC_HPDx_RX_INT_ACK; 7434 WREG32(DC_HPD1_INT_CONTROL, tmp); 7435 } 7436 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) { 7437 tmp = RREG32(DC_HPD2_INT_CONTROL); 7438 tmp |= DC_HPDx_RX_INT_ACK; 7439 WREG32(DC_HPD2_INT_CONTROL, tmp); 7440 } 7441 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { 7442 tmp = RREG32(DC_HPD3_INT_CONTROL); 7443 tmp |= DC_HPDx_RX_INT_ACK; 7444 WREG32(DC_HPD3_INT_CONTROL, tmp); 7445 } 7446 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { 7447 tmp = RREG32(DC_HPD4_INT_CONTROL); 7448 tmp |= DC_HPDx_RX_INT_ACK; 7449 WREG32(DC_HPD4_INT_CONTROL, tmp); 7450 } 7451 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { 7452 tmp = RREG32(DC_HPD5_INT_CONTROL); 7453 tmp |= DC_HPDx_RX_INT_ACK; 7454 WREG32(DC_HPD5_INT_CONTROL, tmp); 7455 } 7456 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { 7457 tmp = RREG32(DC_HPD6_INT_CONTROL); 7458 tmp |= DC_HPDx_RX_INT_ACK; 7459 WREG32(DC_HPD6_INT_CONTROL, tmp); 7460 } 7461 } 7462 7463 /** 7464 * cik_irq_disable - disable interrupts 7465 * 7466 * @rdev: radeon_device pointer 7467 * 7468 * Disable interrupts on the hw (CIK). 7469 */ 7470 static void cik_irq_disable(struct radeon_device *rdev) 7471 { 7472 cik_disable_interrupts(rdev); 7473 /* Wait and acknowledge irq */ 7474 mdelay(1); 7475 cik_irq_ack(rdev); 7476 cik_disable_interrupt_state(rdev); 7477 } 7478 7479 /** 7480 * cik_irq_disable - disable interrupts for suspend 7481 * 7482 * @rdev: radeon_device pointer 7483 * 7484 * Disable interrupts and stop the RLC (CIK). 7485 * Used for suspend. 7486 */ 7487 static void cik_irq_suspend(struct radeon_device *rdev) 7488 { 7489 cik_irq_disable(rdev); 7490 cik_rlc_stop(rdev); 7491 } 7492 7493 /** 7494 * cik_irq_fini - tear down interrupt support 7495 * 7496 * @rdev: radeon_device pointer 7497 * 7498 * Disable interrupts on the hw and free the IH ring 7499 * buffer (CIK). 7500 * Used for driver unload. 7501 */ 7502 static void cik_irq_fini(struct radeon_device *rdev) 7503 { 7504 cik_irq_suspend(rdev); 7505 r600_ih_ring_fini(rdev); 7506 } 7507 7508 /** 7509 * cik_get_ih_wptr - get the IH ring buffer wptr 7510 * 7511 * @rdev: radeon_device pointer 7512 * 7513 * Get the IH ring buffer wptr from either the register 7514 * or the writeback memory buffer (CIK). Also check for 7515 * ring buffer overflow and deal with it. 7516 * Used by cik_irq_process(). 7517 * Returns the value of the wptr. 7518 */ 7519 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7520 { 7521 u32 wptr, tmp; 7522 7523 if (rdev->wb.enabled) 7524 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7525 else 7526 wptr = RREG32(IH_RB_WPTR); 7527 7528 if (wptr & RB_OVERFLOW) { 7529 wptr &= ~RB_OVERFLOW; 7530 /* When a ring buffer overflow happen start parsing interrupt 7531 * from the last not overwritten vector (wptr + 16). Hopefully 7532 * this should allow us to catchup. 7533 */ 7534 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 7535 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 7536 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7537 tmp = RREG32(IH_RB_CNTL); 7538 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7539 WREG32(IH_RB_CNTL, tmp); 7540 } 7541 return (wptr & rdev->ih.ptr_mask); 7542 } 7543 7544 /* CIK IV Ring 7545 * Each IV ring entry is 128 bits: 7546 * [7:0] - interrupt source id 7547 * [31:8] - reserved 7548 * [59:32] - interrupt source data 7549 * [63:60] - reserved 7550 * [71:64] - RINGID 7551 * CP: 7552 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7553 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7554 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7555 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7556 * PIPE_ID - ME0 0=3D 7557 * - ME1&2 compute dispatcher (4 pipes each) 7558 * SDMA: 7559 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7560 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7561 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7562 * [79:72] - VMID 7563 * [95:80] - PASID 7564 * [127:96] - reserved 7565 */ 7566 /** 7567 * cik_irq_process - interrupt handler 7568 * 7569 * @rdev: radeon_device pointer 7570 * 7571 * Interrupt hander (CIK). Walk the IH ring, 7572 * ack interrupts and schedule work to handle 7573 * interrupt events. 7574 * Returns irq process return code. 7575 */ 7576 int cik_irq_process(struct radeon_device *rdev) 7577 { 7578 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7579 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7580 u32 wptr; 7581 u32 rptr; 7582 u32 src_id, src_data, ring_id; 7583 u8 me_id, pipe_id, queue_id; 7584 u32 ring_index; 7585 bool queue_hotplug = false; 7586 bool queue_dp = false; 7587 bool queue_reset = false; 7588 u32 addr, status, mc_client; 7589 bool queue_thermal = false; 7590 7591 if (!rdev->ih.enabled || rdev->shutdown) 7592 return IRQ_NONE; 7593 7594 wptr = cik_get_ih_wptr(rdev); 7595 7596 restart_ih: 7597 /* is somebody else already processing irqs? */ 7598 if (atomic_xchg(&rdev->ih.lock, 1)) 7599 return IRQ_NONE; 7600 7601 rptr = rdev->ih.rptr; 7602 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7603 7604 /* Order reading of wptr vs. reading of IH ring data */ 7605 rmb(); 7606 7607 /* display interrupts */ 7608 cik_irq_ack(rdev); 7609 7610 while (rptr != wptr) { 7611 /* wptr/rptr are in bytes! */ 7612 ring_index = rptr / 4; 7613 7614 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7615 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7616 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7617 7618 switch (src_id) { 7619 case 1: /* D1 vblank/vline */ 7620 switch (src_data) { 7621 case 0: /* D1 vblank */ 7622 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)) 7623 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7624 7625 if (rdev->irq.crtc_vblank_int[0]) { 7626 drm_handle_vblank(rdev->ddev, 0); 7627 rdev->pm.vblank_sync = true; 7628 wake_up(&rdev->irq.vblank_queue); 7629 } 7630 if (atomic_read(&rdev->irq.pflip[0])) 7631 radeon_crtc_handle_vblank(rdev, 0); 7632 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7633 DRM_DEBUG("IH: D1 vblank\n"); 7634 7635 break; 7636 case 1: /* D1 vline */ 7637 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)) 7638 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7639 7640 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7641 DRM_DEBUG("IH: D1 vline\n"); 7642 7643 break; 7644 default: 7645 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7646 break; 7647 } 7648 break; 7649 case 2: /* D2 vblank/vline */ 7650 switch (src_data) { 7651 case 0: /* D2 vblank */ 7652 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) 7653 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7654 7655 if (rdev->irq.crtc_vblank_int[1]) { 7656 drm_handle_vblank(rdev->ddev, 1); 7657 rdev->pm.vblank_sync = true; 7658 wake_up(&rdev->irq.vblank_queue); 7659 } 7660 if (atomic_read(&rdev->irq.pflip[1])) 7661 radeon_crtc_handle_vblank(rdev, 1); 7662 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7663 DRM_DEBUG("IH: D2 vblank\n"); 7664 7665 break; 7666 case 1: /* D2 vline */ 7667 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)) 7668 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7669 7670 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7671 DRM_DEBUG("IH: D2 vline\n"); 7672 7673 break; 7674 default: 7675 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7676 break; 7677 } 7678 break; 7679 case 3: /* D3 vblank/vline */ 7680 switch (src_data) { 7681 case 0: /* D3 vblank */ 7682 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) 7683 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7684 7685 if (rdev->irq.crtc_vblank_int[2]) { 7686 drm_handle_vblank(rdev->ddev, 2); 7687 rdev->pm.vblank_sync = true; 7688 wake_up(&rdev->irq.vblank_queue); 7689 } 7690 if (atomic_read(&rdev->irq.pflip[2])) 7691 radeon_crtc_handle_vblank(rdev, 2); 7692 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7693 DRM_DEBUG("IH: D3 vblank\n"); 7694 7695 break; 7696 case 1: /* D3 vline */ 7697 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) 7698 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7699 7700 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7701 DRM_DEBUG("IH: D3 vline\n"); 7702 7703 break; 7704 default: 7705 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7706 break; 7707 } 7708 break; 7709 case 4: /* D4 vblank/vline */ 7710 switch (src_data) { 7711 case 0: /* D4 vblank */ 7712 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) 7713 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7714 7715 if (rdev->irq.crtc_vblank_int[3]) { 7716 drm_handle_vblank(rdev->ddev, 3); 7717 rdev->pm.vblank_sync = true; 7718 wake_up(&rdev->irq.vblank_queue); 7719 } 7720 if (atomic_read(&rdev->irq.pflip[3])) 7721 radeon_crtc_handle_vblank(rdev, 3); 7722 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7723 DRM_DEBUG("IH: D4 vblank\n"); 7724 7725 break; 7726 case 1: /* D4 vline */ 7727 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) 7728 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7729 7730 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7731 DRM_DEBUG("IH: D4 vline\n"); 7732 7733 break; 7734 default: 7735 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7736 break; 7737 } 7738 break; 7739 case 5: /* D5 vblank/vline */ 7740 switch (src_data) { 7741 case 0: /* D5 vblank */ 7742 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) 7743 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7744 7745 if (rdev->irq.crtc_vblank_int[4]) { 7746 drm_handle_vblank(rdev->ddev, 4); 7747 rdev->pm.vblank_sync = true; 7748 wake_up(&rdev->irq.vblank_queue); 7749 } 7750 if (atomic_read(&rdev->irq.pflip[4])) 7751 radeon_crtc_handle_vblank(rdev, 4); 7752 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7753 DRM_DEBUG("IH: D5 vblank\n"); 7754 7755 break; 7756 case 1: /* D5 vline */ 7757 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) 7758 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7759 7760 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7761 DRM_DEBUG("IH: D5 vline\n"); 7762 7763 break; 7764 default: 7765 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7766 break; 7767 } 7768 break; 7769 case 6: /* D6 vblank/vline */ 7770 switch (src_data) { 7771 case 0: /* D6 vblank */ 7772 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) 7773 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7774 7775 if (rdev->irq.crtc_vblank_int[5]) { 7776 drm_handle_vblank(rdev->ddev, 5); 7777 rdev->pm.vblank_sync = true; 7778 wake_up(&rdev->irq.vblank_queue); 7779 } 7780 if (atomic_read(&rdev->irq.pflip[5])) 7781 radeon_crtc_handle_vblank(rdev, 5); 7782 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7783 DRM_DEBUG("IH: D6 vblank\n"); 7784 7785 break; 7786 case 1: /* D6 vline */ 7787 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) 7788 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7789 7790 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7791 DRM_DEBUG("IH: D6 vline\n"); 7792 7793 break; 7794 default: 7795 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7796 break; 7797 } 7798 break; 7799 case 8: /* D1 page flip */ 7800 case 10: /* D2 page flip */ 7801 case 12: /* D3 page flip */ 7802 case 14: /* D4 page flip */ 7803 case 16: /* D5 page flip */ 7804 case 18: /* D6 page flip */ 7805 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 7806 if (radeon_use_pflipirq > 0) 7807 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 7808 break; 7809 case 42: /* HPD hotplug */ 7810 switch (src_data) { 7811 case 0: 7812 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT)) 7813 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7814 7815 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 7816 queue_hotplug = true; 7817 DRM_DEBUG("IH: HPD1\n"); 7818 7819 break; 7820 case 1: 7821 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT)) 7822 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7823 7824 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 7825 queue_hotplug = true; 7826 DRM_DEBUG("IH: HPD2\n"); 7827 7828 break; 7829 case 2: 7830 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT)) 7831 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7832 7833 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 7834 queue_hotplug = true; 7835 DRM_DEBUG("IH: HPD3\n"); 7836 7837 break; 7838 case 3: 7839 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT)) 7840 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7841 7842 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 7843 queue_hotplug = true; 7844 DRM_DEBUG("IH: HPD4\n"); 7845 7846 break; 7847 case 4: 7848 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT)) 7849 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7850 7851 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 7852 queue_hotplug = true; 7853 DRM_DEBUG("IH: HPD5\n"); 7854 7855 break; 7856 case 5: 7857 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT)) 7858 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7859 7860 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 7861 queue_hotplug = true; 7862 DRM_DEBUG("IH: HPD6\n"); 7863 7864 break; 7865 case 6: 7866 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT)) 7867 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7868 7869 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT; 7870 queue_dp = true; 7871 DRM_DEBUG("IH: HPD_RX 1\n"); 7872 7873 break; 7874 case 7: 7875 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT)) 7876 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7877 7878 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; 7879 queue_dp = true; 7880 DRM_DEBUG("IH: HPD_RX 2\n"); 7881 7882 break; 7883 case 8: 7884 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) 7885 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7886 7887 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; 7888 queue_dp = true; 7889 DRM_DEBUG("IH: HPD_RX 3\n"); 7890 7891 break; 7892 case 9: 7893 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) 7894 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7895 7896 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; 7897 queue_dp = true; 7898 DRM_DEBUG("IH: HPD_RX 4\n"); 7899 7900 break; 7901 case 10: 7902 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) 7903 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7904 7905 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; 7906 queue_dp = true; 7907 DRM_DEBUG("IH: HPD_RX 5\n"); 7908 7909 break; 7910 case 11: 7911 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) 7912 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 7913 7914 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; 7915 queue_dp = true; 7916 DRM_DEBUG("IH: HPD_RX 6\n"); 7917 7918 break; 7919 default: 7920 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7921 break; 7922 } 7923 break; 7924 case 96: 7925 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 7926 WREG32(SRBM_INT_ACK, 0x1); 7927 break; 7928 case 124: /* UVD */ 7929 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 7930 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 7931 break; 7932 case 146: 7933 case 147: 7934 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 7935 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 7936 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 7937 /* reset addr and status */ 7938 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 7939 if (addr == 0x0 && status == 0x0) 7940 break; 7941 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 7942 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 7943 addr); 7944 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 7945 status); 7946 cik_vm_decode_fault(rdev, status, addr, mc_client); 7947 break; 7948 case 167: /* VCE */ 7949 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 7950 switch (src_data) { 7951 case 0: 7952 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 7953 break; 7954 case 1: 7955 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 7956 break; 7957 default: 7958 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 7959 break; 7960 } 7961 break; 7962 case 176: /* GFX RB CP_INT */ 7963 case 177: /* GFX IB CP_INT */ 7964 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 7965 break; 7966 case 181: /* CP EOP event */ 7967 DRM_DEBUG("IH: CP EOP\n"); 7968 /* XXX check the bitfield order! */ 7969 me_id = (ring_id & 0x60) >> 5; 7970 pipe_id = (ring_id & 0x18) >> 3; 7971 queue_id = (ring_id & 0x7) >> 0; 7972 switch (me_id) { 7973 case 0: 7974 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 7975 break; 7976 case 1: 7977 case 2: 7978 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 7979 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 7980 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 7981 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 7982 break; 7983 } 7984 break; 7985 case 184: /* CP Privileged reg access */ 7986 DRM_ERROR("Illegal register access in command stream\n"); 7987 /* XXX check the bitfield order! */ 7988 me_id = (ring_id & 0x60) >> 5; 7989 pipe_id = (ring_id & 0x18) >> 3; 7990 queue_id = (ring_id & 0x7) >> 0; 7991 switch (me_id) { 7992 case 0: 7993 /* This results in a full GPU reset, but all we need to do is soft 7994 * reset the CP for gfx 7995 */ 7996 queue_reset = true; 7997 break; 7998 case 1: 7999 /* XXX compute */ 8000 queue_reset = true; 8001 break; 8002 case 2: 8003 /* XXX compute */ 8004 queue_reset = true; 8005 break; 8006 } 8007 break; 8008 case 185: /* CP Privileged inst */ 8009 DRM_ERROR("Illegal instruction in command stream\n"); 8010 /* XXX check the bitfield order! */ 8011 me_id = (ring_id & 0x60) >> 5; 8012 pipe_id = (ring_id & 0x18) >> 3; 8013 queue_id = (ring_id & 0x7) >> 0; 8014 switch (me_id) { 8015 case 0: 8016 /* This results in a full GPU reset, but all we need to do is soft 8017 * reset the CP for gfx 8018 */ 8019 queue_reset = true; 8020 break; 8021 case 1: 8022 /* XXX compute */ 8023 queue_reset = true; 8024 break; 8025 case 2: 8026 /* XXX compute */ 8027 queue_reset = true; 8028 break; 8029 } 8030 break; 8031 case 224: /* SDMA trap event */ 8032 /* XXX check the bitfield order! */ 8033 me_id = (ring_id & 0x3) >> 0; 8034 queue_id = (ring_id & 0xc) >> 2; 8035 DRM_DEBUG("IH: SDMA trap\n"); 8036 switch (me_id) { 8037 case 0: 8038 switch (queue_id) { 8039 case 0: 8040 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 8041 break; 8042 case 1: 8043 /* XXX compute */ 8044 break; 8045 case 2: 8046 /* XXX compute */ 8047 break; 8048 } 8049 break; 8050 case 1: 8051 switch (queue_id) { 8052 case 0: 8053 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8054 break; 8055 case 1: 8056 /* XXX compute */ 8057 break; 8058 case 2: 8059 /* XXX compute */ 8060 break; 8061 } 8062 break; 8063 } 8064 break; 8065 case 230: /* thermal low to high */ 8066 DRM_DEBUG("IH: thermal low to high\n"); 8067 rdev->pm.dpm.thermal.high_to_low = false; 8068 queue_thermal = true; 8069 break; 8070 case 231: /* thermal high to low */ 8071 DRM_DEBUG("IH: thermal high to low\n"); 8072 rdev->pm.dpm.thermal.high_to_low = true; 8073 queue_thermal = true; 8074 break; 8075 case 233: /* GUI IDLE */ 8076 DRM_DEBUG("IH: GUI idle\n"); 8077 break; 8078 case 241: /* SDMA Privileged inst */ 8079 case 247: /* SDMA Privileged inst */ 8080 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 8081 /* XXX check the bitfield order! */ 8082 me_id = (ring_id & 0x3) >> 0; 8083 queue_id = (ring_id & 0xc) >> 2; 8084 switch (me_id) { 8085 case 0: 8086 switch (queue_id) { 8087 case 0: 8088 queue_reset = true; 8089 break; 8090 case 1: 8091 /* XXX compute */ 8092 queue_reset = true; 8093 break; 8094 case 2: 8095 /* XXX compute */ 8096 queue_reset = true; 8097 break; 8098 } 8099 break; 8100 case 1: 8101 switch (queue_id) { 8102 case 0: 8103 queue_reset = true; 8104 break; 8105 case 1: 8106 /* XXX compute */ 8107 queue_reset = true; 8108 break; 8109 case 2: 8110 /* XXX compute */ 8111 queue_reset = true; 8112 break; 8113 } 8114 break; 8115 } 8116 break; 8117 default: 8118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8119 break; 8120 } 8121 8122 /* wptr/rptr are in bytes! */ 8123 rptr += 16; 8124 rptr &= rdev->ih.ptr_mask; 8125 WREG32(IH_RB_RPTR, rptr); 8126 } 8127 if (queue_dp) 8128 schedule_work(&rdev->dp_work); 8129 if (queue_hotplug) 8130 schedule_delayed_work(&rdev->hotplug_work, 0); 8131 if (queue_reset) { 8132 rdev->needs_reset = true; 8133 wake_up_all(&rdev->fence_queue); 8134 } 8135 if (queue_thermal) 8136 schedule_work(&rdev->pm.dpm.thermal.work); 8137 rdev->ih.rptr = rptr; 8138 atomic_set(&rdev->ih.lock, 0); 8139 8140 /* make sure wptr hasn't changed while processing */ 8141 wptr = cik_get_ih_wptr(rdev); 8142 if (wptr != rptr) 8143 goto restart_ih; 8144 8145 return IRQ_HANDLED; 8146 } 8147 8148 /* 8149 * startup/shutdown callbacks 8150 */ 8151 static void cik_uvd_init(struct radeon_device *rdev) 8152 { 8153 int r; 8154 8155 if (!rdev->has_uvd) 8156 return; 8157 8158 r = radeon_uvd_init(rdev); 8159 if (r) { 8160 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); 8161 /* 8162 * At this point rdev->uvd.vcpu_bo is NULL which trickles down 8163 * to early fails cik_uvd_start() and thus nothing happens 8164 * there. So it is pointless to try to go through that code 8165 * hence why we disable uvd here. 8166 */ 8167 rdev->has_uvd = 0; 8168 return; 8169 } 8170 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 8171 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); 8172 } 8173 8174 static void cik_uvd_start(struct radeon_device *rdev) 8175 { 8176 int r; 8177 8178 if (!rdev->has_uvd) 8179 return; 8180 8181 r = radeon_uvd_resume(rdev); 8182 if (r) { 8183 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); 8184 goto error; 8185 } 8186 r = uvd_v4_2_resume(rdev); 8187 if (r) { 8188 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r); 8189 goto error; 8190 } 8191 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); 8192 if (r) { 8193 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); 8194 goto error; 8195 } 8196 return; 8197 8198 error: 8199 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 8200 } 8201 8202 static void cik_uvd_resume(struct radeon_device *rdev) 8203 { 8204 struct radeon_ring *ring; 8205 int r; 8206 8207 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) 8208 return; 8209 8210 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8211 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0)); 8212 if (r) { 8213 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); 8214 return; 8215 } 8216 r = uvd_v1_0_init(rdev); 8217 if (r) { 8218 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); 8219 return; 8220 } 8221 } 8222 8223 static void cik_vce_init(struct radeon_device *rdev) 8224 { 8225 int r; 8226 8227 if (!rdev->has_vce) 8228 return; 8229 8230 r = radeon_vce_init(rdev); 8231 if (r) { 8232 dev_err(rdev->dev, "failed VCE (%d) init.\n", r); 8233 /* 8234 * At this point rdev->vce.vcpu_bo is NULL which trickles down 8235 * to early fails cik_vce_start() and thus nothing happens 8236 * there. So it is pointless to try to go through that code 8237 * hence why we disable vce here. 8238 */ 8239 rdev->has_vce = 0; 8240 return; 8241 } 8242 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL; 8243 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096); 8244 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL; 8245 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096); 8246 } 8247 8248 static void cik_vce_start(struct radeon_device *rdev) 8249 { 8250 int r; 8251 8252 if (!rdev->has_vce) 8253 return; 8254 8255 r = radeon_vce_resume(rdev); 8256 if (r) { 8257 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); 8258 goto error; 8259 } 8260 r = vce_v2_0_resume(rdev); 8261 if (r) { 8262 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); 8263 goto error; 8264 } 8265 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX); 8266 if (r) { 8267 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r); 8268 goto error; 8269 } 8270 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX); 8271 if (r) { 8272 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r); 8273 goto error; 8274 } 8275 return; 8276 8277 error: 8278 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 8279 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 8280 } 8281 8282 static void cik_vce_resume(struct radeon_device *rdev) 8283 { 8284 struct radeon_ring *ring; 8285 int r; 8286 8287 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size) 8288 return; 8289 8290 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8291 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); 8292 if (r) { 8293 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); 8294 return; 8295 } 8296 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8297 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); 8298 if (r) { 8299 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); 8300 return; 8301 } 8302 r = vce_v1_0_init(rdev); 8303 if (r) { 8304 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r); 8305 return; 8306 } 8307 } 8308 8309 /** 8310 * cik_startup - program the asic to a functional state 8311 * 8312 * @rdev: radeon_device pointer 8313 * 8314 * Programs the asic to a functional state (CIK). 8315 * Called by cik_init() and cik_resume(). 8316 * Returns 0 for success, error for failure. 8317 */ 8318 static int cik_startup(struct radeon_device *rdev) 8319 { 8320 struct radeon_ring *ring; 8321 u32 nop; 8322 int r; 8323 8324 /* enable pcie gen2/3 link */ 8325 cik_pcie_gen3_enable(rdev); 8326 /* enable aspm */ 8327 cik_program_aspm(rdev); 8328 8329 /* scratch needs to be initialized before MC */ 8330 r = r600_vram_scratch_init(rdev); 8331 if (r) 8332 return r; 8333 8334 cik_mc_program(rdev); 8335 8336 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 8337 r = ci_mc_load_microcode(rdev); 8338 if (r) { 8339 DRM_ERROR("Failed to load MC firmware!\n"); 8340 return r; 8341 } 8342 } 8343 8344 r = cik_pcie_gart_enable(rdev); 8345 if (r) 8346 return r; 8347 cik_gpu_init(rdev); 8348 8349 /* allocate rlc buffers */ 8350 if (rdev->flags & RADEON_IS_IGP) { 8351 if (rdev->family == CHIP_KAVERI) { 8352 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 8353 rdev->rlc.reg_list_size = 8354 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 8355 } else { 8356 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 8357 rdev->rlc.reg_list_size = 8358 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 8359 } 8360 } 8361 rdev->rlc.cs_data = ci_cs_data; 8362 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */ 8363 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */ 8364 r = sumo_rlc_init(rdev); 8365 if (r) { 8366 DRM_ERROR("Failed to init rlc BOs!\n"); 8367 return r; 8368 } 8369 8370 /* allocate wb buffer */ 8371 r = radeon_wb_init(rdev); 8372 if (r) 8373 return r; 8374 8375 /* allocate mec buffers */ 8376 r = cik_mec_init(rdev); 8377 if (r) { 8378 DRM_ERROR("Failed to init MEC BOs!\n"); 8379 return r; 8380 } 8381 8382 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 8383 if (r) { 8384 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8385 return r; 8386 } 8387 8388 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8389 if (r) { 8390 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8391 return r; 8392 } 8393 8394 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8395 if (r) { 8396 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8397 return r; 8398 } 8399 8400 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 8401 if (r) { 8402 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8403 return r; 8404 } 8405 8406 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8407 if (r) { 8408 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8409 return r; 8410 } 8411 8412 cik_uvd_start(rdev); 8413 cik_vce_start(rdev); 8414 8415 /* Enable IRQ */ 8416 if (!rdev->irq.installed) { 8417 r = radeon_irq_kms_init(rdev); 8418 if (r) 8419 return r; 8420 } 8421 8422 r = cik_irq_init(rdev); 8423 if (r) { 8424 DRM_ERROR("radeon: IH init failed (%d).\n", r); 8425 radeon_irq_kms_fini(rdev); 8426 return r; 8427 } 8428 cik_irq_set(rdev); 8429 8430 if (rdev->family == CHIP_HAWAII) { 8431 if (rdev->new_fw) 8432 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8433 else 8434 nop = RADEON_CP_PACKET2; 8435 } else { 8436 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8437 } 8438 8439 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8440 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 8441 nop); 8442 if (r) 8443 return r; 8444 8445 /* set up the compute queues */ 8446 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8447 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8448 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 8449 nop); 8450 if (r) 8451 return r; 8452 ring->me = 1; /* first MEC */ 8453 ring->pipe = 0; /* first pipe */ 8454 ring->queue = 0; /* first queue */ 8455 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 8456 8457 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8458 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8459 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 8460 nop); 8461 if (r) 8462 return r; 8463 /* dGPU only have 1 MEC */ 8464 ring->me = 1; /* first MEC */ 8465 ring->pipe = 0; /* first pipe */ 8466 ring->queue = 1; /* second queue */ 8467 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 8468 8469 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8470 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 8471 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8472 if (r) 8473 return r; 8474 8475 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8476 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 8477 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8478 if (r) 8479 return r; 8480 8481 r = cik_cp_resume(rdev); 8482 if (r) 8483 return r; 8484 8485 r = cik_sdma_resume(rdev); 8486 if (r) 8487 return r; 8488 8489 cik_uvd_resume(rdev); 8490 cik_vce_resume(rdev); 8491 8492 r = radeon_ib_pool_init(rdev); 8493 if (r) { 8494 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 8495 return r; 8496 } 8497 8498 r = radeon_vm_manager_init(rdev); 8499 if (r) { 8500 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 8501 return r; 8502 } 8503 8504 r = radeon_audio_init(rdev); 8505 if (r) 8506 return r; 8507 8508 return 0; 8509 } 8510 8511 /** 8512 * cik_resume - resume the asic to a functional state 8513 * 8514 * @rdev: radeon_device pointer 8515 * 8516 * Programs the asic to a functional state (CIK). 8517 * Called at resume. 8518 * Returns 0 for success, error for failure. 8519 */ 8520 int cik_resume(struct radeon_device *rdev) 8521 { 8522 int r; 8523 8524 /* post card */ 8525 atom_asic_init(rdev->mode_info.atom_context); 8526 8527 /* init golden registers */ 8528 cik_init_golden_registers(rdev); 8529 8530 if (rdev->pm.pm_method == PM_METHOD_DPM) 8531 radeon_pm_resume(rdev); 8532 8533 rdev->accel_working = true; 8534 r = cik_startup(rdev); 8535 if (r) { 8536 DRM_ERROR("cik startup failed on resume\n"); 8537 rdev->accel_working = false; 8538 return r; 8539 } 8540 8541 return r; 8542 8543 } 8544 8545 /** 8546 * cik_suspend - suspend the asic 8547 * 8548 * @rdev: radeon_device pointer 8549 * 8550 * Bring the chip into a state suitable for suspend (CIK). 8551 * Called at suspend. 8552 * Returns 0 for success. 8553 */ 8554 int cik_suspend(struct radeon_device *rdev) 8555 { 8556 radeon_pm_suspend(rdev); 8557 radeon_audio_fini(rdev); 8558 radeon_vm_manager_fini(rdev); 8559 cik_cp_enable(rdev, false); 8560 cik_sdma_enable(rdev, false); 8561 if (rdev->has_uvd) { 8562 uvd_v1_0_fini(rdev); 8563 radeon_uvd_suspend(rdev); 8564 } 8565 if (rdev->has_vce) 8566 radeon_vce_suspend(rdev); 8567 cik_fini_pg(rdev); 8568 cik_fini_cg(rdev); 8569 cik_irq_suspend(rdev); 8570 radeon_wb_disable(rdev); 8571 cik_pcie_gart_disable(rdev); 8572 return 0; 8573 } 8574 8575 /* Plan is to move initialization in that function and use 8576 * helper function so that radeon_device_init pretty much 8577 * do nothing more than calling asic specific function. This 8578 * should also allow to remove a bunch of callback function 8579 * like vram_info. 8580 */ 8581 /** 8582 * cik_init - asic specific driver and hw init 8583 * 8584 * @rdev: radeon_device pointer 8585 * 8586 * Setup asic specific driver variables and program the hw 8587 * to a functional state (CIK). 8588 * Called at driver startup. 8589 * Returns 0 for success, errors for failure. 8590 */ 8591 int cik_init(struct radeon_device *rdev) 8592 { 8593 struct radeon_ring *ring; 8594 int r; 8595 8596 /* Read BIOS */ 8597 if (!radeon_get_bios(rdev)) { 8598 if (ASIC_IS_AVIVO(rdev)) 8599 return -EINVAL; 8600 } 8601 /* Must be an ATOMBIOS */ 8602 if (!rdev->is_atom_bios) { 8603 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8604 return -EINVAL; 8605 } 8606 r = radeon_atombios_init(rdev); 8607 if (r) 8608 return r; 8609 8610 /* Post card if necessary */ 8611 if (!radeon_card_posted(rdev)) { 8612 if (!rdev->bios) { 8613 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8614 return -EINVAL; 8615 } 8616 DRM_INFO("GPU not posted. posting now...\n"); 8617 atom_asic_init(rdev->mode_info.atom_context); 8618 } 8619 /* init golden registers */ 8620 cik_init_golden_registers(rdev); 8621 /* Initialize scratch registers */ 8622 cik_scratch_init(rdev); 8623 /* Initialize surface registers */ 8624 radeon_surface_init(rdev); 8625 /* Initialize clocks */ 8626 radeon_get_clock_info(rdev->ddev); 8627 8628 /* Fence driver */ 8629 r = radeon_fence_driver_init(rdev); 8630 if (r) 8631 return r; 8632 8633 /* initialize memory controller */ 8634 r = cik_mc_init(rdev); 8635 if (r) 8636 return r; 8637 /* Memory manager */ 8638 r = radeon_bo_init(rdev); 8639 if (r) 8640 return r; 8641 8642 if (rdev->flags & RADEON_IS_IGP) { 8643 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8644 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8645 r = cik_init_microcode(rdev); 8646 if (r) { 8647 DRM_ERROR("Failed to load firmware!\n"); 8648 return r; 8649 } 8650 } 8651 } else { 8652 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8653 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8654 !rdev->mc_fw) { 8655 r = cik_init_microcode(rdev); 8656 if (r) { 8657 DRM_ERROR("Failed to load firmware!\n"); 8658 return r; 8659 } 8660 } 8661 } 8662 8663 /* Initialize power management */ 8664 radeon_pm_init(rdev); 8665 8666 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8667 ring->ring_obj = NULL; 8668 r600_ring_init(rdev, ring, 1024 * 1024); 8669 8670 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8671 ring->ring_obj = NULL; 8672 r600_ring_init(rdev, ring, 1024 * 1024); 8673 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8674 if (r) 8675 return r; 8676 8677 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8678 ring->ring_obj = NULL; 8679 r600_ring_init(rdev, ring, 1024 * 1024); 8680 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8681 if (r) 8682 return r; 8683 8684 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8685 ring->ring_obj = NULL; 8686 r600_ring_init(rdev, ring, 256 * 1024); 8687 8688 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8689 ring->ring_obj = NULL; 8690 r600_ring_init(rdev, ring, 256 * 1024); 8691 8692 cik_uvd_init(rdev); 8693 cik_vce_init(rdev); 8694 8695 rdev->ih.ring_obj = NULL; 8696 r600_ih_ring_init(rdev, 64 * 1024); 8697 8698 r = r600_pcie_gart_init(rdev); 8699 if (r) 8700 return r; 8701 8702 rdev->accel_working = true; 8703 r = cik_startup(rdev); 8704 if (r) { 8705 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8706 cik_cp_fini(rdev); 8707 cik_sdma_fini(rdev); 8708 cik_irq_fini(rdev); 8709 sumo_rlc_fini(rdev); 8710 cik_mec_fini(rdev); 8711 radeon_wb_fini(rdev); 8712 radeon_ib_pool_fini(rdev); 8713 radeon_vm_manager_fini(rdev); 8714 radeon_irq_kms_fini(rdev); 8715 cik_pcie_gart_fini(rdev); 8716 rdev->accel_working = false; 8717 } 8718 8719 /* Don't start up if the MC ucode is missing. 8720 * The default clocks and voltages before the MC ucode 8721 * is loaded are not suffient for advanced operations. 8722 */ 8723 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8724 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8725 return -EINVAL; 8726 } 8727 8728 return 0; 8729 } 8730 8731 /** 8732 * cik_fini - asic specific driver and hw fini 8733 * 8734 * @rdev: radeon_device pointer 8735 * 8736 * Tear down the asic specific driver variables and program the hw 8737 * to an idle state (CIK). 8738 * Called at driver unload. 8739 */ 8740 void cik_fini(struct radeon_device *rdev) 8741 { 8742 radeon_pm_fini(rdev); 8743 cik_cp_fini(rdev); 8744 cik_sdma_fini(rdev); 8745 cik_fini_pg(rdev); 8746 cik_fini_cg(rdev); 8747 cik_irq_fini(rdev); 8748 sumo_rlc_fini(rdev); 8749 cik_mec_fini(rdev); 8750 radeon_wb_fini(rdev); 8751 radeon_vm_manager_fini(rdev); 8752 radeon_ib_pool_fini(rdev); 8753 radeon_irq_kms_fini(rdev); 8754 uvd_v1_0_fini(rdev); 8755 radeon_uvd_fini(rdev); 8756 radeon_vce_fini(rdev); 8757 cik_pcie_gart_fini(rdev); 8758 r600_vram_scratch_fini(rdev); 8759 radeon_gem_fini(rdev); 8760 radeon_fence_driver_fini(rdev); 8761 radeon_bo_fini(rdev); 8762 radeon_atombios_fini(rdev); 8763 kfree(rdev->bios); 8764 rdev->bios = NULL; 8765 } 8766 8767 void dce8_program_fmt(struct drm_encoder *encoder) 8768 { 8769 struct drm_device *dev = encoder->dev; 8770 struct radeon_device *rdev = dev->dev_private; 8771 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8772 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8773 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8774 int bpc = 0; 8775 u32 tmp = 0; 8776 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8777 8778 if (connector) { 8779 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8780 bpc = radeon_get_monitor_bpc(connector); 8781 dither = radeon_connector->dither; 8782 } 8783 8784 /* LVDS/eDP FMT is set up by atom */ 8785 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8786 return; 8787 8788 /* not needed for analog */ 8789 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8790 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8791 return; 8792 8793 if (bpc == 0) 8794 return; 8795 8796 switch (bpc) { 8797 case 6: 8798 if (dither == RADEON_FMT_DITHER_ENABLE) 8799 /* XXX sort out optimal dither settings */ 8800 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8801 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8802 else 8803 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8804 break; 8805 case 8: 8806 if (dither == RADEON_FMT_DITHER_ENABLE) 8807 /* XXX sort out optimal dither settings */ 8808 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8809 FMT_RGB_RANDOM_ENABLE | 8810 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8811 else 8812 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8813 break; 8814 case 10: 8815 if (dither == RADEON_FMT_DITHER_ENABLE) 8816 /* XXX sort out optimal dither settings */ 8817 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8818 FMT_RGB_RANDOM_ENABLE | 8819 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8820 else 8821 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8822 break; 8823 default: 8824 /* not needed */ 8825 break; 8826 } 8827 8828 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8829 } 8830 8831 /* display watermark setup */ 8832 /** 8833 * dce8_line_buffer_adjust - Set up the line buffer 8834 * 8835 * @rdev: radeon_device pointer 8836 * @radeon_crtc: the selected display controller 8837 * @mode: the current display mode on the selected display 8838 * controller 8839 * 8840 * Setup up the line buffer allocation for 8841 * the selected display controller (CIK). 8842 * Returns the line buffer size in pixels. 8843 */ 8844 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8845 struct radeon_crtc *radeon_crtc, 8846 struct drm_display_mode *mode) 8847 { 8848 u32 tmp, buffer_alloc, i; 8849 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8850 /* 8851 * Line Buffer Setup 8852 * There are 6 line buffers, one for each display controllers. 8853 * There are 3 partitions per LB. Select the number of partitions 8854 * to enable based on the display width. For display widths larger 8855 * than 4096, you need use to use 2 display controllers and combine 8856 * them using the stereo blender. 8857 */ 8858 if (radeon_crtc->base.enabled && mode) { 8859 if (mode->crtc_hdisplay < 1920) { 8860 tmp = 1; 8861 buffer_alloc = 2; 8862 } else if (mode->crtc_hdisplay < 2560) { 8863 tmp = 2; 8864 buffer_alloc = 2; 8865 } else if (mode->crtc_hdisplay < 4096) { 8866 tmp = 0; 8867 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8868 } else { 8869 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8870 tmp = 0; 8871 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8872 } 8873 } else { 8874 tmp = 1; 8875 buffer_alloc = 0; 8876 } 8877 8878 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8879 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8880 8881 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8882 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8883 for (i = 0; i < rdev->usec_timeout; i++) { 8884 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8885 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8886 break; 8887 udelay(1); 8888 } 8889 8890 if (radeon_crtc->base.enabled && mode) { 8891 switch (tmp) { 8892 case 0: 8893 default: 8894 return 4096 * 2; 8895 case 1: 8896 return 1920 * 2; 8897 case 2: 8898 return 2560 * 2; 8899 } 8900 } 8901 8902 /* controller not enabled, so no lb used */ 8903 return 0; 8904 } 8905 8906 /** 8907 * cik_get_number_of_dram_channels - get the number of dram channels 8908 * 8909 * @rdev: radeon_device pointer 8910 * 8911 * Look up the number of video ram channels (CIK). 8912 * Used for display watermark bandwidth calculations 8913 * Returns the number of dram channels 8914 */ 8915 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8916 { 8917 u32 tmp = RREG32(MC_SHARED_CHMAP); 8918 8919 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8920 case 0: 8921 default: 8922 return 1; 8923 case 1: 8924 return 2; 8925 case 2: 8926 return 4; 8927 case 3: 8928 return 8; 8929 case 4: 8930 return 3; 8931 case 5: 8932 return 6; 8933 case 6: 8934 return 10; 8935 case 7: 8936 return 12; 8937 case 8: 8938 return 16; 8939 } 8940 } 8941 8942 struct dce8_wm_params { 8943 u32 dram_channels; /* number of dram channels */ 8944 u32 yclk; /* bandwidth per dram data pin in kHz */ 8945 u32 sclk; /* engine clock in kHz */ 8946 u32 disp_clk; /* display clock in kHz */ 8947 u32 src_width; /* viewport width */ 8948 u32 active_time; /* active display time in ns */ 8949 u32 blank_time; /* blank time in ns */ 8950 bool interlaced; /* mode is interlaced */ 8951 fixed20_12 vsc; /* vertical scale ratio */ 8952 u32 num_heads; /* number of active crtcs */ 8953 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 8954 u32 lb_size; /* line buffer allocated to pipe */ 8955 u32 vtaps; /* vertical scaler taps */ 8956 }; 8957 8958 /** 8959 * dce8_dram_bandwidth - get the dram bandwidth 8960 * 8961 * @wm: watermark calculation data 8962 * 8963 * Calculate the raw dram bandwidth (CIK). 8964 * Used for display watermark bandwidth calculations 8965 * Returns the dram bandwidth in MBytes/s 8966 */ 8967 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 8968 { 8969 /* Calculate raw DRAM Bandwidth */ 8970 fixed20_12 dram_efficiency; /* 0.7 */ 8971 fixed20_12 yclk, dram_channels, bandwidth; 8972 fixed20_12 a; 8973 8974 a.full = dfixed_const(1000); 8975 yclk.full = dfixed_const(wm->yclk); 8976 yclk.full = dfixed_div(yclk, a); 8977 dram_channels.full = dfixed_const(wm->dram_channels * 4); 8978 a.full = dfixed_const(10); 8979 dram_efficiency.full = dfixed_const(7); 8980 dram_efficiency.full = dfixed_div(dram_efficiency, a); 8981 bandwidth.full = dfixed_mul(dram_channels, yclk); 8982 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 8983 8984 return dfixed_trunc(bandwidth); 8985 } 8986 8987 /** 8988 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 8989 * 8990 * @wm: watermark calculation data 8991 * 8992 * Calculate the dram bandwidth used for display (CIK). 8993 * Used for display watermark bandwidth calculations 8994 * Returns the dram bandwidth for display in MBytes/s 8995 */ 8996 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 8997 { 8998 /* Calculate DRAM Bandwidth and the part allocated to display. */ 8999 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 9000 fixed20_12 yclk, dram_channels, bandwidth; 9001 fixed20_12 a; 9002 9003 a.full = dfixed_const(1000); 9004 yclk.full = dfixed_const(wm->yclk); 9005 yclk.full = dfixed_div(yclk, a); 9006 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9007 a.full = dfixed_const(10); 9008 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 9009 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 9010 bandwidth.full = dfixed_mul(dram_channels, yclk); 9011 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 9012 9013 return dfixed_trunc(bandwidth); 9014 } 9015 9016 /** 9017 * dce8_data_return_bandwidth - get the data return bandwidth 9018 * 9019 * @wm: watermark calculation data 9020 * 9021 * Calculate the data return bandwidth used for display (CIK). 9022 * Used for display watermark bandwidth calculations 9023 * Returns the data return bandwidth in MBytes/s 9024 */ 9025 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 9026 { 9027 /* Calculate the display Data return Bandwidth */ 9028 fixed20_12 return_efficiency; /* 0.8 */ 9029 fixed20_12 sclk, bandwidth; 9030 fixed20_12 a; 9031 9032 a.full = dfixed_const(1000); 9033 sclk.full = dfixed_const(wm->sclk); 9034 sclk.full = dfixed_div(sclk, a); 9035 a.full = dfixed_const(10); 9036 return_efficiency.full = dfixed_const(8); 9037 return_efficiency.full = dfixed_div(return_efficiency, a); 9038 a.full = dfixed_const(32); 9039 bandwidth.full = dfixed_mul(a, sclk); 9040 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 9041 9042 return dfixed_trunc(bandwidth); 9043 } 9044 9045 /** 9046 * dce8_dmif_request_bandwidth - get the dmif bandwidth 9047 * 9048 * @wm: watermark calculation data 9049 * 9050 * Calculate the dmif bandwidth used for display (CIK). 9051 * Used for display watermark bandwidth calculations 9052 * Returns the dmif bandwidth in MBytes/s 9053 */ 9054 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 9055 { 9056 /* Calculate the DMIF Request Bandwidth */ 9057 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 9058 fixed20_12 disp_clk, bandwidth; 9059 fixed20_12 a, b; 9060 9061 a.full = dfixed_const(1000); 9062 disp_clk.full = dfixed_const(wm->disp_clk); 9063 disp_clk.full = dfixed_div(disp_clk, a); 9064 a.full = dfixed_const(32); 9065 b.full = dfixed_mul(a, disp_clk); 9066 9067 a.full = dfixed_const(10); 9068 disp_clk_request_efficiency.full = dfixed_const(8); 9069 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 9070 9071 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 9072 9073 return dfixed_trunc(bandwidth); 9074 } 9075 9076 /** 9077 * dce8_available_bandwidth - get the min available bandwidth 9078 * 9079 * @wm: watermark calculation data 9080 * 9081 * Calculate the min available bandwidth used for display (CIK). 9082 * Used for display watermark bandwidth calculations 9083 * Returns the min available bandwidth in MBytes/s 9084 */ 9085 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 9086 { 9087 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 9088 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 9089 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 9090 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 9091 9092 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 9093 } 9094 9095 /** 9096 * dce8_average_bandwidth - get the average available bandwidth 9097 * 9098 * @wm: watermark calculation data 9099 * 9100 * Calculate the average available bandwidth used for display (CIK). 9101 * Used for display watermark bandwidth calculations 9102 * Returns the average available bandwidth in MBytes/s 9103 */ 9104 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 9105 { 9106 /* Calculate the display mode Average Bandwidth 9107 * DisplayMode should contain the source and destination dimensions, 9108 * timing, etc. 9109 */ 9110 fixed20_12 bpp; 9111 fixed20_12 line_time; 9112 fixed20_12 src_width; 9113 fixed20_12 bandwidth; 9114 fixed20_12 a; 9115 9116 a.full = dfixed_const(1000); 9117 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 9118 line_time.full = dfixed_div(line_time, a); 9119 bpp.full = dfixed_const(wm->bytes_per_pixel); 9120 src_width.full = dfixed_const(wm->src_width); 9121 bandwidth.full = dfixed_mul(src_width, bpp); 9122 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 9123 bandwidth.full = dfixed_div(bandwidth, line_time); 9124 9125 return dfixed_trunc(bandwidth); 9126 } 9127 9128 /** 9129 * dce8_latency_watermark - get the latency watermark 9130 * 9131 * @wm: watermark calculation data 9132 * 9133 * Calculate the latency watermark (CIK). 9134 * Used for display watermark bandwidth calculations 9135 * Returns the latency watermark in ns 9136 */ 9137 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 9138 { 9139 /* First calculate the latency in ns */ 9140 u32 mc_latency = 2000; /* 2000 ns. */ 9141 u32 available_bandwidth = dce8_available_bandwidth(wm); 9142 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 9143 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 9144 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 9145 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 9146 (wm->num_heads * cursor_line_pair_return_time); 9147 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 9148 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 9149 u32 tmp, dmif_size = 12288; 9150 fixed20_12 a, b, c; 9151 9152 if (wm->num_heads == 0) 9153 return 0; 9154 9155 a.full = dfixed_const(2); 9156 b.full = dfixed_const(1); 9157 if ((wm->vsc.full > a.full) || 9158 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 9159 (wm->vtaps >= 5) || 9160 ((wm->vsc.full >= a.full) && wm->interlaced)) 9161 max_src_lines_per_dst_line = 4; 9162 else 9163 max_src_lines_per_dst_line = 2; 9164 9165 a.full = dfixed_const(available_bandwidth); 9166 b.full = dfixed_const(wm->num_heads); 9167 a.full = dfixed_div(a, b); 9168 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512); 9169 tmp = min(dfixed_trunc(a), tmp); 9170 9171 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000); 9172 9173 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 9174 b.full = dfixed_const(1000); 9175 c.full = dfixed_const(lb_fill_bw); 9176 b.full = dfixed_div(c, b); 9177 a.full = dfixed_div(a, b); 9178 line_fill_time = dfixed_trunc(a); 9179 9180 if (line_fill_time < wm->active_time) 9181 return latency; 9182 else 9183 return latency + (line_fill_time - wm->active_time); 9184 9185 } 9186 9187 /** 9188 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 9189 * average and available dram bandwidth 9190 * 9191 * @wm: watermark calculation data 9192 * 9193 * Check if the display average bandwidth fits in the display 9194 * dram bandwidth (CIK). 9195 * Used for display watermark bandwidth calculations 9196 * Returns true if the display fits, false if not. 9197 */ 9198 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9199 { 9200 if (dce8_average_bandwidth(wm) <= 9201 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 9202 return true; 9203 else 9204 return false; 9205 } 9206 9207 /** 9208 * dce8_average_bandwidth_vs_available_bandwidth - check 9209 * average and available bandwidth 9210 * 9211 * @wm: watermark calculation data 9212 * 9213 * Check if the display average bandwidth fits in the display 9214 * available bandwidth (CIK). 9215 * Used for display watermark bandwidth calculations 9216 * Returns true if the display fits, false if not. 9217 */ 9218 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 9219 { 9220 if (dce8_average_bandwidth(wm) <= 9221 (dce8_available_bandwidth(wm) / wm->num_heads)) 9222 return true; 9223 else 9224 return false; 9225 } 9226 9227 /** 9228 * dce8_check_latency_hiding - check latency hiding 9229 * 9230 * @wm: watermark calculation data 9231 * 9232 * Check latency hiding (CIK). 9233 * Used for display watermark bandwidth calculations 9234 * Returns true if the display fits, false if not. 9235 */ 9236 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 9237 { 9238 u32 lb_partitions = wm->lb_size / wm->src_width; 9239 u32 line_time = wm->active_time + wm->blank_time; 9240 u32 latency_tolerant_lines; 9241 u32 latency_hiding; 9242 fixed20_12 a; 9243 9244 a.full = dfixed_const(1); 9245 if (wm->vsc.full > a.full) 9246 latency_tolerant_lines = 1; 9247 else { 9248 if (lb_partitions <= (wm->vtaps + 1)) 9249 latency_tolerant_lines = 1; 9250 else 9251 latency_tolerant_lines = 2; 9252 } 9253 9254 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 9255 9256 if (dce8_latency_watermark(wm) <= latency_hiding) 9257 return true; 9258 else 9259 return false; 9260 } 9261 9262 /** 9263 * dce8_program_watermarks - program display watermarks 9264 * 9265 * @rdev: radeon_device pointer 9266 * @radeon_crtc: the selected display controller 9267 * @lb_size: line buffer size 9268 * @num_heads: number of display controllers in use 9269 * 9270 * Calculate and program the display watermarks for the 9271 * selected display controller (CIK). 9272 */ 9273 static void dce8_program_watermarks(struct radeon_device *rdev, 9274 struct radeon_crtc *radeon_crtc, 9275 u32 lb_size, u32 num_heads) 9276 { 9277 struct drm_display_mode *mode = &radeon_crtc->base.mode; 9278 struct dce8_wm_params wm_low, wm_high; 9279 u32 active_time; 9280 u32 line_time = 0; 9281 u32 latency_watermark_a = 0, latency_watermark_b = 0; 9282 u32 tmp, wm_mask; 9283 9284 if (radeon_crtc->base.enabled && num_heads && mode) { 9285 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000, 9286 (u32)mode->clock); 9287 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000, 9288 (u32)mode->clock); 9289 line_time = min(line_time, (u32)65535); 9290 9291 /* watermark for high clocks */ 9292 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9293 rdev->pm.dpm_enabled) { 9294 wm_high.yclk = 9295 radeon_dpm_get_mclk(rdev, false) * 10; 9296 wm_high.sclk = 9297 radeon_dpm_get_sclk(rdev, false) * 10; 9298 } else { 9299 wm_high.yclk = rdev->pm.current_mclk * 10; 9300 wm_high.sclk = rdev->pm.current_sclk * 10; 9301 } 9302 9303 wm_high.disp_clk = mode->clock; 9304 wm_high.src_width = mode->crtc_hdisplay; 9305 wm_high.active_time = active_time; 9306 wm_high.blank_time = line_time - wm_high.active_time; 9307 wm_high.interlaced = false; 9308 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9309 wm_high.interlaced = true; 9310 wm_high.vsc = radeon_crtc->vsc; 9311 wm_high.vtaps = 1; 9312 if (radeon_crtc->rmx_type != RMX_OFF) 9313 wm_high.vtaps = 2; 9314 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9315 wm_high.lb_size = lb_size; 9316 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 9317 wm_high.num_heads = num_heads; 9318 9319 /* set for high clocks */ 9320 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 9321 9322 /* possibly force display priority to high */ 9323 /* should really do this at mode validation time... */ 9324 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 9325 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 9326 !dce8_check_latency_hiding(&wm_high) || 9327 (rdev->disp_priority == 2)) { 9328 DRM_DEBUG_KMS("force priority to high\n"); 9329 } 9330 9331 /* watermark for low clocks */ 9332 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9333 rdev->pm.dpm_enabled) { 9334 wm_low.yclk = 9335 radeon_dpm_get_mclk(rdev, true) * 10; 9336 wm_low.sclk = 9337 radeon_dpm_get_sclk(rdev, true) * 10; 9338 } else { 9339 wm_low.yclk = rdev->pm.current_mclk * 10; 9340 wm_low.sclk = rdev->pm.current_sclk * 10; 9341 } 9342 9343 wm_low.disp_clk = mode->clock; 9344 wm_low.src_width = mode->crtc_hdisplay; 9345 wm_low.active_time = active_time; 9346 wm_low.blank_time = line_time - wm_low.active_time; 9347 wm_low.interlaced = false; 9348 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9349 wm_low.interlaced = true; 9350 wm_low.vsc = radeon_crtc->vsc; 9351 wm_low.vtaps = 1; 9352 if (radeon_crtc->rmx_type != RMX_OFF) 9353 wm_low.vtaps = 2; 9354 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9355 wm_low.lb_size = lb_size; 9356 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 9357 wm_low.num_heads = num_heads; 9358 9359 /* set for low clocks */ 9360 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 9361 9362 /* possibly force display priority to high */ 9363 /* should really do this at mode validation time... */ 9364 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 9365 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 9366 !dce8_check_latency_hiding(&wm_low) || 9367 (rdev->disp_priority == 2)) { 9368 DRM_DEBUG_KMS("force priority to high\n"); 9369 } 9370 9371 /* Save number of lines the linebuffer leads before the scanout */ 9372 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); 9373 } 9374 9375 /* select wm A */ 9376 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9377 tmp = wm_mask; 9378 tmp &= ~LATENCY_WATERMARK_MASK(3); 9379 tmp |= LATENCY_WATERMARK_MASK(1); 9380 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9381 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9382 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 9383 LATENCY_HIGH_WATERMARK(line_time))); 9384 /* select wm B */ 9385 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9386 tmp &= ~LATENCY_WATERMARK_MASK(3); 9387 tmp |= LATENCY_WATERMARK_MASK(2); 9388 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9389 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9390 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 9391 LATENCY_HIGH_WATERMARK(line_time))); 9392 /* restore original selection */ 9393 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 9394 9395 /* save values for DPM */ 9396 radeon_crtc->line_time = line_time; 9397 radeon_crtc->wm_high = latency_watermark_a; 9398 radeon_crtc->wm_low = latency_watermark_b; 9399 } 9400 9401 /** 9402 * dce8_bandwidth_update - program display watermarks 9403 * 9404 * @rdev: radeon_device pointer 9405 * 9406 * Calculate and program the display watermarks and line 9407 * buffer allocation (CIK). 9408 */ 9409 void dce8_bandwidth_update(struct radeon_device *rdev) 9410 { 9411 struct drm_display_mode *mode = NULL; 9412 u32 num_heads = 0, lb_size; 9413 int i; 9414 9415 if (!rdev->mode_info.mode_config_initialized) 9416 return; 9417 9418 radeon_update_display_priority(rdev); 9419 9420 for (i = 0; i < rdev->num_crtc; i++) { 9421 if (rdev->mode_info.crtcs[i]->base.enabled) 9422 num_heads++; 9423 } 9424 for (i = 0; i < rdev->num_crtc; i++) { 9425 mode = &rdev->mode_info.crtcs[i]->base.mode; 9426 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 9427 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 9428 } 9429 } 9430 9431 /** 9432 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 9433 * 9434 * @rdev: radeon_device pointer 9435 * 9436 * Fetches a GPU clock counter snapshot (SI). 9437 * Returns the 64 bit clock counter snapshot. 9438 */ 9439 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 9440 { 9441 uint64_t clock; 9442 9443 mutex_lock(&rdev->gpu_clock_mutex); 9444 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 9445 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 9446 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 9447 mutex_unlock(&rdev->gpu_clock_mutex); 9448 return clock; 9449 } 9450 9451 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 9452 u32 cntl_reg, u32 status_reg) 9453 { 9454 int r, i; 9455 struct atom_clock_dividers dividers; 9456 uint32_t tmp; 9457 9458 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9459 clock, false, ÷rs); 9460 if (r) 9461 return r; 9462 9463 tmp = RREG32_SMC(cntl_reg); 9464 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 9465 tmp |= dividers.post_divider; 9466 WREG32_SMC(cntl_reg, tmp); 9467 9468 for (i = 0; i < 100; i++) { 9469 if (RREG32_SMC(status_reg) & DCLK_STATUS) 9470 break; 9471 mdelay(10); 9472 } 9473 if (i == 100) 9474 return -ETIMEDOUT; 9475 9476 return 0; 9477 } 9478 9479 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 9480 { 9481 int r = 0; 9482 9483 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 9484 if (r) 9485 return r; 9486 9487 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 9488 return r; 9489 } 9490 9491 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 9492 { 9493 int r, i; 9494 struct atom_clock_dividers dividers; 9495 u32 tmp; 9496 9497 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9498 ecclk, false, ÷rs); 9499 if (r) 9500 return r; 9501 9502 for (i = 0; i < 100; i++) { 9503 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9504 break; 9505 mdelay(10); 9506 } 9507 if (i == 100) 9508 return -ETIMEDOUT; 9509 9510 tmp = RREG32_SMC(CG_ECLK_CNTL); 9511 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 9512 tmp |= dividers.post_divider; 9513 WREG32_SMC(CG_ECLK_CNTL, tmp); 9514 9515 for (i = 0; i < 100; i++) { 9516 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9517 break; 9518 mdelay(10); 9519 } 9520 if (i == 100) 9521 return -ETIMEDOUT; 9522 9523 return 0; 9524 } 9525 9526 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 9527 { 9528 struct pci_dev *root = rdev->pdev->bus->self; 9529 int bridge_pos, gpu_pos; 9530 u32 speed_cntl, mask, current_data_rate; 9531 int ret, i; 9532 u16 tmp16; 9533 9534 if (pci_is_root_bus(rdev->pdev->bus)) 9535 return; 9536 9537 if (radeon_pcie_gen2 == 0) 9538 return; 9539 9540 if (rdev->flags & RADEON_IS_IGP) 9541 return; 9542 9543 if (!(rdev->flags & RADEON_IS_PCIE)) 9544 return; 9545 9546 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9547 if (ret != 0) 9548 return; 9549 9550 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9551 return; 9552 9553 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9554 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9555 LC_CURRENT_DATA_RATE_SHIFT; 9556 if (mask & DRM_PCIE_SPEED_80) { 9557 if (current_data_rate == 2) { 9558 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9559 return; 9560 } 9561 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9562 } else if (mask & DRM_PCIE_SPEED_50) { 9563 if (current_data_rate == 1) { 9564 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9565 return; 9566 } 9567 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9568 } 9569 9570 bridge_pos = pci_pcie_cap(root); 9571 if (!bridge_pos) 9572 return; 9573 9574 gpu_pos = pci_pcie_cap(rdev->pdev); 9575 if (!gpu_pos) 9576 return; 9577 9578 if (mask & DRM_PCIE_SPEED_80) { 9579 /* re-try equalization if gen3 is not already enabled */ 9580 if (current_data_rate != 2) { 9581 u16 bridge_cfg, gpu_cfg; 9582 u16 bridge_cfg2, gpu_cfg2; 9583 u32 max_lw, current_lw, tmp; 9584 9585 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9586 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9587 9588 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9589 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9590 9591 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9592 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9593 9594 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9595 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9596 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9597 9598 if (current_lw < max_lw) { 9599 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9600 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9601 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9602 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9603 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9604 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9605 } 9606 } 9607 9608 for (i = 0; i < 10; i++) { 9609 /* check status */ 9610 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9611 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9612 break; 9613 9614 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9615 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9616 9617 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9618 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9619 9620 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9621 tmp |= LC_SET_QUIESCE; 9622 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9623 9624 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9625 tmp |= LC_REDO_EQ; 9626 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9627 9628 mdelay(100); 9629 9630 /* linkctl */ 9631 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9632 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9633 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9634 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9635 9636 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9637 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9638 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9639 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9640 9641 /* linkctl2 */ 9642 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9643 tmp16 &= ~((1 << 4) | (7 << 9)); 9644 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9645 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9646 9647 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9648 tmp16 &= ~((1 << 4) | (7 << 9)); 9649 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9650 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9651 9652 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9653 tmp &= ~LC_SET_QUIESCE; 9654 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9655 } 9656 } 9657 } 9658 9659 /* set the link speed */ 9660 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9661 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9662 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9663 9664 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9665 tmp16 &= ~0xf; 9666 if (mask & DRM_PCIE_SPEED_80) 9667 tmp16 |= 3; /* gen3 */ 9668 else if (mask & DRM_PCIE_SPEED_50) 9669 tmp16 |= 2; /* gen2 */ 9670 else 9671 tmp16 |= 1; /* gen1 */ 9672 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9673 9674 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9675 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9676 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9677 9678 for (i = 0; i < rdev->usec_timeout; i++) { 9679 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9680 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9681 break; 9682 udelay(1); 9683 } 9684 } 9685 9686 static void cik_program_aspm(struct radeon_device *rdev) 9687 { 9688 u32 data, orig; 9689 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9690 bool disable_clkreq = false; 9691 9692 if (radeon_aspm == 0) 9693 return; 9694 9695 /* XXX double check IGPs */ 9696 if (rdev->flags & RADEON_IS_IGP) 9697 return; 9698 9699 if (!(rdev->flags & RADEON_IS_PCIE)) 9700 return; 9701 9702 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9703 data &= ~LC_XMIT_N_FTS_MASK; 9704 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9705 if (orig != data) 9706 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9707 9708 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9709 data |= LC_GO_TO_RECOVERY; 9710 if (orig != data) 9711 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9712 9713 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9714 data |= P_IGNORE_EDB_ERR; 9715 if (orig != data) 9716 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9717 9718 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9719 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9720 data |= LC_PMI_TO_L1_DIS; 9721 if (!disable_l0s) 9722 data |= LC_L0S_INACTIVITY(7); 9723 9724 if (!disable_l1) { 9725 data |= LC_L1_INACTIVITY(7); 9726 data &= ~LC_PMI_TO_L1_DIS; 9727 if (orig != data) 9728 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9729 9730 if (!disable_plloff_in_l1) { 9731 bool clk_req_support; 9732 9733 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9734 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9735 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9736 if (orig != data) 9737 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9738 9739 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9740 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9741 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9742 if (orig != data) 9743 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9744 9745 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9746 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9747 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9748 if (orig != data) 9749 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9750 9751 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9752 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9753 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9754 if (orig != data) 9755 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9756 9757 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9758 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9759 data |= LC_DYN_LANES_PWR_STATE(3); 9760 if (orig != data) 9761 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9762 9763 if (!disable_clkreq && 9764 !pci_is_root_bus(rdev->pdev->bus)) { 9765 struct pci_dev *root = rdev->pdev->bus->self; 9766 u32 lnkcap; 9767 9768 clk_req_support = false; 9769 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9770 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9771 clk_req_support = true; 9772 } else { 9773 clk_req_support = false; 9774 } 9775 9776 if (clk_req_support) { 9777 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9778 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9779 if (orig != data) 9780 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9781 9782 orig = data = RREG32_SMC(THM_CLK_CNTL); 9783 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9784 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9785 if (orig != data) 9786 WREG32_SMC(THM_CLK_CNTL, data); 9787 9788 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9789 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9790 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9791 if (orig != data) 9792 WREG32_SMC(MISC_CLK_CTRL, data); 9793 9794 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9795 data &= ~BCLK_AS_XCLK; 9796 if (orig != data) 9797 WREG32_SMC(CG_CLKPIN_CNTL, data); 9798 9799 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9800 data &= ~FORCE_BIF_REFCLK_EN; 9801 if (orig != data) 9802 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9803 9804 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9805 data &= ~MPLL_CLKOUT_SEL_MASK; 9806 data |= MPLL_CLKOUT_SEL(4); 9807 if (orig != data) 9808 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9809 } 9810 } 9811 } else { 9812 if (orig != data) 9813 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9814 } 9815 9816 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9817 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9818 if (orig != data) 9819 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9820 9821 if (!disable_l0s) { 9822 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9823 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9824 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9825 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9826 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9827 data &= ~LC_L0S_INACTIVITY_MASK; 9828 if (orig != data) 9829 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9830 } 9831 } 9832 } 9833 } 9834